diff --git a/faststream_gen/_code_generator/constants.py b/faststream_gen/_code_generator/constants.py index c4e401f..be43cbf 100644 --- a/faststream_gen/_code_generator/constants.py +++ b/faststream_gen/_code_generator/constants.py @@ -4,12 +4,16 @@ __all__ = ['APPLICATION_FILE_PATH', 'TEST_FILE_PATH', 'TOML_FILE_NAME', 'LOGS_DIR_NAME', 'STEP_LOG_DIR_NAMES', 'DEFAULT_PARAMS', 'MAX_RETRIES', 'MAX_RESTARTS', 'MAX_ASYNC_SPEC_RETRIES', 'TOKEN_TYPES', 'MODEL_PRICING', 'OPENAI_KEY_EMPTY_ERROR', 'OPENAI_KEY_NOT_SET_ERROR', 'EMPTY_DESCRIPTION_ERROR', 'INCOMPLETE_DESCRIPTION', - 'DESCRIPTION_EXAMPLE', 'MAX_NUM_FIXES_MSG', 'INCOMPLETE_APP_ERROR_MSG', 'FASTSTREAM_REPO_ZIP_URL', - 'FASTSTREAM_DOCS_DIR_SUFFIX', 'FASTSTREAM_EXAMPLES_DIR_SUFFIX', 'FASTSTREAM_EXAMPLE_FILES', - 'FASTSTREAM_TMP_DIR_PREFIX', 'FASTSTREAM_DIR_TO_EXCLUDE', 'FASTSTREAM_TEMPLATE_ZIP_URL', + 'DESCRIPTION_EXAMPLE', 'MAX_NUM_FIXES_MSG', 'INCOMPLETE_APP_ERROR_MSG', 'FASTSTREAM_GEN_REPO_ZIP_URL', + 'FASTSTREAM_GEN_EXAMPLES_DIR_SUFFIX', 'FASTSTREAM_REPO_ZIP_URL', 'FASTSTREAM_ROOT_DIR_NAME', + 'FASTSTREAM_DOCS_DIR_SUFFIX', 'FASTSTREAM_EN_DOCS_DIR', 'FASTSTREAM_EXAMPLE_FILES', + 'FASTSTREAM_TMP_DIR_PREFIX', 'FASTSTREAM_DIR_TO_EXCLUDE', 'STAT_0o775', 'FASTSTREAM_TEMPLATE_ZIP_URL', 'FASTSTREAM_TEMPLATE_DIR_SUFFIX', 'OpenAIModel'] -# %% ../../nbs/Constants.ipynb 2 +# %% ../../nbs/Constants.ipynb 1 +import stat + +# %% ../../nbs/Constants.ipynb 3 APPLICATION_FILE_PATH = "app/application.py" TEST_FILE_PATH = "tests/test_application.py" TOML_FILE_NAME = "pyproject.toml" @@ -21,7 +25,7 @@ "requirements": "requirements-generation-logs", } -# %% ../../nbs/Constants.ipynb 4 +# %% ../../nbs/Constants.ipynb 5 DEFAULT_PARAMS = { "temperature": 0.7, } @@ -37,7 +41,7 @@ class OpenAIModel(str, Enum): gpt4 = "gpt-4" -# %% ../../nbs/Constants.ipynb 7 +# %% ../../nbs/Constants.ipynb 8 TOKEN_TYPES = ["prompt_tokens", "completion_tokens", "total_tokens"] MODEL_PRICING = { @@ -51,7 +55,7 @@ class OpenAIModel(str, Enum): }, } -# %% ../../nbs/Constants.ipynb 9 +# %% ../../nbs/Constants.ipynb 10 OPENAI_KEY_EMPTY_ERROR = "Error: OPENAI_API_KEY cannot be empty. Please set a valid OpenAI API key in OPENAI_API_KEY environment variable and try again.\nYou can generate API keys in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details." OPENAI_KEY_NOT_SET_ERROR = "Error: OPENAI_API_KEY not found in environment variables. Set a valid OpenAI API key in OPENAI_API_KEY environment variable and try again. You can generate API keys in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details." @@ -79,14 +83,25 @@ class OpenAIModel(str, Enum): Please run the following command to start manual debugging:""" -# %% ../../nbs/Constants.ipynb 11 +# %% ../../nbs/Constants.ipynb 12 +FASTSTREAM_GEN_REPO_ZIP_URL = "http://github.com/airtai/faststream-gen/archive/main.zip" +FASTSTREAM_GEN_EXAMPLES_DIR_SUFFIX = "faststream-gen-main/search/examples" + FASTSTREAM_REPO_ZIP_URL = "http://github.com/airtai/faststream/archive/main.zip" -FASTSTREAM_DOCS_DIR_SUFFIX = "faststream-main/.faststream_gen" -FASTSTREAM_EXAMPLES_DIR_SUFFIX = "faststream-main/faststream_gen_examples" +FASTSTREAM_ROOT_DIR_NAME = "faststream-main" +FASTSTREAM_DOCS_DIR_SUFFIX = ".faststream_gen" +FASTSTREAM_EN_DOCS_DIR = "docs/docs/en" + FASTSTREAM_EXAMPLE_FILES = ['description.txt', 'app_skeleton.py', 'app.py', 'test_app.py'] FASTSTREAM_TMP_DIR_PREFIX = "appended_examples" FASTSTREAM_DIR_TO_EXCLUDE = "api" # %% ../../nbs/Constants.ipynb 13 +STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP + | stat.S_IROTH | stat.S_IXOTH ) + + +# %% ../../nbs/Constants.ipynb 15 FASTSTREAM_TEMPLATE_ZIP_URL = "http://github.com/airtai/faststream-template/archive/main.zip" FASTSTREAM_TEMPLATE_DIR_SUFFIX = "faststream-template-main" diff --git a/faststream_gen/_code_generator/helper.py b/faststream_gen/_code_generator/helper.py index edbe6d9..3eaf9fa 100644 --- a/faststream_gen/_code_generator/helper.py +++ b/faststream_gen/_code_generator/helper.py @@ -3,8 +3,7 @@ # %% auto 0 __all__ = ['logger', 'examples_delimiter', 'set_cwd', 'set_logger_level', 'retry_on_error', 'ensure_openai_api_key_set', 'add_tokens_usage', 'get_relevant_prompt_examples', 'strip_white_spaces', 'write_file_contents', - 'read_file_contents', 'mock_openai_create', 'download_and_extract_faststream_archive', - 'validate_python_code'] + 'read_file_contents', 'mock_openai_create', 'download_and_extract_github_repo', 'validate_python_code'] # %% ../../nbs/Helper.ipynb 1 from typing import * @@ -308,7 +307,7 @@ def _fetch_content(url: str) -> requests.models.Response: # type: ignore # %% ../../nbs/Helper.ipynb 35 @contextmanager -def download_and_extract_faststream_archive(url: str) -> Generator[Path, None, None]: +def download_and_extract_github_repo(url: str) -> Generator[Path, None, None]: with TemporaryDirectory() as d: try: input_path = Path(f"{d}/archive.zip") diff --git a/faststream_gen/_components/embeddings.py b/faststream_gen/_components/embeddings.py index 3c69d4d..55e020f 100644 --- a/faststream_gen/_components/embeddings.py +++ b/faststream_gen/_components/embeddings.py @@ -6,6 +6,8 @@ # %% ../../nbs/Embeddings_CLI.ipynb 1 from typing import * import shutil +import re +import os from tempfile import TemporaryDirectory from contextlib import contextmanager from pathlib import Path @@ -22,13 +24,16 @@ from faststream_gen._code_generator.constants import ( FASTSTREAM_REPO_ZIP_URL, FASTSTREAM_DOCS_DIR_SUFFIX, - FASTSTREAM_EXAMPLES_DIR_SUFFIX, + FASTSTREAM_GEN_REPO_ZIP_URL, + FASTSTREAM_GEN_EXAMPLES_DIR_SUFFIX, FASTSTREAM_EXAMPLE_FILES, FASTSTREAM_TMP_DIR_PREFIX, - FASTSTREAM_DIR_TO_EXCLUDE + FASTSTREAM_DIR_TO_EXCLUDE, + FASTSTREAM_ROOT_DIR_NAME, + FASTSTREAM_EN_DOCS_DIR, ) from .package_data import get_root_data_path -from .._code_generator.helper import download_and_extract_faststream_archive +from .._code_generator.helper import download_and_extract_github_repo # %% ../../nbs/Embeddings_CLI.ipynb 3 def _create_documents( @@ -119,6 +124,82 @@ def _delete_directory(d: str) -> None: print(f"Error deleting directory: {e}") # %% ../../nbs/Embeddings_CLI.ipynb 11 +def _read_lines_from_file(file_path: Path, lines_spec: str) -> str: + with open(file_path, "r") as file: + all_lines = file.readlines() + + # Check if lines_spec is empty (indicating all lines should be read) + if not lines_spec: + return "".join(all_lines) + + selected_lines = [] + line_specs = lines_spec.split(",") + + for line_spec in line_specs: + if "-" in line_spec: + # Handle line ranges (e.g., "1-10") + start, end = map(int, line_spec.split("-")) + selected_lines.extend(all_lines[start - 1 : end]) + else: + # Handle single line numbers + line_number = int(line_spec) + if 1 <= line_number <= len(all_lines): + selected_lines.append(all_lines[line_number - 1]) + + return "".join(selected_lines) + + +def _extract_lines(embedded_line: str, root_path: Path) -> str: + to_expand_path = re.search("{!>(.*)!}", embedded_line).group(1).strip() # type: ignore + lines_spec = "" + if "[ln:" in to_expand_path: + to_expand_path, lines_spec = to_expand_path.split("[ln:") + to_expand_path = to_expand_path.strip() + lines_spec = lines_spec[:-1] + + if Path(f"{root_path}/docs/docs_src").exists(): + to_expand_path = Path(f"{root_path}/docs") / to_expand_path + elif Path(f"{root_path}/docs_src").exists(): + to_expand_path = Path(f"{root_path}/") / to_expand_path + else: + raise ValueError(f"Couldn't find docs_src directory") + return _read_lines_from_file(to_expand_path, lines_spec) + + +def _expand_markdown( + input_markdown_path: Path, + output_markdown_path: Path, + root_path: Path +) -> None: + with open(input_markdown_path, "r") as input_file, open( + output_markdown_path, "w" + ) as output_file: + for line in input_file: + # Check if the line does not contain the "{!>" pattern + if "{!>" not in line: + # Write the line to the output file + output_file.write(line) + else: + output_file.write(_extract_lines(embedded_line=line, root_path=root_path)) + +# %% ../../nbs/Embeddings_CLI.ipynb 12 +def _expand_faststream_docs(root_path: Path) -> None: + docs_suffix = root_path / FASTSTREAM_DOCS_DIR_SUFFIX + docs_suffix.mkdir(exist_ok=True) + md_files = (root_path / FASTSTREAM_EN_DOCS_DIR).glob("**/*.md") + + def expand_doc(input_path: Path) -> None: + relative_path = os.path.relpath(input_path, docs_suffix) + output_path = docs_suffix / relative_path.replace("../docs/docs/en/", "") + output_path.parent.mkdir(parents=True, exist_ok=True) + _expand_markdown( + input_markdown_path=input_path, output_markdown_path=output_path, root_path=root_path + ) + + for md_file in md_files: + expand_doc(md_file) + +# %% ../../nbs/Embeddings_CLI.ipynb 14 def _generate_docs_db(input_path: Path, output_path: Path) -> None: """Generate Document Embeddings Database. @@ -127,20 +208,22 @@ def _generate_docs_db(input_path: Path, output_path: Path) -> None: to the specified output directory. Args: - input_path (Path): The path to the directory containing input documents. + input_path (Path): The path to the directory containing the extracted files. output_path (Path): The path to the directory where the embeddings database will be saved. """ with yaspin( text="Creating embeddings for the docs...", color="cyan", spinner="clock" ) as sp: - docs = _create_documents(input_path) + _expand_faststream_docs(input_path / FASTSTREAM_ROOT_DIR_NAME) + + docs = _create_documents(input_path / FASTSTREAM_ROOT_DIR_NAME / FASTSTREAM_DOCS_DIR_SUFFIX) _save_embeddings_db(docs, output_path) sp.text = "" sp.ok(f" ✔ Docs embeddings created and saved to: {output_path}") -# %% ../../nbs/Embeddings_CLI.ipynb 13 +# %% ../../nbs/Embeddings_CLI.ipynb 16 def _check_all_files_exist(d: Path, required_files: List[str]) -> bool: """Check if all required files exist in a directory. @@ -155,7 +238,7 @@ def _check_all_files_exist(d: Path, required_files: List[str]) -> bool: """ return all((d / file_name).exists() for file_name in required_files) -# %% ../../nbs/Embeddings_CLI.ipynb 16 +# %% ../../nbs/Embeddings_CLI.ipynb 19 def _append_file_contents(d: Path, parent_d: Path, required_files: List[str]) -> None: """Append contents of specified files to a result file. @@ -179,7 +262,7 @@ def _append_file_contents(d: Path, parent_d: Path, required_files: List[str]) -> f"==== {file_name} starts ====\n{file.read()}\n==== {file_name} ends ====\n" ) -# %% ../../nbs/Embeddings_CLI.ipynb 18 +# %% ../../nbs/Embeddings_CLI.ipynb 21 def _format_examples(input_path: Path, required_files: List[str]) -> None: """Format Examples by Appending File Contents. @@ -237,12 +320,12 @@ def _generate_examples_db( sp.text = "" sp.ok(f" ✔ Examples embeddings created and saved to: {output_path}") -# %% ../../nbs/Embeddings_CLI.ipynb 20 +# %% ../../nbs/Embeddings_CLI.ipynb 23 app = typer.Typer( short_help="Download the zipped FastKafka documentation markdown files, generate embeddings, and save them in a vector database.", ) -# %% ../../nbs/Embeddings_CLI.ipynb 21 +# %% ../../nbs/Embeddings_CLI.ipynb 24 @app.command( "generate", help="Download the docs and examples from FastStream repo, generate embeddings, and save them in a vector database.", @@ -256,26 +339,30 @@ def generate( ) ) -> None: typer.echo( - f"Downloading files docs and examples from FastStream repo and generating embeddings." + f"Downloading documentation and examples for semantic search." ) + try: + _delete_directory(db_path) - with download_and_extract_faststream_archive( - FASTSTREAM_REPO_ZIP_URL - ) as extracted_path: - try: - _delete_directory(db_path) + with download_and_extract_github_repo( + FASTSTREAM_REPO_ZIP_URL + ) as extracted_path: _generate_docs_db( - extracted_path / FASTSTREAM_DOCS_DIR_SUFFIX, Path(db_path) / "docs" + extracted_path, Path(db_path) / "docs" ) + + with download_and_extract_github_repo( + FASTSTREAM_GEN_REPO_ZIP_URL + ) as extracted_path: _generate_examples_db( - extracted_path / FASTSTREAM_EXAMPLES_DIR_SUFFIX, + extracted_path / FASTSTREAM_GEN_EXAMPLES_DIR_SUFFIX, Path(db_path) / "examples", ) - typer.echo( - f"\nSuccessfully generated all the embeddings and saved to: {db_path}" - ) - except Exception as e: - fg = typer.colors.RED - typer.secho(f"Unexpected internal error: {e}", err=True, fg=fg) - raise typer.Exit(code=1) + typer.echo( + f"\nSuccessfully generated all the embeddings and saved to: {db_path}" + ) + except Exception as e: + fg = typer.colors.RED + typer.secho(f"Unexpected internal error: {e}", err=True, fg=fg) + raise typer.Exit(code=1) diff --git a/faststream_gen/_components/integration_test_generator.py b/faststream_gen/_components/integration_test_generator.py index 7b0272c..8bcb23a 100644 --- a/faststream_gen/_components/integration_test_generator.py +++ b/faststream_gen/_components/integration_test_generator.py @@ -31,7 +31,6 @@ from .._code_generator.chat import CustomAIChat, ValidateAndFixResponse from faststream_gen._code_generator.helper import ( - download_and_extract_faststream_archive, write_file_contents, read_file_contents, set_cwd, diff --git a/faststream_gen/_components/new_project_generator.py b/faststream_gen/_components/new_project_generator.py index e699102..43f8808 100644 --- a/faststream_gen/_components/new_project_generator.py +++ b/faststream_gen/_components/new_project_generator.py @@ -9,34 +9,29 @@ from yaspin import yaspin import shutil import os -import stat from faststream_gen._code_generator.helper import ( - download_and_extract_faststream_archive, + download_and_extract_github_repo, ) from faststream_gen._code_generator.constants import ( FASTSTREAM_TEMPLATE_ZIP_URL, FASTSTREAM_TEMPLATE_DIR_SUFFIX, APPLICATION_FILE_PATH, TEST_FILE_PATH, + STAT_0o775 ) from .._code_generator.helper import write_file_contents # %% ../../nbs/New_Project_Generator.ipynb 3 -_STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR - | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP - | stat.S_IROTH | stat.S_IXOTH ) - - def create_project( output_path: str, ) -> None: with yaspin( text="Creating a new FastStream project...", color="cyan", spinner="clock" ) as sp: - with download_and_extract_faststream_archive( + with download_and_extract_github_repo( FASTSTREAM_TEMPLATE_ZIP_URL ) as extracted_path: app_file = str( @@ -51,7 +46,7 @@ def create_project( for p in ( Path(extracted_path) / FASTSTREAM_TEMPLATE_DIR_SUFFIX / "scripts" ).glob("*.sh"): - p.chmod(_STAT_0o775) + p.chmod(STAT_0o775) shutil.copytree( str(extracted_path / FASTSTREAM_TEMPLATE_DIR_SUFFIX), diff --git a/faststream_gen/_modidx.py b/faststream_gen/_modidx.py index 1c339e9..e1e7819 100644 --- a/faststream_gen/_modidx.py +++ b/faststream_gen/_modidx.py @@ -55,8 +55,8 @@ 'faststream_gen/_code_generator/helper.py'), 'faststream_gen._code_generator.helper.add_tokens_usage': ( 'helper.html#add_tokens_usage', 'faststream_gen/_code_generator/helper.py'), - 'faststream_gen._code_generator.helper.download_and_extract_faststream_archive': ( 'helper.html#download_and_extract_faststream_archive', - 'faststream_gen/_code_generator/helper.py'), + 'faststream_gen._code_generator.helper.download_and_extract_github_repo': ( 'helper.html#download_and_extract_github_repo', + 'faststream_gen/_code_generator/helper.py'), 'faststream_gen._code_generator.helper.ensure_openai_api_key_set': ( 'helper.html#ensure_openai_api_key_set', 'faststream_gen/_code_generator/helper.py'), 'faststream_gen._code_generator.helper.get_relevant_prompt_examples': ( 'helper.html#get_relevant_prompt_examples', @@ -86,12 +86,20 @@ 'faststream_gen/_components/embeddings.py'), 'faststream_gen._components.embeddings._delete_directory': ( 'embeddings_cli.html#_delete_directory', 'faststream_gen/_components/embeddings.py'), + 'faststream_gen._components.embeddings._expand_faststream_docs': ( 'embeddings_cli.html#_expand_faststream_docs', + 'faststream_gen/_components/embeddings.py'), + 'faststream_gen._components.embeddings._expand_markdown': ( 'embeddings_cli.html#_expand_markdown', + 'faststream_gen/_components/embeddings.py'), + 'faststream_gen._components.embeddings._extract_lines': ( 'embeddings_cli.html#_extract_lines', + 'faststream_gen/_components/embeddings.py'), 'faststream_gen._components.embeddings._format_examples': ( 'embeddings_cli.html#_format_examples', 'faststream_gen/_components/embeddings.py'), 'faststream_gen._components.embeddings._generate_docs_db': ( 'embeddings_cli.html#_generate_docs_db', 'faststream_gen/_components/embeddings.py'), 'faststream_gen._components.embeddings._generate_examples_db': ( 'embeddings_cli.html#_generate_examples_db', 'faststream_gen/_components/embeddings.py'), + 'faststream_gen._components.embeddings._read_lines_from_file': ( 'embeddings_cli.html#_read_lines_from_file', + 'faststream_gen/_components/embeddings.py'), 'faststream_gen._components.embeddings._save_embeddings_db': ( 'embeddings_cli.html#_save_embeddings_db', 'faststream_gen/_components/embeddings.py'), 'faststream_gen._components.embeddings._split_document_into_chunks': ( 'embeddings_cli.html#_split_document_into_chunks', diff --git a/faststream_gen/package_data/docs/index.faiss b/faststream_gen/package_data/docs/index.faiss index 93d5651..c3b1fdd 100644 Binary files a/faststream_gen/package_data/docs/index.faiss and b/faststream_gen/package_data/docs/index.faiss differ diff --git a/faststream_gen/package_data/docs/index.pkl b/faststream_gen/package_data/docs/index.pkl index 8200955..6aa1cb0 100644 Binary files a/faststream_gen/package_data/docs/index.pkl and b/faststream_gen/package_data/docs/index.pkl differ diff --git a/faststream_gen/package_data/examples/index.faiss b/faststream_gen/package_data/examples/index.faiss index 8766244..2830bef 100644 Binary files a/faststream_gen/package_data/examples/index.faiss and b/faststream_gen/package_data/examples/index.faiss differ diff --git a/faststream_gen/package_data/examples/index.pkl b/faststream_gen/package_data/examples/index.pkl index cfe2381..6f4bc0b 100644 Binary files a/faststream_gen/package_data/examples/index.pkl and b/faststream_gen/package_data/examples/index.pkl differ diff --git a/nbs/Constants.ipynb b/nbs/Constants.ipynb index c578841..8071441 100644 --- a/nbs/Constants.ipynb +++ b/nbs/Constants.ipynb @@ -10,6 +10,18 @@ "# | default_exp _code_generator.constants" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac6196d0", + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "\n", + "import stat" + ] + }, { "cell_type": "markdown", "id": "8b3de9f5", @@ -183,14 +195,33 @@ "source": [ "# | export\n", "\n", + "FASTSTREAM_GEN_REPO_ZIP_URL = \"http://github.com/airtai/faststream-gen/archive/main.zip\"\n", + "FASTSTREAM_GEN_EXAMPLES_DIR_SUFFIX = \"faststream-gen-main/search/examples\"\n", + "\n", "FASTSTREAM_REPO_ZIP_URL = \"http://github.com/airtai/faststream/archive/main.zip\"\n", - "FASTSTREAM_DOCS_DIR_SUFFIX = \"faststream-main/.faststream_gen\"\n", - "FASTSTREAM_EXAMPLES_DIR_SUFFIX = \"faststream-main/faststream_gen_examples\"\n", + "FASTSTREAM_ROOT_DIR_NAME = \"faststream-main\"\n", + "FASTSTREAM_DOCS_DIR_SUFFIX = \".faststream_gen\"\n", + "FASTSTREAM_EN_DOCS_DIR = \"docs/docs/en\"\n", + "\n", "FASTSTREAM_EXAMPLE_FILES = ['description.txt', 'app_skeleton.py', 'app.py', 'test_app.py']\n", "FASTSTREAM_TMP_DIR_PREFIX = \"appended_examples\"\n", "FASTSTREAM_DIR_TO_EXCLUDE = \"api\"" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "c22be41e", + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "\n", + "STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR\n", + " | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP\n", + " | stat.S_IROTH | stat.S_IXOTH )\n" + ] + }, { "cell_type": "markdown", "id": "5a403855", diff --git a/nbs/Embeddings_CLI.ipynb b/nbs/Embeddings_CLI.ipynb index d9a9b6f..d98b8be 100644 --- a/nbs/Embeddings_CLI.ipynb +++ b/nbs/Embeddings_CLI.ipynb @@ -21,6 +21,8 @@ "\n", "from typing import *\n", "import shutil\n", + "import re\n", + "import os\n", "from tempfile import TemporaryDirectory\n", "from contextlib import contextmanager\n", "from pathlib import Path\n", @@ -37,13 +39,16 @@ "from faststream_gen._code_generator.constants import (\n", " FASTSTREAM_REPO_ZIP_URL,\n", " FASTSTREAM_DOCS_DIR_SUFFIX,\n", - " FASTSTREAM_EXAMPLES_DIR_SUFFIX,\n", + " FASTSTREAM_GEN_REPO_ZIP_URL,\n", + " FASTSTREAM_GEN_EXAMPLES_DIR_SUFFIX,\n", " FASTSTREAM_EXAMPLE_FILES,\n", " FASTSTREAM_TMP_DIR_PREFIX,\n", - " FASTSTREAM_DIR_TO_EXCLUDE\n", + " FASTSTREAM_DIR_TO_EXCLUDE,\n", + " FASTSTREAM_ROOT_DIR_NAME,\n", + " FASTSTREAM_EN_DOCS_DIR,\n", ")\n", "from faststream_gen._components.package_data import get_root_data_path\n", - "from faststream_gen._code_generator.helper import download_and_extract_faststream_archive" + "from faststream_gen._code_generator.helper import download_and_extract_github_repo" ] }, { @@ -53,6 +58,8 @@ "metadata": {}, "outputs": [], "source": [ + "import pytest\n", + "\n", "from typer.testing import CliRunner" ] }, @@ -251,9 +258,9 @@ "output_type": "stream", "text": [ "total 12\r\n", - "drwx------ 3 harish harish 4096 Sep 26 11:09 .\r\n", - "drwxrwxrwt 1 root root 4096 Sep 26 11:09 ..\r\n", - "drwxrwxr-x 2 harish harish 4096 Sep 26 11:09 faiss_index\r\n" + "drwx------ 3 harish harish 4096 Oct 10 08:17 .\r\n", + "drwxrwxrwt 1 root root 4096 Oct 10 08:17 ..\r\n", + "drwxrwxr-x 2 harish harish 4096 Oct 10 08:17 faiss_index\r\n" ] } ], @@ -300,12 +307,12 @@ "output_type": "stream", "text": [ "total 12\n", - "drwx------ 3 harish harish 4096 Sep 26 11:09 .\n", - "drwxrwxrwt 1 root root 4096 Sep 26 11:09 ..\n", - "drwxrwxr-x 2 harish harish 4096 Sep 26 11:09 some_dir\n", + "drwx------ 3 harish harish 4096 Oct 10 08:17 .\n", + "drwxrwxrwt 1 root root 4096 Oct 10 08:17 ..\n", + "drwxrwxr-x 2 harish harish 4096 Oct 10 08:17 some_dir\n", "total 8\n", - "drwx------ 2 harish harish 4096 Sep 26 11:09 .\n", - "drwxrwxrwt 1 root root 4096 Sep 26 11:09 ..\n" + "drwx------ 2 harish harish 4096 Oct 10 08:17 .\n", + "drwxrwxrwt 1 root root 4096 Oct 10 08:17 ..\n" ] } ], @@ -324,6 +331,145 @@ " !ls -la {d}" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f4beb86", + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "\n", + "\n", + "def _read_lines_from_file(file_path: Path, lines_spec: str) -> str:\n", + " with open(file_path, \"r\") as file:\n", + " all_lines = file.readlines()\n", + "\n", + " # Check if lines_spec is empty (indicating all lines should be read)\n", + " if not lines_spec:\n", + " return \"\".join(all_lines)\n", + "\n", + " selected_lines = []\n", + " line_specs = lines_spec.split(\",\")\n", + "\n", + " for line_spec in line_specs:\n", + " if \"-\" in line_spec:\n", + " # Handle line ranges (e.g., \"1-10\")\n", + " start, end = map(int, line_spec.split(\"-\"))\n", + " selected_lines.extend(all_lines[start - 1 : end])\n", + " else:\n", + " # Handle single line numbers\n", + " line_number = int(line_spec)\n", + " if 1 <= line_number <= len(all_lines):\n", + " selected_lines.append(all_lines[line_number - 1])\n", + "\n", + " return \"\".join(selected_lines)\n", + "\n", + "\n", + "def _extract_lines(embedded_line: str, root_path: Path) -> str:\n", + " to_expand_path = re.search(\"{!>(.*)!}\", embedded_line).group(1).strip() # type: ignore \n", + " lines_spec = \"\"\n", + " if \"[ln:\" in to_expand_path:\n", + " to_expand_path, lines_spec = to_expand_path.split(\"[ln:\")\n", + " to_expand_path = to_expand_path.strip()\n", + " lines_spec = lines_spec[:-1]\n", + "\n", + " if Path(f\"{root_path}/docs/docs_src\").exists():\n", + " to_expand_path = Path(f\"{root_path}/docs\") / to_expand_path\n", + " elif Path(f\"{root_path}/docs_src\").exists():\n", + " to_expand_path = Path(f\"{root_path}/\") / to_expand_path\n", + " else:\n", + " raise ValueError(f\"Couldn't find docs_src directory\")\n", + " return _read_lines_from_file(to_expand_path, lines_spec)\n", + "\n", + "\n", + "def _expand_markdown(\n", + " input_markdown_path: Path,\n", + " output_markdown_path: Path,\n", + " root_path: Path\n", + ") -> None:\n", + " with open(input_markdown_path, \"r\") as input_file, open(\n", + " output_markdown_path, \"w\"\n", + " ) as output_file:\n", + " for line in input_file:\n", + " # Check if the line does not contain the \"{!>\" pattern\n", + " if \"{!>\" not in line:\n", + " # Write the line to the output file\n", + " output_file.write(line)\n", + " else:\n", + " output_file.write(_extract_lines(embedded_line=line, root_path=root_path))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "871cc1b4", + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "\n", + "\n", + "def _expand_faststream_docs(root_path: Path) -> None:\n", + " docs_suffix = root_path / FASTSTREAM_DOCS_DIR_SUFFIX\n", + " docs_suffix.mkdir(exist_ok=True)\n", + " md_files = (root_path / FASTSTREAM_EN_DOCS_DIR).glob(\"**/*.md\")\n", + "\n", + " def expand_doc(input_path: Path) -> None:\n", + " relative_path = os.path.relpath(input_path, docs_suffix)\n", + " output_path = docs_suffix / relative_path.replace(\"../docs/docs/en/\", \"\")\n", + " output_path.parent.mkdir(parents=True, exist_ok=True)\n", + " _expand_markdown(\n", + " input_markdown_path=input_path, output_markdown_path=output_path, root_path=root_path\n", + " )\n", + "\n", + " for md_file in md_files:\n", + " expand_doc(md_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac08ba92", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 12\r\n", + "drwxrwxr-x 2 harish harish 4096 Oct 10 08:17 .\r\n", + "drwx------ 4 harish harish 4096 Oct 10 08:17 ..\r\n", + "-rw-rw-r-- 1 harish harish 7 Oct 10 08:17 sample.md\r\n" + ] + } + ], + "source": [ + "fixture = \"\"\"\n", + "def expand_markdown(input_markdown_path, output_markdown_path):\n", + " pass\n", + "\"\"\"\n", + "\n", + "with TemporaryDirectory() as d:\n", + " docs_dir = Path(d) / \"docs\" / \"docs\" / \"en\"\n", + " docs_dir.mkdir(parents=True, exist_ok=True)\n", + " \n", + " fixture_md_file = docs_dir / \"sample.md\"\n", + " \n", + " with open(fixture_md_file, \"w\") as f:\n", + " f.write(\"# Hello\")\n", + " \n", + " fixture_expand_markdown_file = Path(d) / \"docs\" / \"expand_markdown.py\"\n", + " with open(fixture_expand_markdown_file, \"w\") as f:\n", + " f.write(fixture)\n", + "\n", + " _expand_faststream_docs(Path(d))\n", + " assert (Path(d) / FASTSTREAM_DOCS_DIR_SUFFIX).exists()\n", + " \n", + " !ls -la {d}/{FASTSTREAM_DOCS_DIR_SUFFIX}\n", + " assert (Path(d) / FASTSTREAM_DOCS_DIR_SUFFIX/ \"sample.md\").exists()" + ] + }, { "cell_type": "code", "execution_count": null, @@ -342,14 +488,16 @@ " to the specified output directory.\n", "\n", " Args:\n", - " input_path (Path): The path to the directory containing input documents.\n", + " input_path (Path): The path to the directory containing the extracted files.\n", " output_path (Path): The path to the directory where the embeddings\n", " database will be saved.\n", " \"\"\"\n", " with yaspin(\n", " text=\"Creating embeddings for the docs...\", color=\"cyan\", spinner=\"clock\"\n", " ) as sp:\n", - " docs = _create_documents(input_path)\n", + " _expand_faststream_docs(input_path / FASTSTREAM_ROOT_DIR_NAME)\n", + "\n", + " docs = _create_documents(input_path / FASTSTREAM_ROOT_DIR_NAME / FASTSTREAM_DOCS_DIR_SUFFIX) \n", " _save_embeddings_db(docs, output_path)\n", "\n", " sp.text = \"\"\n", @@ -384,21 +532,29 @@ "name": "stdout", "output_type": "stream", "text": [ - " ✔ Docs embeddings created and saved to: /tmp/tmp5sx47e5m/output_path \n" + " ✔ Docs embeddings created and saved to: /tmp/tmpwjfc8_fo/faststream-main/output_path \n" ] } ], "source": [ "with TemporaryDirectory() as d:\n", - " input_path = Path(d) / \"input_path\"\n", - " input_path.mkdir(parents=True, exist_ok=True)\n", - " output_path = Path(d) / \"output_path\"\n", - " output_path.mkdir(parents=True, exist_ok=True)\n", + " root_dir = Path(d) / \"faststream-main\"\n", + " root_dir.mkdir(parents=True, exist_ok=True)\n", " \n", - " with open(f\"{input_path}/sample.md\", \"w\") as f:\n", - " f.write(\"# Hello world!\")\n", + " docs_dir = root_dir / \"docs\" / \"docs\" / \"en\"\n", + " docs_dir.mkdir(parents=True, exist_ok=True)\n", " \n", - " _generate_docs_db(input_path, output_path)\n", + " with open(f\"{docs_dir}/sample.md\", \"w\") as f:\n", + " f.write(\"# Hello\")\n", + " \n", + " fixture_expand_markdown_file = root_dir / \"docs\" / \"expand_markdown.py\"\n", + " with open(fixture_expand_markdown_file, \"w\") as f:\n", + " f.write(fixture)\n", + " \n", + " output_path = root_dir / \"output_path\"\n", + " output_path.mkdir(parents=True, exist_ok=True)\n", + " \n", + " _generate_docs_db(Path(d), output_path)\n", " \n", " assert (output_path / \"index.faiss\").exists()" ] @@ -679,11 +835,11 @@ "output_type": "stream", "text": [ "⠋ Creating embeddings for the examples...\n", - "Required files are missing. Skipping directory: /tmp/tmpdetstdh2/output_path\n", + "Required files are missing. Skipping directory: /tmp/tmprbobxbd0/output_path\n", "\n", "Below files are included in the embeddings:\n", " - example_1.txt\n", - " ✔ Examples embeddings created and saved to: /tmp/tmpdetstdh2/output_path \n" + " ✔ Examples embeddings created and saved to: /tmp/tmprbobxbd0/output_path \n" ] } ], @@ -740,29 +896,33 @@ " )\n", ") -> None:\n", " typer.echo(\n", - " f\"Downloading files docs and examples from FastStream repo and generating embeddings.\"\n", + " f\"Downloading documentation and examples for semantic search.\"\n", " )\n", + " try:\n", + " _delete_directory(db_path)\n", "\n", - " with download_and_extract_faststream_archive(\n", - " FASTSTREAM_REPO_ZIP_URL\n", - " ) as extracted_path:\n", - " try:\n", - " _delete_directory(db_path)\n", + " with download_and_extract_github_repo(\n", + " FASTSTREAM_REPO_ZIP_URL\n", + " ) as extracted_path:\n", " _generate_docs_db(\n", - " extracted_path / FASTSTREAM_DOCS_DIR_SUFFIX, Path(db_path) / \"docs\"\n", + " extracted_path, Path(db_path) / \"docs\"\n", " )\n", + "\n", + " with download_and_extract_github_repo(\n", + " FASTSTREAM_GEN_REPO_ZIP_URL\n", + " ) as extracted_path:\n", " _generate_examples_db(\n", - " extracted_path / FASTSTREAM_EXAMPLES_DIR_SUFFIX,\n", + " extracted_path / FASTSTREAM_GEN_EXAMPLES_DIR_SUFFIX,\n", " Path(db_path) / \"examples\",\n", " )\n", "\n", - " typer.echo(\n", - " f\"\\nSuccessfully generated all the embeddings and saved to: {db_path}\"\n", - " )\n", - " except Exception as e:\n", - " fg = typer.colors.RED\n", - " typer.secho(f\"Unexpected internal error: {e}\", err=True, fg=fg)\n", - " raise typer.Exit(code=1)" + " typer.echo(\n", + " f\"\\nSuccessfully generated all the embeddings and saved to: {db_path}\"\n", + " )\n", + " except Exception as e:\n", + " fg = typer.colors.RED\n", + " typer.secho(f\"Unexpected internal error: {e}\", err=True, fg=fg)\n", + " raise typer.Exit(code=1)" ] }, { @@ -846,7 +1006,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Downloading files docs and examples from FastStream repo and generating embeddings.\n", + "Downloading documentation and examples for semantic search.\n", "\r", "\r", "\r", @@ -883,10 +1043,26 @@ "⠋ Creating embeddings for the docs...\r", " \r", "\r", - "⠙ Creating embeddings for the docs...\n", + "⠙ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠹ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠸ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠼ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠴ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠦ Creating embeddings for the docs...\n", "Below files are included in the embeddings:\n", " - index.md\n", " - release.md\n", + " - kafka/ack.md\n", " - kafka/index.md\n", " - kafka/message.md\n", " - kafka/Publisher/batch_publisher.md\n", @@ -931,11 +1107,22 @@ " - getting-started/publishing/broker.md\n", " - getting-started/publishing/test.md\n", " - getting-started/routers/index.md\n", + " - getting-started/template/index.md\n", " - getting-started/contributing/CONTRIBUTING.md\n", " - getting-started/contributing/docs.md\n", " - getting-started/lifespan/index.md\n", " - getting-started/lifespan/test.md\n", " - getting-started/lifespan/hooks.md\n", + " - nats/index.md\n", + " - nats/rpc.md\n", + " - nats/message.md\n", + " - nats/publishing/index.md\n", + " - nats/examples/direct.md\n", + " - nats/examples/pattern.md\n", + " - nats/jetstream/object.md\n", + " - nats/jetstream/ack.md\n", + " - nats/jetstream/key-value.md\n", + " - nats/jetstream/index.md\n", " - rabbit/ack.md\n", " - rabbit/publishing.md\n", " - rabbit/index.md\n", @@ -951,6 +1138,51 @@ "\r", " \r", "\r", + "⠧ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠇ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠏ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠋ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠙ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠹ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠸ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠼ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠴ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠦ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠧ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠇ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠏ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠋ Creating embeddings for the docs...\r", + " \r", + "\r", + "⠙ Creating embeddings for the docs...\r", + " \r", + "\r", "⠹ Creating embeddings for the docs...\r", " \r", "\r", @@ -1052,12 +1284,15 @@ "\r", "⠴ Creating embeddings for the docs...\r", " \r", - " ✔ Docs embeddings created and saved to: /tmp/tmpqz_2srur/docs \n", + "\r", + "⠦ Creating embeddings for the docs...\r", + " \r", + " ✔ Docs embeddings created and saved to: /tmp/tmp70b1gjwi/docs \n", "\r", "\r", "\r", "⠋ Creating embeddings for the examples...\n", - "Required files are missing. Skipping directory: /tmp/tmpg4sidt3a/extrated_path/faststream-main/faststream_gen_examples/__init__.py\n", + "Required files are missing. Skipping directory: /tmp/tmpcprm018z/extrated_path/faststream-gen-main/search/examples/__init__.py\n", "\n", "Below files are included in the embeddings:\n", " - example_add_and_publish_with_key.txt\n", @@ -1124,9 +1359,15 @@ "\r", "⠸ Creating embeddings for the examples...\r", " \r", - " ✔ Examples embeddings created and saved to: /tmp/tmpqz_2srur/examples \n", + "\r", + "⠼ Creating embeddings for the examples...\r", + " \r", + "\r", + "⠴ Creating embeddings for the examples...\r", + " \r", + " ✔ Examples embeddings created and saved to: /tmp/tmp70b1gjwi/examples \n", "\n", - "Successfully generated all the embeddings and saved to: /tmp/tmpqz_2srur\n", + "Successfully generated all the embeddings and saved to: /tmp/tmp70b1gjwi\n", "\n" ] } diff --git a/nbs/Helper.ipynb b/nbs/Helper.ipynb index cbb031a..ea8aa06 100644 --- a/nbs/Helper.ipynb +++ b/nbs/Helper.ipynb @@ -62,7 +62,7 @@ "import sys\n", "from unittest.mock import patch\n", "\n", - "from faststream_gen._code_generator.constants import FASTSTREAM_DOCS_DIR_SUFFIX, FASTSTREAM_REPO_ZIP_URL, OpenAIModel\n", + "from faststream_gen._code_generator.constants import FASTSTREAM_DOCS_DIR_SUFFIX, FASTSTREAM_REPO_ZIP_URL, OpenAIModel, FASTSTREAM_ROOT_DIR_NAME\n", "\n", "import pytest\n", "import openai" @@ -926,7 +926,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "/tmp/tmp48rete8s/grand-parent/parent/child/application.py\n", + "/tmp/tmp4492dn2w/grand-parent/parent/child/application.py\n", "\n", "\n", "print(\"Hello World\")\n", @@ -995,7 +995,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "/tmp/tmpqz3iy293/grand-parent/parent/child/application.py\n", + "/tmp/tmpv8rje5w0/grand-parent/parent/child/application.py\n", "\n", "\n", "print(\"Hello World\")\n", @@ -1031,7 +1031,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\n" + "\n" ] } ], @@ -1171,7 +1171,7 @@ "\n", "\n", "@contextmanager\n", - "def download_and_extract_faststream_archive(url: str) -> Generator[Path, None, None]:\n", + "def download_and_extract_github_repo(url: str) -> Generator[Path, None, None]:\n", " with TemporaryDirectory() as d:\n", " try:\n", " input_path = Path(f\"{d}/archive.zip\")\n", @@ -1205,13 +1205,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "['api', 'kafka', 'getting-started', 'index', 'release', 'rabbit']\n" + "['api', 'kafka', 'getting-started', 'index', 'release', 'nats', 'rabbit']\n" ] } ], "source": [ - "with download_and_extract_faststream_archive(FASTSTREAM_REPO_ZIP_URL) as extracted_path:\n", - " files = [p.stem for p in list(Path(extracted_path/FASTSTREAM_DOCS_DIR_SUFFIX).glob(\"*\"))]\n", + "with download_and_extract_github_repo(FASTSTREAM_REPO_ZIP_URL) as extracted_path:\n", + " files = [p.stem for p in list(Path(extracted_path/FASTSTREAM_ROOT_DIR_NAME/FASTSTREAM_DOCS_DIR_SUFFIX).glob(\"*\"))]\n", " print(files)\n", " assert \"index\" in files" ] diff --git a/nbs/Integration_Test_Generator.ipynb b/nbs/Integration_Test_Generator.ipynb index 457f478..d1ab85d 100644 --- a/nbs/Integration_Test_Generator.ipynb +++ b/nbs/Integration_Test_Generator.ipynb @@ -47,7 +47,6 @@ "from faststream_gen._code_generator.chat import CustomAIChat, ValidateAndFixResponse\n", "\n", "from faststream_gen._code_generator.helper import (\n", - " download_and_extract_faststream_archive,\n", " write_file_contents,\n", " read_file_contents,\n", " set_cwd,\n", @@ -223,8 +222,8 @@ "output_type": "stream", "text": [ "\u001b[1m============================= test session starts ==============================\u001b[0m\n", - "platform linux -- Python 3.11.5, pytest-7.4.2, pluggy-1.3.0\n", - "rootdir: /tmp/tmpsvameqy_\n", + "platform linux -- Python 3.11.4, pytest-7.4.2, pluggy-1.3.0\n", + "rootdir: /tmp/tmplojftuue\n", "configfile: pyproject.toml\n", "plugins: anyio-3.7.1, asyncio-0.21.1\n", "asyncio: mode=Mode.STRICT\n", @@ -232,7 +231,7 @@ "\n", "==================================== ERRORS ====================================\n", "\u001b[31m\u001b[1m__________________ ERROR collecting tests/test_application.py __________________\u001b[0m\n", - "\u001b[31mImportError while importing test module '/tmp/tmpsvameqy_/tests/test_application.py'.\n", + "\u001b[31mImportError while importing test module '/tmp/tmplojftuue/tests/test_application.py'.\n", "Hint: make sure your test modules/packages have valid Python names.\n", "Traceback:\n", "\u001b[1m\u001b[31m/usr/lib/python3.11/importlib/__init__.py\u001b[0m:126: in import_module\n", @@ -612,8 +611,8 @@ "output_type": "stream", "text": [ "\u001b[1m============================= test session starts ==============================\u001b[0m\n", - "platform linux -- Python 3.11.5, pytest-7.4.2, pluggy-1.3.0\n", - "rootdir: /tmp/tmpsgqeroqd\n", + "platform linux -- Python 3.11.4, pytest-7.4.2, pluggy-1.3.0\n", + "rootdir: /tmp/tmpl0aw7_87\n", "configfile: pyproject.toml\n", "plugins: anyio-3.7.1, asyncio-0.21.1\n", "asyncio: mode=Mode.STRICT\n", diff --git a/nbs/New_Project_Generator.ipynb b/nbs/New_Project_Generator.ipynb index fd13ee9..8ab7553 100644 --- a/nbs/New_Project_Generator.ipynb +++ b/nbs/New_Project_Generator.ipynb @@ -24,17 +24,17 @@ "from yaspin import yaspin\n", "import shutil\n", "import os\n", - "import stat\n", "\n", "\n", "from faststream_gen._code_generator.helper import (\n", - " download_and_extract_faststream_archive,\n", + " download_and_extract_github_repo,\n", ")\n", "from faststream_gen._code_generator.constants import (\n", " FASTSTREAM_TEMPLATE_ZIP_URL,\n", " FASTSTREAM_TEMPLATE_DIR_SUFFIX,\n", " APPLICATION_FILE_PATH,\n", " TEST_FILE_PATH,\n", + " STAT_0o775\n", ")\n", "\n", "from faststream_gen._code_generator.helper import write_file_contents" @@ -61,18 +61,13 @@ "source": [ "# | export\n", "\n", - "_STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR\n", - " | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP\n", - " | stat.S_IROTH | stat.S_IXOTH )\n", - "\n", - "\n", "def create_project(\n", " output_path: str,\n", ") -> None:\n", " with yaspin(\n", " text=\"Creating a new FastStream project...\", color=\"cyan\", spinner=\"clock\"\n", " ) as sp:\n", - " with download_and_extract_faststream_archive(\n", + " with download_and_extract_github_repo(\n", " FASTSTREAM_TEMPLATE_ZIP_URL\n", " ) as extracted_path:\n", " app_file = str(\n", @@ -87,7 +82,7 @@ " for p in (\n", " Path(extracted_path) / FASTSTREAM_TEMPLATE_DIR_SUFFIX / \"scripts\"\n", " ).glob(\"*.sh\"):\n", - " p.chmod(_STAT_0o775)\n", + " p.chmod(STAT_0o775)\n", "\n", " shutil.copytree(\n", " str(extracted_path / FASTSTREAM_TEMPLATE_DIR_SUFFIX),\n",