update docs script

docs/.gitignore

@@ -0,0 +1 @@
+generated

docs/generate.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
+
+import os
+import pathlib
+import shutil
+
+import pdoc
+
+import airbyte as ab
+
+import typing
+
+typing.TYPE_CHECKING = True
+
+
+def run() -> None:
+    """Generate docs for all public modules in airbyte_lib and save them to docs/generated.
+
+    Public modules are:
+    * The main airbyte_lib module
+    * All directory modules in airbyte_lib that don't start with an underscore.
+    """
+    public_modules = ["airbyte"]
+
+    # recursively delete the docs/generated folder if it exists
+    if pathlib.Path("docs/generated").exists():
+        shutil.rmtree("docs/generated")
+
+    # All files and folders in `airbyte_lib` that don't start with "_" are treated as public.
+    for submodule in os.listdir("airbyte"):
+        submodule_path = pathlib.Path(f"airbyte/{submodule}")
+        if not submodule.startswith("_"):
+            public_modules.append(submodule_path)
+
+    pdoc.render.configure(
+        template_directory="docs",
+        show_source=False,
+        search=False,
+    )
+    pdoc.pdoc(
+        *public_modules,
+        output_directory=pathlib.Path("docs/generated"),
+    )

pyproject.toml

@@ -247,7 +247,7 @@ module = [
 ignore_missing_imports = true  # No stubs yet (😢)
 
 [tool.poetry.scripts]
-generate-docs = "docs:run"
+generate-docs = "docs.generate:run"
 airbyte-lib-validate-source = "airbyte.validate:run"
 
 [tool.poe.tasks]

tests/docs_tests/test_docs_checked_in.py

@@ -2,7 +2,7 @@
 
 import os
 
-import docs
+import docs.generate as generate
 
 
 def test_docs_checked_in():
@@ -13,7 +13,7 @@ def test_docs_checked_in():
     It will fail if there are any differences.
     """
 
-    docs.run()
+    generate.run()
 
     # compare the generated docs with the checked in docs
     diff = os.system("git diff --exit-code docs/generated")