Add documentation publish and preview workflows

.github/workflows/pydoc_preview.yml

@@ -0,0 +1,38 @@
+name: Test Docs Generation
+
+on:
+  push:
+    branches:
+    - main
+  pull_request: {}
+
+
+jobs:
+  preview_docs:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Set up Poetry
+      uses: Gr1N/setup-poetry@v8
+      with:
+        poetry-version: "1.7.1"
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+        cache: 'poetry'
+
+    - name: Install dependencies
+      run: poetry install
+
+    - name: Generate documentation
+      run: |
+        poetry run generate-docs
+
+    - name: Upload artifact
+      uses: actions/upload-pages-artifact@v3
+      with:
+        # Upload entire repository
+        path: 'docs/generated'

.github/workflows/pydoc_publish.yml

@@ -0,0 +1,60 @@
+name: Publish Documentation Site
+
+on:
+  push:
+    branches:
+    - main
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  publish_docs:
+    runs-on: ubuntu-latest
+    environment:
+      name: "github-pages"
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Set up Poetry
+      uses: Gr1N/setup-poetry@v8
+      with:
+        poetry-version: "1.7.1"
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+        cache: 'poetry'
+    - name: Setup Pages
+      uses: actions/configure-pages@v4
+
+    - name: Install dependencies
+      run: poetry install
+
+    - name: Generate documentation
+      run: |
+        poetry run generate-docs
+
+    - name: Upload artifact
+      uses: actions/upload-pages-artifact@v3
+      with:
+        # Upload entire repository
+        path: 'docs/generated'
+
+    - name: Deploy to GitHub Pages
+      id: deployment
+      uses: actions/deploy-pages@v4

CONTRIBUTING.md

@@ -0,0 +1,35 @@
+# Contributing to PyAirbyte
+
+Learn how you can become a contributor to PyAirbyte.
+
+## Development
+
+- Make sure [Poetry is installed](https://python-poetry.org/docs/#).
+- Run `poetry install`
+- For examples, check out the `examples` folder. They can be run via `poetry run python examples/<example file>`
+- Unit tests and type checks can be run via `poetry run pytest`
+
+## Documentation
+
+Regular documentation lives in the `/docs` folder. Based on the doc strings of public methods, we generate API documentation using [pdoc](https://pdoc.dev).
+
+To generate the documentation, run:
+
+```console
+poetry run generate-docs
+```
+
+The `generate-docs` CLI command is mapped to the `run()` function of `docs/generate.py`.
+
+Documentation pages will be generated in the `docs/generated` folder. The `test_docs.py` test in pytest will automatically update generated content. This updates must be manually committed before docs tests will pass.
+
+## Release
+
+- In your PR:
+  - Bump the version in `pyproject.toml`
+  - Add a changelog entry to the table below
+- Once the PR is merged, go to Github and trigger the `Publish PyAirbyte Manually` workflow. This will publish the new version to PyPI.
+
+## Versioning
+
+Versioning follows [Semantic Versioning](https://semver.org/). For new features, bump the minor version. For bug fixes, bump the patch version. For pre-releases, append `dev.N` to the version. For example, `0.1.0dev.1` is the first pre-release of the `0.1.0` version.

README.md

@@ -1,36 +1,22 @@
 # PyAirbyte
 
-PyAirbyte is a library that allows to run Airbyte syncs embedded into any Python application, without the need to run Airbyte server.
-
-## Development
-
-- Make sure [Poetry is installed](https://python-poetry.org/docs/#).
-- Run `poetry install`
-- For examples, check out the `examples` folder. They can be run via `poetry run python examples/<example file>`
-- Unit tests and type checks can be run via `poetry run pytest`
-
-## Release
-
-- In your PR:
-  - Bump the version in `pyproject.toml`
-  - Add a changelog entry to the table below
-- Once the PR is merged, go to Github and trigger the `Publish AirbyteLib Manually` workflow. This will publish the new version to PyPI.
+PyAirbyte is a library that allows to run Airbyte syncs embedded into any Python application, without requiring connectivity to a hosted Airbyte instance.
 
 ## Secrets Management
 
-AirbyteLib can auto-import secrets from the following sources:
+PyAirbyte can auto-import secrets from the following sources:
 
 1. Environment variables.
 2. Variables defined in a local `.env` ("Dotenv") file.
 3. [Google Colab secrets](https://medium.com/@parthdasawant/how-to-use-secrets-in-google-colab-450c38e3ec75).
 4. Manual entry via [`getpass`](https://docs.python.org/3.9/library/getpass.html).
 
-_Note: Additional secret store options may be supported in the future. [More info here.](https://github.com/airbytehq/PyAirbyte-private-beta/discussions/5)_
+_Note: Additional secret store options may be supported in the future. [More info here.](https://github.com/airbytehq/airbyte-lib-private-beta/discussions/5)_
 
 ### Retrieving Secrets
 
 ```python
-from airbyte import get_secret, SecretSource
+from airbyte_lib import get_secret, SecretSource
 
 source = get_connection("source-github")
 source.set_config(
@@ -40,26 +26,17 @@ source.set_config(
 )
 ```
 
-The `get_secret()` function accepts an optional `source` argument of enum type `SecretSource`. If omitted or set to `SecretSource.ANY`, AirbyteLib will search all available secrets sources. If `source` is set to a specific source, then only that source will be checked. If a list of `SecretSource` entries is passed, then the sources will be checked using the provided ordering.
-
-By default, AirbyteLib will prompt the user for any requested secrets that are not provided via other secret managers. You can disable this prompt by passing `prompt=False` to `get_secret()`.
-
-### Versioning
+The `get_secret()` function accepts an optional `source` argument of enum type `SecretSource`. If omitted or set to `SecretSource.ANY`, PyAirbyte will search all available secrets sources. If `source` is set to a specific source, then only that source will be checked. If a list of `SecretSource` entries is passed, then the sources will be checked using the provided ordering.
 
-Versioning follows [Semantic Versioning](https://semver.org/). For new features, bump the minor version. For bug fixes, bump the patch version. For pre-releases, append `dev.N` to the version. For example, `0.1.0dev.1` is the first pre-release of the `0.1.0` version.
-
-## Documentation
-
-Regular documentation lives in the `/docs` folder. Based on the doc strings of public methods, we generate API documentation using [pdoc](https://pdoc.dev). To generate the documentation, run `poetry run generate-docs`. The documentation will be generated in the `docs/generate` folder. This needs to be done manually when changing the public interface of the library.
-
-A unit test validates the documentation is up to date.
+By default, PyAirbyte will prompt the user for any requested secrets that are not provided via other secret managers. You can disable this prompt by passing `prompt=False` to `get_secret()`.
 
 ## Connector compatibility
 
 To make a connector compatible with PyAirbyte, the following requirements must be met:
-* The connector must be a Python package, with a `pyproject.toml` or a `setup.py` file.
-* In the package, there must be a `run.py` file that contains a `run` method. This method should read arguments from the command line, and run the connector with them, outputting messages to stdout.
-* The `pyproject.toml` or `setup.py` file must specify a command line entry point for the `run` method called `source-<connector name>`. This is usually done by adding a `console_scripts` section to the `pyproject.toml` file, or a `entry_points` section to the `setup.py` file. For example:
+
+- The connector must be a Python package, with a `pyproject.toml` or a `setup.py` file.
+- In the package, there must be a `run.py` file that contains a `run` method. This method should read arguments from the command line, and run the connector with them, outputting messages to stdout.
+- The `pyproject.toml` or `setup.py` file must specify a command line entry point for the `run` method called `source-<connector name>`. This is usually done by adding a `console_scripts` section to the `pyproject.toml` file, or a `entry_points` section to the `setup.py` file. For example:
 
 ```toml
 [tool.poetry.scripts]
@@ -101,6 +78,10 @@ The script will install the python package in the provided directory, and run th
 
 For a more lightweight check, the `--validate-install-only` flag can be used. This will only check that the connector can be installed and returns a spec, no sample config required.
 
+## Contributing
+
+To learn how you can contribute to PyAirbyte, please see our [PyAirbyte Contributors Guide](./CONTRIBUTING.md).
+
 ## Changelog
 
 | Version     | PR                                                         | Description                                                                                                       |

airbyte/__init__.py

@@ -1,4 +1,8 @@
-"""AirbyteLib brings Airbyte ELT to every Python developer."""
+"""PyAirbyte brings Airbyte ELT to every Python developer.
+
+.. include:: ../README.md
+
+"""
 from __future__ import annotations
 
 from airbyte._factories.cache_factories import get_default_cache, new_local_cache
@@ -24,3 +28,5 @@
     "SecretSource",
     "Source",
 ]
+
+__docformat__ = "google"

airbyte/exceptions.py

@@ -106,7 +106,7 @@ def __repr__(self) -> str:
         return f"{class_name}({properties_str})"
 
 
-# AirbyteLib Internal Errors (these are probably bugs)
+# PyAirbyte Internal Errors (these are probably bugs)
 
 
 @dataclass
@@ -117,12 +117,12 @@ class AirbyteLibInternalError(AirbyteError):
     help_url = NEW_ISSUE_URL
 
 
-# AirbyteLib Input Errors (replaces ValueError for user input)
+# PyAirbyte Input Errors (replaces ValueError for user input)
 
 
 @dataclass
 class AirbyteLibInputError(AirbyteError, ValueError):
-    """The input provided to AirbyteLib did not match expected validation rules.
+    """The input provided to PyAirbyte did not match expected validation rules.
 
     This inherits from ValueError so that it can be used as a drop-in replacement for
     ValueError in the Airbyte Lib API.
@@ -146,7 +146,7 @@ class AirbyteLibNoStreamsSelectedError(AirbyteLibInputError):
     available_streams: list[str] | None = None
 
 
-# AirbyteLib Cache Errors
+# PyAirbyte Cache Errors
 
 
 class AirbyteLibCacheError(AirbyteError):

airbyte/secrets.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-"""Secrets management for AirbyteLib."""
+"""Secrets management for PyAirbyte."""
 from __future__ import annotations
 
 import contextlib

airbyte/strategies.py

@@ -1,13 +1,13 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
-"""Read and write strategies for AirbyteLib."""
+"""Read and write strategies for PyAirbyte."""
 from __future__ import annotations
 
 from enum import Enum
 
 
 class WriteStrategy(str, Enum):
-    """Read strategies for AirbyteLib."""
+    """Read strategies for PyAirbyte."""
 
     MERGE = "merge"
     """Merge new records with existing records.

docs/.gitignore

@@ -0,0 +1 @@
+generated

docs/generate.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
+
+import os
+import pathlib
+import shutil
+
+import pdoc
+
+import airbyte as ab
+
+
+def run() -> None:
+    """Generate docs for all public modules in airbyte_lib and save them to docs/generated.
+
+    Public modules are:
+    * The main airbyte_lib module
+    * All directory modules in airbyte_lib that don't start with an underscore.
+    """
+    public_modules = ["airbyte"]
+
+    # recursively delete the docs/generated folder if it exists
+    if pathlib.Path("docs/generated").exists():
+        shutil.rmtree("docs/generated")
+
+    # All files and folders in `airbyte_lib` that don't start with "_" are treated as public.
+    for submodule in os.listdir("airbyte"):
+        submodule_path = pathlib.Path(f"airbyte/{submodule}")
+        if not submodule.startswith("_"):
+            public_modules.append(submodule_path)
+
+    pdoc.render.configure(
+        template_directory="docs",
+        show_source=False,
+        search=False,
+    )
+    pdoc.pdoc(
+        *public_modules,
+        output_directory=pathlib.Path("docs/generated"),
+    )