diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml new file mode 100644 index 00000000..f513c27b --- /dev/null +++ b/.github/workflows/autofix.yml @@ -0,0 +1,40 @@ +name: Auto-fix when '/autofix' Slash Command is used + +on: + workflow_dispatch: {} + repository_dispatch: + types: [autofix-command] + +jobs: + python-autofix: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: 'poetry' + + - name: Install dependencies + run: poetry install + + - name: Format code + run: poetry run ruff format . + + - name: Commit changes + run: | + git config --global user.name "Airbyte Automation Bot" + git config --global user.email "no-reply@airbyte.io" + git add . + git diff-index --quiet HEAD || git commit -m "Format code with black" + + - name: Push changes + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/python_format_checks.yml b/.github/workflows/python_format_checks.yml new file mode 100644 index 00000000..834f2fa1 --- /dev/null +++ b/.github/workflows/python_format_checks.yml @@ -0,0 +1,29 @@ +name: Python Format Check + +on: + push: + branches: + - main + pull_request: {} + +jobs: + python-format-check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + + - name: Install dependencies + run: poetry install + + - name: Check code format + run: poetry run ruff format --check . diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml new file mode 100644 index 00000000..bcc8d337 --- /dev/null +++ b/.github/workflows/python_lint.yml @@ -0,0 +1,29 @@ +name: Python Lint Check + +on: + push: + branches: + - main + pull_request: {} + +jobs: + python-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + + - name: Install dependencies + run: poetry install + + - name: Format code + run: poetry run ruff check . diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml new file mode 100644 index 00000000..99af6997 --- /dev/null +++ b/.github/workflows/python_pytest.yml @@ -0,0 +1,47 @@ +# This workflow will run pytest, prioritizing rebustness over speed. +# This is in contrast to the 'failfast' workflow, which prioritizes speed over robustness. +name: Pytest (All) + +on: + push: + branches: + - main + pull_request: {} + +jobs: + pytest: + name: Pytest (All, Python ${{ matrix.python-version }}) + # Don't run on forks + if: github.repository_owner == 'airbytehq' + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ + # TODO: Re-enable 3.9 and 3.11 once we have stable tests across all versions. + # '3.9', + '3.10', + # '3.11', + ] + fail-fast: false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + + - name: Install dependencies + run: poetry install + + - name: Run Pytest + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: poetry run pytest diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml new file mode 100644 index 00000000..1a829f59 --- /dev/null +++ b/.github/workflows/python_pytest_fast.yml @@ -0,0 +1,33 @@ +# This workflow will run pytest, prioritizing speed over robustness. +# This is in contrast to the 'all' workflow, which prioritizes robustness over speed. +name: Pytest (Fast Tests Only) + +on: + push: + branches: + - main + pull_request: {} + +jobs: + pytest-fast: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + + - name: Install dependencies + run: poetry install + + - name: Run Pytest (Fast Tests Only) + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: poetry run pytest -m "not slow and not requires_creds" --exitfirst diff --git a/.github/workflows/slash_command_dispatch.yml b/.github/workflows/slash_command_dispatch.yml new file mode 100644 index 00000000..1da8ec92 --- /dev/null +++ b/.github/workflows/slash_command_dispatch.yml @@ -0,0 +1,16 @@ +name: Slash Command Dispatch + +on: + issue_comment: + types: [created] + +jobs: + slashCommandDispatch: + runs-on: ubuntu-latest + steps: + - name: Slash Command Dispatch + uses: peter-evans/slash-command-dispatch@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commands: | + autofix diff --git a/airbyte/caches/base.py b/airbyte/caches/base.py index 0d44daa4..02a1e977 100644 --- a/airbyte/caches/base.py +++ b/airbyte/caches/base.py @@ -1,5 +1,4 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. - """A SQL Cache implementation.""" from __future__ import annotations diff --git a/docs/generated/airbyte.html b/docs/generated/airbyte.html index 2c5ba3eb..5ad45f6e 100644 --- a/docs/generated/airbyte.html +++ b/docs/generated/airbyte.html @@ -2,14 +2,14 @@
- + class CachedDataset(airbyte.datasets._sql.SQLDataset): - +
- +

A dataset backed by a SQL table cache.

Because this dataset includes all records from the underlying table, we also expose the @@ -19,14 +19,14 @@

- + CachedDataset(cache: 'SQLCacheBase', stream_name: str) - +
- - + +
@@ -36,10 +36,10 @@ def to_pandas(self) -> pandas.core.frame.DataFrame: - +
- +

Return a pandas DataFrame representation of the dataset.

The base implementation simply passes the record iterator to Panda's DataFrame constructor.

@@ -49,15 +49,15 @@
- + def to_sql_table(self) -> 'Table': - +
- - + +
@@ -73,14 +73,14 @@
Inherited Members
- + class DuckDBCache(airbyte.caches.duckdb.DuckDBCacheBase): - +
- +

A DuckDB implementation of the cache.

Parquet is used for local file storage before bulk loading. @@ -94,11 +94,11 @@

Inherited Members
file_writer_class = <class 'airbyte._file_writers.parquet.ParquetWriter'> - +
- - + +
@@ -143,14 +143,14 @@
Inherited Members
- + class DuckDBCacheConfig(airbyte.caches.base.SQLCacheConfigBase, airbyte._file_writers.parquet.ParquetWriterConfig): - +
- +

Configuration for the DuckDB cache.

Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

@@ -161,10 +161,10 @@
Inherited Members
db_path: pathlib.Path | str - +
- +

Normally db_path is a Path object.

There are some cases, such as when connecting to MotherDuck, where it could be a string that @@ -177,10 +177,10 @@

Inherited Members
schema_name: str - +
- +

The name of the schema to write to. Defaults to "main".

@@ -193,10 +193,10 @@
Inherited Members
def get_sql_alchemy_url(self) -> str: - +
- +

Return the SQLAlchemy URL to use.

@@ -204,14 +204,14 @@
Inherited Members
- + def get_database_name(self) -> str: - +
- +

Return the name of the database.

@@ -252,14 +252,14 @@
Inherited Members
- + def get_available_connectors() -> list[str]: - +
- +

Return a list of all available connectors.

Connectors will be returned in alphabetical order, with the standard prefix "source-".

@@ -269,14 +269,14 @@
Inherited Members
- + def get_source( name: str, config: dict[str, typing.Any] | None = None, *, version: str | None = None, pip_url: str | None = None, local_executable: pathlib.Path | str | None = None, install_if_missing: bool = True) -> Source: - +
- +

Get a connector by name and version.

Args: @@ -299,14 +299,14 @@

Inherited Members
- + def get_default_cache() -> DuckDBCache: - +
- +

Get a local cache for storing data, using the default database path.

Cache files are stored in the .cache directory, relative to the current @@ -317,14 +317,14 @@

Inherited Members
- + def get_secret( secret_name: str, source: SecretSource | list[SecretSource] = <SecretSource.ANY: 4>, *, prompt: bool = True) -> str: - +
- +

Get a secret from the environment.

The optional source argument of enum type SecretSource or list of SecretSource options. @@ -340,14 +340,14 @@

Inherited Members
- + def new_local_cache( cache_name: str | None = None, cache_dir: str | pathlib.Path | None = None, *, cleanup: bool = True) -> DuckDBCache: - +
- +

Get a local cache for storing data, using a name string to seed the path.

Args: @@ -363,14 +363,14 @@

Inherited Members
- + class ReadResult(collections.abc.Mapping[str, airbyte.datasets._sql.CachedDataset]): - +
- +

A Mapping is a generic container for associating key/value pairs.

@@ -381,60 +381,60 @@
Inherited Members
- + ReadResult( processed_records: int, cache: airbyte.caches.base.SQLCacheBase, processed_streams: list[str]) - +
- - + +
processed_records - +
- - + +
- + def get_sql_engine(self) -> sqlalchemy.engine.base.Engine: - +
- - + +
streams: collections.abc.Mapping[str, CachedDataset] - +
- - + +
cache: airbyte.caches.base.SQLCacheBase - +
- - + +
@@ -452,14 +452,14 @@
Inherited Members
- + class SecretSource(enum.Enum): - +
- +

An enumeration.

@@ -469,11 +469,11 @@
Inherited Members
ENV = <SecretSource.ENV: 1> - + - - + +
@@ -481,11 +481,11 @@
Inherited Members
DOTENV = <SecretSource.DOTENV: 2> - +
- - + +
@@ -493,11 +493,11 @@
Inherited Members
GOOGLE_COLAB = <SecretSource.GOOGLE_COLAB: 3> - +
- - + +
@@ -505,11 +505,11 @@
Inherited Members
ANY = <SecretSource.ANY: 4> - +
- - + +
@@ -517,11 +517,11 @@
Inherited Members
PROMPT = <SecretSource.PROMPT: 5> - +
- - + +
@@ -537,27 +537,27 @@
Inherited Members
- + class Source: - +
- +

A class representing a source that can be called.

- + Source( executor: airbyte._executor.Executor, name: str, config: dict[str, typing.Any] | None = None, streams: list[str] | None = None, *, validate: bool = False) - +
- +

Initialize the source.

If config is provided, it will be validated against the spec if validate is True.

@@ -569,34 +569,34 @@
Inherited Members
executor - +
- - + +
name - +
- - + +
- + def set_streams(self, streams: list[str]) -> None: - +
- +

Deprecated. See select_streams().

@@ -604,14 +604,14 @@
Inherited Members
- + def select_all_streams(self) -> None: - +
- +

Select all streams.

This is a more streamlined equivalent to:

@@ -625,14 +625,14 @@
Inherited Members
- + def select_streams(self, streams: list[str]) -> None: - +
- +

Select the stream names that should be read from the connector.

Currently, if this is not set, all streams will be read.

@@ -642,14 +642,14 @@
Inherited Members
- + def get_selected_streams(self) -> list[str]: - +
- +

Get the selected streams.

If no streams are selected, return an empty list.

@@ -659,14 +659,14 @@
Inherited Members
- + def set_config(self, config: dict[str, typing.Any], *, validate: bool = False) -> None: - +
- +

Set the config for the connector.

If validate is True, raise an exception if the config fails validation.

@@ -679,14 +679,14 @@
Inherited Members
- + def get_config(self) -> dict[str, typing.Any]: - +
- +

Get the config for the connector.

@@ -694,14 +694,14 @@
Inherited Members
- + def validate_config(self, config: dict[str, typing.Any] | None = None) -> None: - +
- +

Validate the config against the spec.

If config is not provided, the already-set config will be validated.

@@ -711,14 +711,14 @@
Inherited Members
- + def get_available_streams(self) -> list[str]: - +
- +

Get the available streams from the spec.

@@ -728,10 +728,10 @@
Inherited Members
docs_url: str - +
- +

Get the URL to the connector's documentation.

@@ -741,10 +741,10 @@
Inherited Members
discovered_catalog: airbyte_protocol.models.airbyte_protocol.AirbyteCatalog - +
- +

Get the raw catalog for the given streams.

If the catalog is not yet known, we call discover to get it.

@@ -756,10 +756,10 @@
Inherited Members
configured_catalog: airbyte_protocol.models.airbyte_protocol.ConfiguredAirbyteCatalog - +
- +

Get the configured catalog for the given streams.

If the raw catalog is not yet known, we call discover to get it.

@@ -775,14 +775,14 @@
Inherited Members
- + def get_records(self, stream: str) -> airbyte.datasets._lazy.LazyDataset: - +
- +

Read a stream from the connector.

This involves the following steps:

@@ -801,14 +801,14 @@
Inherited Members
- + def check(self) -> None: - +
- +

Call check on the connector.

This involves the following steps:

@@ -825,14 +825,14 @@
Inherited Members
- + def install(self) -> None: - +
- +

Install the connector if it is not yet installed.

@@ -840,14 +840,14 @@
Inherited Members
- + def uninstall(self) -> None: - +
- +

Uninstall the connector if it is installed.

This only works if the use_local_install flag wasn't used and installation is managed by @@ -858,14 +858,14 @@

Inherited Members
- + def read( self, cache: airbyte.caches.base.SQLCacheBase | None = None, *, write_strategy: str | airbyte.strategies.WriteStrategy = <WriteStrategy.AUTO: 'auto'>, force_full_refresh: bool = False) -> ReadResult: - +
- +

Read from the connector and write to the cache.

Args: @@ -886,4 +886,4 @@

Inherited Members
- + \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 86d14143..9819e186 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" @@ -1413,22 +1413,22 @@ testing = ["google-api-core[grpc] (>=1.31.5)"] [[package]] name = "protobuf" -version = "4.25.2" +version = "4.25.3" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, - {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, - {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, - {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, - {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, - {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, - {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, - {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, - {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, - {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, - {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, + {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, + {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, + {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, + {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, + {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, + {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, + {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, + {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, + {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, ] [[package]] @@ -1924,7 +1924,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1932,16 +1931,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1958,7 +1949,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1966,7 +1956,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2684,4 +2673,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "3b46151e994684f0953be1041a850ac8efcedea10632f8fa86aaaa6d20385174" +content-hash = "a61f9755ed5b078d77d06d6930ddbc394a88e480e41ad872547fb542aeb7ec0c" diff --git a/pyproject.toml b/pyproject.toml index ddaa8c37..f8defa35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,27 +10,26 @@ packages = [{include = "airbyte"}] python = "^3.9" airbyte-cdk = "^0.58.3" -# airbyte-protocol-models = "^1.0.1" # Conflicts with airbyte-cdk # TODO: delete or resolve +duckdb-engine = "^0.10.0" jsonschema = "3.2.0" orjson = "^3.9.10" overrides = "^7.4.0" pandas = "2.1.4" # 2.2.0 breaks sqlalchemy interop - TODO: optionally retest higher versions +pendulum = "<=3.0.0" psycopg2-binary = "^2.9.9" +# psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} +# Psycopg3 is not supported in SQLAlchemy 1.x: +pyarrow = "^14.0.2" +pydantic = "<=2.0" +python-dotenv = "^1.0.1" python-ulid = "^2.2.0" -types-pyyaml = "^6.0.12.12" -ulid = "^1.1" -sqlalchemy = "1.4.51" -snowflake-connector-python = "3.6.0" -snowflake-sqlalchemy = "^1.5.1" -duckdb-engine = "^0.10.0" requests = "^2.31.0" -pyarrow = "^14.0.2" - -# Psycopg3 is not supported in SQLAlchemy 1.x: -# psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} rich = "^13.7.0" -pendulum = "<=3.0.0" -python-dotenv = "^1.0.1" +snowflake-connector-python = "3.6.0" +snowflake-sqlalchemy = "^1.5.1" +sqlalchemy = "1.4.51" +types-pyyaml = "^6.0.12.12" +ulid = "^1.1" [tool.poetry.group.dev.dependencies]