diff --git a/config_examples/s3_delta.yaml b/config_examples/s3_delta.yaml index 457e657..f9d4db1 100644 --- a/config_examples/s3_delta.yaml +++ b/config_examples/s3_delta.yaml @@ -11,8 +11,25 @@ plugins: # aws_session_token: # optional if you want to specify the aws session token delta_tables: - bucket: bucket - prefix: delta_data + prefix: delta_data # Prefix to DeltaTable or directory where delta tables are stored +# S3 Delta Lake Adapter with Filter +default_pulling_interval: 10 +token: +platform_host_url: "http://localhost:8080" +plugins: + - type: s3_delta + name: s3_adapter + aws_access_key_id: + aws_secret_access_key: + aws_region: + # aws_session_token: # optional if you want to specify the aws session token + delta_tables: + - bucket: bucket + prefix: delta_data + object_filter: # will exclude all folders with _pii at the end of name from ingestion + exclude: + - ".*_pii" # Minio S3 Delta Lake Adapter default_pulling_interval: 10 diff --git a/odd_collector_aws/adapters/s3/file_system.py b/odd_collector_aws/adapters/s3/file_system.py index 97c197d..8b0f9ec 100644 --- a/odd_collector_aws/adapters/s3/file_system.py +++ b/odd_collector_aws/adapters/s3/file_system.py @@ -155,5 +155,8 @@ def get_folder(self, path: str, recursive: bool = True) -> Folder: def remove_protocol(path: str) -> str: if path.startswith("s3://"): - path = path[5:] - return path + return path.removeprefix("s3://") + elif path.startswith(("s3a://", "s3n://")): + return path[6:] + else: + return path diff --git a/odd_collector_aws/adapters/s3_delta/adapter.py b/odd_collector_aws/adapters/s3_delta/adapter.py index 4305560..5855eed 100644 --- a/odd_collector_aws/adapters/s3_delta/adapter.py +++ b/odd_collector_aws/adapters/s3_delta/adapter.py @@ -1,6 +1,6 @@ from typing import Union -from funcy import lmap, partial +from funcy import lmap, mapcat, partial from odd_collector_sdk.domain.adapter import BaseAdapter from odd_models.models import DataEntityList from oddrn_generator.generators import Generator, S3Generator @@ -30,10 +30,10 @@ def create_generator(self) -> Generator: def get_data_entity_list(self) -> DataEntityList: logger.debug(f"Getting data entity list for {self.config.delta_tables}") - tables = lmap(self.client.get_table, self.config.delta_tables) + tables = mapcat(self.client.get_table, self.config.delta_tables) data_entities = lmap(partial(map_delta_table, self.generator), tables) return DataEntityList( data_source_oddrn=self.generator.get_data_source_oddrn(), - items=list(data_entities), + items=data_entities, ) diff --git a/odd_collector_aws/adapters/s3_delta/client.py b/odd_collector_aws/adapters/s3_delta/client.py index b3f30e4..a167e1c 100644 --- a/odd_collector_aws/adapters/s3_delta/client.py +++ b/odd_collector_aws/adapters/s3_delta/client.py @@ -1,12 +1,13 @@ import datetime import traceback as tb from dataclasses import asdict, dataclass -from typing import Optional +from typing import Any, Iterable, Optional from deltalake import DeltaTable from funcy import complement, isnone, last, partial, select_values, silent, walk from odd_collector_aws.domain.plugin import DeltaTableConfig, S3DeltaPlugin +from odd_collector_aws.filesystem.pyarrow_fs import FileSystem from .logger import logger from .models.table import DTable @@ -51,50 +52,46 @@ def to_dict(self) -> dict[str, str]: return select_values(complement(isnone), asdict(self)) +class IsNotDeltaTable(Exception): + ... + + class DeltaClient: def __init__(self, config: S3DeltaPlugin) -> None: self.storage_options: StorageOptions = StorageOptions.from_config(config) + self.fs = FileSystem(config) - def get_table(self, delta_table_config: DeltaTableConfig) -> DTable: - # sourcery skip: raise-specific-error + def load_delta_table(self, delta_table_config: DeltaTableConfig) -> None: try: - table = DeltaTable( - delta_table_config.path, storage_options=self.storage_options.to_dict() + return DeltaTable( + delta_table_config.path, + storage_options=self.storage_options.to_dict(), ) + except Exception as e: + raise IsNotDeltaTable() from e + + def handle_folder(self, config: DeltaTableConfig) -> Iterable[DTable]: + logger.debug(f"Getting delta tables from folder {config.path}") + + objects = self.fs.get_file_info(remove_protocol(config.path)) + allowed = filter( + lambda object: not object.is_file and config.allow(object.base_name), + objects, + ) + + for object in allowed: + config = config.append_prefix(object.base_name) + yield from self.get_table(config) + + def get_table(self, delta_table_config: DeltaTableConfig) -> Iterable[DTable]: + # sourcery skip: raise-specific-error + try: + logger.debug(f"Getting delta table {delta_table_config.path}") + table = self.load_delta_table(delta_table_config) + + metadata = get_metadata(table) - metadata = {} - - try: - logger.debug(f"Getting actions list for {delta_table_config.path}") - actions = table.get_add_actions(flatten=True).to_pydict() - - metadata |= walk( - partial(handle_values, actions), - {"size_bytes": sum, "num_records": sum, "modification_time": last}, - ) - except Exception as e: - logger.error( - f"Failed to get actions list for {delta_table_config.path}" - ) - - try: - logger.debug(f"Getting metadata for {delta_table_config.path}") - delta_metadata = table.metadata() - metadata |= { - "id": delta_metadata.id, - "name": delta_metadata.name, - "description": delta_metadata.description, - "partition_columns": ",".join(delta_metadata.partition_columns), - "configuration": delta_metadata.configuration, - "created_time": delta_metadata.created_time, - } - except Exception as e: - logger.debug(tb.format_exc()) - logger.error( - f"Failed to get metadata for {delta_table_config.path}. {e}" - ) - - return DTable( + yield DTable( table_uri=table.table_uri, schema=table.schema(), num_rows=metadata.get("num_records"), @@ -102,7 +99,54 @@ def get_table(self, delta_table_config: DeltaTableConfig) -> DTable: created_at=silent(from_ms)(metadata.get("created_time")), updated_at=silent(add_utc_timezone)(metadata.get("modification_time")), ) + except IsNotDeltaTable: + logger.warning( + f"Path {delta_table_config.path} is not a delta table. Searching for" + " delta tables in subfolders." + ) + yield from self.handle_folder(delta_table_config) except Exception as e: raise Exception( f"Failed to get delta table {delta_table_config.path}. {e}" ) from e + + +def get_metadata(table: DeltaTable) -> dict[str, Any]: + metadata = {} + + try: + logger.debug(f"Getting actions list for {table.table_uri}") + actions = table.get_add_actions(flatten=True).to_pydict() + + metadata |= walk( + partial(handle_values, actions), + {"size_bytes": sum, "num_records": sum, "modification_time": last}, + ) + except Exception as e: + logger.error(f"Failed to get actions list for {table.table_uri}") + + try: + logger.debug(f"Getting metadata for {table.table_uri}") + delta_metadata = table.metadata() + metadata |= { + "id": delta_metadata.id, + "name": delta_metadata.name, + "description": delta_metadata.description, + "partition_columns": ",".join(delta_metadata.partition_columns), + "configuration": delta_metadata.configuration, + "created_time": delta_metadata.created_time, + } + except Exception as e: + logger.debug(tb.format_exc()) + logger.error(f"Failed to get metadata for {table.table_uri}. {e}") + + return metadata + + +def remove_protocol(path: str) -> str: + if path.startswith("s3://"): + return path.removeprefix("s3://") + elif path.startswith(("s3a://", "s3n://")): + return path[6:] + else: + return path diff --git a/odd_collector_aws/domain/plugin.py b/odd_collector_aws/domain/plugin.py index 7c94efc..0adcada 100644 --- a/odd_collector_aws/domain/plugin.py +++ b/odd_collector_aws/domain/plugin.py @@ -1,5 +1,6 @@ from typing import Any, List, Literal, Optional +from odd_collector_sdk.domain.filter import Filter from odd_collector_sdk.domain.plugin import Plugin from odd_collector_sdk.types import PluginFactory from pydantic import BaseModel, Field @@ -41,11 +42,23 @@ class DeltaTableConfig(BaseModel): scheme: str = Field(default="s3", alias="schema") bucket: str prefix: str + object_filter: Optional[Filter] = None @property def path(self) -> str: return f"{self.scheme}://{self.bucket}/{self.prefix.strip('/')}" + def append_prefix(self, path: str) -> "DeltaTableConfig": + return DeltaTableConfig( + schema=self.scheme, + bucket=self.bucket, + prefix=f"{self.prefix}/{path}", + object_filter=self.object_filter, + ) + + def allow(self, name: str) -> bool: + return self.object_filter.is_allowed(name) + class S3DeltaPlugin(AwsPlugin): type: Literal["s3_delta"] diff --git a/odd_collector_aws/filesystem/pyarrow_fs.py b/odd_collector_aws/filesystem/pyarrow_fs.py new file mode 100644 index 0000000..dadfb13 --- /dev/null +++ b/odd_collector_aws/filesystem/pyarrow_fs.py @@ -0,0 +1,34 @@ +from pyarrow._fs import FileInfo, FileSelector +from pyarrow.fs import S3FileSystem + +from odd_collector_aws.domain.plugin import AwsPlugin + + +class FileSystem: + """ + FileSystem hides pyarrow.fs implementation details. + """ + + def __init__(self, config: AwsPlugin): + params = {} + + if config.aws_access_key_id: + params["access_key"] = config.aws_access_key_id + if config.aws_secret_access_key: + params["secret_key"] = config.aws_secret_access_key + if config.aws_session_token: + params["session_token"] = config.aws_session_token + if config.aws_region: + params["region"] = config.aws_region + if config.endpoint_url: + params["endpoint_override"] = config.endpoint_url + + self.fs = S3FileSystem(**params) + + def get_file_info(self, path: str) -> list[FileInfo]: + """ + Get file info from path. + @param path: s3 path to file or folder + @return: FileInfo + """ + return self.fs.get_file_info(FileSelector(base_dir=path)) diff --git a/poetry.lock b/poetry.lock index f35ee9d..aff72bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -874,14 +874,14 @@ files = [ [[package]] name = "odd-collector-sdk" -version = "0.3.27" +version = "0.3.31" description = "ODD Collector" category = "main" optional = false python-versions = ">=3.9,<4.0" files = [ - {file = "odd_collector_sdk-0.3.27-py3-none-any.whl", hash = "sha256:8cd0ca283587224342e15c2c66d79d5563a65dd44147fd55412c9d17e42918d3"}, - {file = "odd_collector_sdk-0.3.27.tar.gz", hash = "sha256:2330bd221f33dfc7e6c7d93da8a611fb23355d2c2de840a39af62535f9c875cf"}, + {file = "odd_collector_sdk-0.3.31-py3-none-any.whl", hash = "sha256:90fbec1dba972ad6f684d22bb56cbda6229580600cc7c1f08425e60ff4fb9ee6"}, + {file = "odd_collector_sdk-0.3.31.tar.gz", hash = "sha256:4ccd76659bf1b54a939544d1b73b901b8e4e9385f4b5d8272f56cc5cb0c0ae23"}, ] [package.dependencies] @@ -919,14 +919,14 @@ sqlparse = "0.4.2" [[package]] name = "oddrn-generator" -version = "0.1.80" +version = "0.1.81" description = "Open Data Discovery Resource Name Generator" category = "main" optional = false python-versions = ">=3.9,<4.0" files = [ - {file = "oddrn_generator-0.1.80-py3-none-any.whl", hash = "sha256:19d8c8615934efb3c3bc6919869a61edfb1973c81521c5bdc5ba2f4329025574"}, - {file = "oddrn_generator-0.1.80.tar.gz", hash = "sha256:c92705fe108df58a8a1fa8a96ce239236126b166f530effee0c7ff40c5ba60f3"}, + {file = "oddrn_generator-0.1.81-py3-none-any.whl", hash = "sha256:188274459ba501db37087d88737018af1b13a69d9fa08cd258e665bdfc99caf3"}, + {file = "oddrn_generator-0.1.81.tar.gz", hash = "sha256:f9161901b4596e7262746f4d143791f07bf95b26bd0eacef5365be8c8fd82a6a"}, ] [package.dependencies] @@ -1065,48 +1065,48 @@ numpy = ">=1.16.6" [[package]] name = "pydantic" -version = "1.10.8" +version = "1.10.9" description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1243d28e9b05003a89d72e7915fdb26ffd1d39bdd39b00b7dbe4afae4b557f9d"}, - {file = "pydantic-1.10.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0ab53b609c11dfc0c060d94335993cc2b95b2150e25583bec37a49b2d6c6c3f"}, - {file = "pydantic-1.10.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9613fadad06b4f3bc5db2653ce2f22e0de84a7c6c293909b48f6ed37b83c61f"}, - {file = "pydantic-1.10.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df7800cb1984d8f6e249351139667a8c50a379009271ee6236138a22a0c0f319"}, - {file = "pydantic-1.10.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0c6fafa0965b539d7aab0a673a046466d23b86e4b0e8019d25fd53f4df62c277"}, - {file = "pydantic-1.10.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e82d4566fcd527eae8b244fa952d99f2ca3172b7e97add0b43e2d97ee77f81ab"}, - {file = "pydantic-1.10.8-cp310-cp310-win_amd64.whl", hash = "sha256:ab523c31e22943713d80d8d342d23b6f6ac4b792a1e54064a8d0cf78fd64e800"}, - {file = "pydantic-1.10.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:666bdf6066bf6dbc107b30d034615d2627e2121506c555f73f90b54a463d1f33"}, - {file = "pydantic-1.10.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:35db5301b82e8661fa9c505c800d0990bc14e9f36f98932bb1d248c0ac5cada5"}, - {file = "pydantic-1.10.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90c1e29f447557e9e26afb1c4dbf8768a10cc676e3781b6a577841ade126b85"}, - {file = "pydantic-1.10.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93e766b4a8226e0708ef243e843105bf124e21331694367f95f4e3b4a92bbb3f"}, - {file = "pydantic-1.10.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88f195f582851e8db960b4a94c3e3ad25692c1c1539e2552f3df7a9e972ef60e"}, - {file = "pydantic-1.10.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:34d327c81e68a1ecb52fe9c8d50c8a9b3e90d3c8ad991bfc8f953fb477d42fb4"}, - {file = "pydantic-1.10.8-cp311-cp311-win_amd64.whl", hash = "sha256:d532bf00f381bd6bc62cabc7d1372096b75a33bc197a312b03f5838b4fb84edd"}, - {file = "pydantic-1.10.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7d5b8641c24886d764a74ec541d2fc2c7fb19f6da2a4001e6d580ba4a38f7878"}, - {file = "pydantic-1.10.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b1f6cb446470b7ddf86c2e57cd119a24959af2b01e552f60705910663af09a4"}, - {file = "pydantic-1.10.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c33b60054b2136aef8cf190cd4c52a3daa20b2263917c49adad20eaf381e823b"}, - {file = "pydantic-1.10.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1952526ba40b220b912cdc43c1c32bcf4a58e3f192fa313ee665916b26befb68"}, - {file = "pydantic-1.10.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bb14388ec45a7a0dc429e87def6396f9e73c8c77818c927b6a60706603d5f2ea"}, - {file = "pydantic-1.10.8-cp37-cp37m-win_amd64.whl", hash = "sha256:16f8c3e33af1e9bb16c7a91fc7d5fa9fe27298e9f299cff6cb744d89d573d62c"}, - {file = "pydantic-1.10.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1ced8375969673929809d7f36ad322934c35de4af3b5e5b09ec967c21f9f7887"}, - {file = "pydantic-1.10.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93e6bcfccbd831894a6a434b0aeb1947f9e70b7468f274154d03d71fabb1d7c6"}, - {file = "pydantic-1.10.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:191ba419b605f897ede9892f6c56fb182f40a15d309ef0142212200a10af4c18"}, - {file = "pydantic-1.10.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:052d8654cb65174d6f9490cc9b9a200083a82cf5c3c5d3985db765757eb3b375"}, - {file = "pydantic-1.10.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ceb6a23bf1ba4b837d0cfe378329ad3f351b5897c8d4914ce95b85fba96da5a1"}, - {file = "pydantic-1.10.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f2e754d5566f050954727c77f094e01793bcb5725b663bf628fa6743a5a9108"}, - {file = "pydantic-1.10.8-cp38-cp38-win_amd64.whl", hash = "sha256:6a82d6cda82258efca32b40040228ecf43a548671cb174a1e81477195ed3ed56"}, - {file = "pydantic-1.10.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e59417ba8a17265e632af99cc5f35ec309de5980c440c255ab1ca3ae96a3e0e"}, - {file = "pydantic-1.10.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84d80219c3f8d4cad44575e18404099c76851bc924ce5ab1c4c8bb5e2a2227d0"}, - {file = "pydantic-1.10.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e4148e635994d57d834be1182a44bdb07dd867fa3c2d1b37002000646cc5459"}, - {file = "pydantic-1.10.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12f7b0bf8553e310e530e9f3a2f5734c68699f42218bf3568ef49cd9b0e44df4"}, - {file = "pydantic-1.10.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42aa0c4b5c3025483240a25b09f3c09a189481ddda2ea3a831a9d25f444e03c1"}, - {file = "pydantic-1.10.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17aef11cc1b997f9d574b91909fed40761e13fac438d72b81f902226a69dac01"}, - {file = "pydantic-1.10.8-cp39-cp39-win_amd64.whl", hash = "sha256:66a703d1983c675a6e0fed8953b0971c44dba48a929a2000a493c3772eb61a5a"}, - {file = "pydantic-1.10.8-py3-none-any.whl", hash = "sha256:7456eb22ed9aaa24ff3e7b4757da20d9e5ce2a81018c1b3ebd81a0b88a18f3b2"}, - {file = "pydantic-1.10.8.tar.gz", hash = "sha256:1410275520dfa70effadf4c21811d755e7ef9bb1f1d077a21958153a92c8d9ca"}, + {file = "pydantic-1.10.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e692dec4a40bfb40ca530e07805b1208c1de071a18d26af4a2a0d79015b352ca"}, + {file = "pydantic-1.10.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c52eb595db83e189419bf337b59154bdcca642ee4b2a09e5d7797e41ace783f"}, + {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:939328fd539b8d0edf244327398a667b6b140afd3bf7e347cf9813c736211896"}, + {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b48d3d634bca23b172f47f2335c617d3fcb4b3ba18481c96b7943a4c634f5c8d"}, + {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f0b7628fb8efe60fe66fd4adadd7ad2304014770cdc1f4934db41fe46cc8825f"}, + {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e1aa5c2410769ca28aa9a7841b80d9d9a1c5f223928ca8bec7e7c9a34d26b1d4"}, + {file = "pydantic-1.10.9-cp310-cp310-win_amd64.whl", hash = "sha256:eec39224b2b2e861259d6f3c8b6290d4e0fbdce147adb797484a42278a1a486f"}, + {file = "pydantic-1.10.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d111a21bbbfd85c17248130deac02bbd9b5e20b303338e0dbe0faa78330e37e0"}, + {file = "pydantic-1.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e9aec8627a1a6823fc62fb96480abe3eb10168fd0d859ee3d3b395105ae19a7"}, + {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07293ab08e7b4d3c9d7de4949a0ea571f11e4557d19ea24dd3ae0c524c0c334d"}, + {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee829b86ce984261d99ff2fd6e88f2230068d96c2a582f29583ed602ef3fc2c"}, + {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4b466a23009ff5cdd7076eb56aca537c745ca491293cc38e72bf1e0e00de5b91"}, + {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7847ca62e581e6088d9000f3c497267868ca2fa89432714e21a4fb33a04d52e8"}, + {file = "pydantic-1.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:7845b31959468bc5b78d7b95ec52fe5be32b55d0d09983a877cca6aedc51068f"}, + {file = "pydantic-1.10.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:517a681919bf880ce1dac7e5bc0c3af1e58ba118fd774da2ffcd93c5f96eaece"}, + {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67195274fd27780f15c4c372f4ba9a5c02dad6d50647b917b6a92bf00b3d301a"}, + {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2196c06484da2b3fded1ab6dbe182bdabeb09f6318b7fdc412609ee2b564c49a"}, + {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6257bb45ad78abacda13f15bde5886efd6bf549dd71085e64b8dcf9919c38b60"}, + {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3283b574b01e8dbc982080d8287c968489d25329a463b29a90d4157de4f2baaf"}, + {file = "pydantic-1.10.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5f8bbaf4013b9a50e8100333cc4e3fa2f81214033e05ac5aa44fa24a98670a29"}, + {file = "pydantic-1.10.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9cd67fb763248cbe38f0593cd8611bfe4b8ad82acb3bdf2b0898c23415a1f82"}, + {file = "pydantic-1.10.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f50e1764ce9353be67267e7fd0da08349397c7db17a562ad036aa7c8f4adfdb6"}, + {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73ef93e5e1d3c8e83f1ff2e7fdd026d9e063c7e089394869a6e2985696693766"}, + {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:128d9453d92e6e81e881dd7e2484e08d8b164da5507f62d06ceecf84bf2e21d3"}, + {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ad428e92ab68798d9326bb3e5515bc927444a3d71a93b4a2ca02a8a5d795c572"}, + {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fab81a92f42d6d525dd47ced310b0c3e10c416bbfae5d59523e63ea22f82b31e"}, + {file = "pydantic-1.10.9-cp38-cp38-win_amd64.whl", hash = "sha256:963671eda0b6ba6926d8fc759e3e10335e1dc1b71ff2a43ed2efd6996634dafb"}, + {file = "pydantic-1.10.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:970b1bdc6243ef663ba5c7e36ac9ab1f2bfecb8ad297c9824b542d41a750b298"}, + {file = "pydantic-1.10.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7e1d5290044f620f80cf1c969c542a5468f3656de47b41aa78100c5baa2b8276"}, + {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83fcff3c7df7adff880622a98022626f4f6dbce6639a88a15a3ce0f96466cb60"}, + {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0da48717dc9495d3a8f215e0d012599db6b8092db02acac5e0d58a65248ec5bc"}, + {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0a2aabdc73c2a5960e87c3ffebca6ccde88665616d1fd6d3db3178ef427b267a"}, + {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9863b9420d99dfa9c064042304868e8ba08e89081428a1c471858aa2af6f57c4"}, + {file = "pydantic-1.10.9-cp39-cp39-win_amd64.whl", hash = "sha256:e7c9900b43ac14110efa977be3da28931ffc74c27e96ee89fbcaaf0b0fe338e1"}, + {file = "pydantic-1.10.9-py3-none-any.whl", hash = "sha256:6cafde02f6699ce4ff643417d1a9223716ec25e228ddc3b436fe7e2d25a1f305"}, + {file = "pydantic-1.10.9.tar.gz", hash = "sha256:95c70da2cd3b6ddf3b9645ecaa8d98f3d80c606624b6d245558d202cd23ea3be"}, ] [package.dependencies] @@ -1574,4 +1574,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "a874606ca40e35dc579439761534dca8fe5092bd8e5d23c7bd283de140cd4c06" +content-hash = "1877f1f3f8b978205762928ee49ab5b9c88ac29a05bde03f0ef3fe567356b2dc" diff --git a/pyproject.toml b/pyproject.toml index bd9f660..e43756b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ pyhumps = "3.0.2" pyarrow = "^10.0.1" humps = "0.2.2" flatdict = "4.0.1" -odd-collector-sdk = "^0.3.27" +odd-collector-sdk = "^0.3.31" lark-parser = "^0.12.0" deltalake = "^0.9.0"