From d3d4f3cfa2dc0d18f5a2b5ea6ab8ca34e3951309 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 3 Apr 2024 09:28:34 +0800 Subject: [PATCH] Revert "[#2113] feat(pyClient): initial Gravitino Python client module (#2676)" This reverts commit 441145996c81eb71e5734f509dc761b46614018e. --- clients/client-python/.gitignore | 163 +----------------- clients/client-python/Makefile | 15 -- clients/client-python/README.md | 11 +- .../{gravitino/exceptions.py => __init__.py} | 2 +- clients/client-python/gravitino/__init__.py | 13 -- .../gravitino/gravitino_client.py | 158 ----------------- clients/client-python/gravitino/service.py | 77 --------- clients/client-python/gravitino/typing.py | 11 -- .../gravitino/utils/exceptions.py | 95 ---------- .../gravitino/utils/http_client.py | 145 ---------------- .../__init__.py} | 4 +- .../core}/__init__.py | 4 +- .../gravitino_client/core/dto.py | 11 ++ .../gravitino_client/core/gravitino_client.py | 56 ++++++ clients/client-python/requirements-dev.txt | 2 - clients/client-python/requirements.txt | 4 +- clients/client-python/setup.py | 23 --- clients/client-python/tests/__init__.py | 2 +- clients/client-python/tests/core/__init__.py | 4 + .../tests/core/test_gravitino_client.py | 45 +++++ clients/client-python/tests/fixtures.py | 104 ----------- .../tests/test_gravitino_client.py | 58 ------- clients/client-python/tests/utils.py | 49 ------ 23 files changed, 131 insertions(+), 925 deletions(-) delete mode 100644 clients/client-python/Makefile rename clients/client-python/{gravitino/exceptions.py => __init__.py} (96%) delete mode 100644 clients/client-python/gravitino/__init__.py delete mode 100644 clients/client-python/gravitino/gravitino_client.py delete mode 100644 clients/client-python/gravitino/service.py delete mode 100644 clients/client-python/gravitino/typing.py delete mode 100644 clients/client-python/gravitino/utils/exceptions.py delete mode 100644 clients/client-python/gravitino/utils/http_client.py rename clients/client-python/{gravitino/constants.py => gravitino_client/__init__.py} (84%) rename clients/client-python/{gravitino/utils => gravitino_client/core}/__init__.py (58%) create mode 100644 clients/client-python/gravitino_client/core/dto.py create mode 100644 clients/client-python/gravitino_client/core/gravitino_client.py delete mode 100644 clients/client-python/requirements-dev.txt delete mode 100644 clients/client-python/setup.py create mode 100644 clients/client-python/tests/core/__init__.py create mode 100644 clients/client-python/tests/core/test_gravitino_client.py delete mode 100644 clients/client-python/tests/fixtures.py delete mode 100644 clients/client-python/tests/test_gravitino_client.py delete mode 100644 clients/client-python/tests/utils.py diff --git a/clients/client-python/.gitignore b/clients/client-python/.gitignore index 6769e21d99a..e0549b75b57 100644 --- a/clients/client-python/.gitignore +++ b/clients/client-python/.gitignore @@ -1,160 +1,5 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +__pycache__/ +.pytest_cache/ \ No newline at end of file diff --git a/clients/client-python/Makefile b/clients/client-python/Makefile deleted file mode 100644 index f769e5ac76a..00000000000 --- a/clients/client-python/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -.PHONY: mkvenv install test clean - -mkvenv: - python -m venv venv && source ./venv/bin/activate && pip install -U pip - -install: - source ./venv/bin/activate && pip install -e . - -test: - source ./venv/bin/activate && python -m unittest - -clean: - rm -rf venv diff --git a/clients/client-python/README.md b/clients/client-python/README.md index d69ba3e14ef..8515e255469 100644 --- a/clients/client-python/README.md +++ b/clients/client-python/README.md @@ -5,14 +5,9 @@ # Quick Start -1. Install current library in your local machine. +1. Install dependency ```bash - pip install -e . + pip install -r requirements.txt ``` -# Development Environment - -1. Install dependency - ```bash - pip install -e .[dev] - ``` \ No newline at end of file +2. After the packages is installed, you can simply run `pytest` in any directory inside gravitino/ \ No newline at end of file diff --git a/clients/client-python/gravitino/exceptions.py b/clients/client-python/__init__.py similarity index 96% rename from clients/client-python/gravitino/exceptions.py rename to clients/client-python/__init__.py index 5779a3ad252..c064f0b0f0c 100644 --- a/clients/client-python/gravitino/exceptions.py +++ b/clients/client-python/__init__.py @@ -1,4 +1,4 @@ """ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. -""" +""" \ No newline at end of file diff --git a/clients/client-python/gravitino/__init__.py b/clients/client-python/gravitino/__init__.py deleted file mode 100644 index fdd4d199d30..00000000000 --- a/clients/client-python/gravitino/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from gravitino.gravitino_client import ( - GravitinoClient, - gravitino_metalake, - MetaLake, - Catalog, - Schema, - Table, -) diff --git a/clients/client-python/gravitino/gravitino_client.py b/clients/client-python/gravitino/gravitino_client.py deleted file mode 100644 index a8f97bb85c7..00000000000 --- a/clients/client-python/gravitino/gravitino_client.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from gravitino.constants import TIMEOUT -from gravitino.service import initialize_service, service - - -class MetaLake: - def __init__(self, metalake_name: str): - self.name = metalake_name - self.service = service["service"] - self.metalake = self.service.get_metalake(self.name) - self.catalogs = self.service.list_catalogs(self.name) - - def __repr__(self): - return f"MetaLake<{self.name}>" - - def __getattr__(self, catalog_name): - if catalog_name in dir(self): - return Catalog(self.name, catalog_name) - - def __dir__(self): - return [catalog["name"] for catalog in self.catalogs] - - def __contains__(self, item): - return item in dir(self) - - -class Catalog: - def __init__(self, metalake_name: str, catalog_name: str): - self.metalake_name = metalake_name - self.catalog_name = catalog_name - self.name = catalog_name - self.service = service["service"] - self.schemas = self.service.list_schemas(metalake_name, catalog_name) - - def __repr__(self): - return f"Catalog<{self.name}>" - - def __getattr__(self, schema_name): - if schema_name in dir(self): - return Schema(self.metalake_name, self.catalog_name, schema_name) - - def __dir__(self): - return [schema["name"] for schema in self.schemas] - - def __contains__(self, item): - return item in dir(self) - - -class Schema: - def __init__(self, metalake_name: str, catalog_name: str, schema_name: str): - self.metalake_name = metalake_name - self.catalog_name = catalog_name - self.schema_name = schema_name - self.name = schema_name - self.service = service["service"] - self.tables = self.service.list_tables(metalake_name, catalog_name, schema_name) - - def __repr__(self): - return f"Schema<{self.name}>" - - def __getattr__(self, table_name): - if table_name in dir(self): - return Table( - self.metalake_name, self.catalog_name, self.schema_name, table_name - ) - - def __dir__(self): - return [table["name"] for table in self.tables] - - def __contains__(self, item): - return item in dir(self) - - -class Table: - def __init__( - self, metalake_name: str, catalog_name: str, schema_name: str, table_name: str - ): - self.metalake_name = metalake_name - self.catalog_name = catalog_name - self.schema_name = schema_name - self.table_name = table_name - self.name = schema_name - self.service = service["service"] - - def __repr__(self): - return f"Table<{self.name}>" - - def info(self): - return self.service.get_table( - self.metalake_name, self.catalog_name, self.schema_name, self.table_name - ) - - -class GravitinoClient: - def __init__( - self, - host: str, - *, - prefix: str = "/api", - timeout: int = TIMEOUT, - debug: bool = False, - ) -> None: - _base_url = f"{host.rstrip('/')}/{prefix.strip('/')}" - initialize_service(_base_url, timeout) - self.service = service["service"] - self.debug = debug - - @classmethod - def initialize_metalake( - cls, - host: str, - metalake_name: str, - *, - prefix: str = "/api", - timeout: int = TIMEOUT, - debug: bool = False, - ) -> MetaLake: - # keep in mind, all constructors should include same interface as __init__ function - client = cls( - host, - prefix=prefix, - timeout=timeout, - debug=debug, - ) - return client.get_metalake(metalake_name) - - @property - def version(self): - return self.service.get_version() - - def get_metalakes(self) -> [MetaLake]: - return [ - MetaLake(metalake.get("name")) for metalake in self.service.list_metalakes() - ] - - def get_metalake(self, metalake: str) -> MetaLake: - return MetaLake(metalake) - - -def gravitino_metalake( - host: str, - metalake_name: str, - *, - prefix: str = "/api", - timeout: int = TIMEOUT, - debug: bool = False, -) -> MetaLake: - return GravitinoClient.initialize_metalake( - host, - metalake_name, - prefix=prefix, - timeout=timeout, - debug=debug, - ) diff --git a/clients/client-python/gravitino/service.py b/clients/client-python/gravitino/service.py deleted file mode 100644 index 9bd4cd6dfbe..00000000000 --- a/clients/client-python/gravitino/service.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from gravitino.utils import HTTPClient, unpack, Response -from gravitino.constants import TIMEOUT - - -class _Service: - def __init__( - self, - url: str, - timeout: int = TIMEOUT, - ) -> None: - self.http_client = HTTPClient(url, timeout=timeout) - - @unpack("version") - def get_version(self) -> Response: - return self.http_client.get("/version") - - @unpack("metalakes") - def list_metalakes(self) -> Response: - return self.http_client.get("/metalakes") - - @unpack("metalake") - def get_metalake(self, metalake: str) -> Response: - return self.http_client.get(f"/metalakes/{metalake}") - - @unpack("identifiers") - def list_catalogs(self, metalake: str) -> Response: - return self.http_client.get(f"/metalakes/{metalake}/catalogs/") - - @unpack("catalog") - def get_catalog(self, metalake: str, catalog: str) -> Response: - return self.http_client.get(f"/metalakes/{metalake}/catalogs/{catalog}") - - @unpack("identifiers") - def list_schemas(self, metalake: str, catalog: str) -> Response: - return self.http_client.get(f"/metalakes/{metalake}/catalogs/{catalog}/schemas") - - @unpack("schema") - def get_schema(self, metalake: str, catalog: str, schema: str) -> Response: - return self.http_client.get( - f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}" - ) - - @unpack("identifiers") - def list_tables(self, metalake: str, catalog: str, schema: str) -> Response: - return self.http_client.get( - f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables" - ) - - @unpack("table") - def get_table( - self, metalake: str, catalog: str, schema: str, table: str - ) -> Response: - return self.http_client.get( - f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}" - ) - - @unpack("names") - def list_partitions( - self, metalake: str, catalog: str, schema: str, table: str - ) -> Response: - return self.http_client.get( - f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}/partitions" - ) - - -service = {} - - -def initialize_service(url: str, timeout: int = TIMEOUT): - global service - if not service: - service["service"] = _Service(url, timeout) diff --git a/clients/client-python/gravitino/typing.py b/clients/client-python/gravitino/typing.py deleted file mode 100644 index 4dc7db893f4..00000000000 --- a/clients/client-python/gravitino/typing.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from typing import Mapping, Sequence, Union - -# https://github.com/python/typing/issues/182#issuecomment-1320974824 -JSON_ro = Union[ - Mapping[str, "JSON_ro"], Sequence["JSON_ro"], str, int, float, bool, None -] diff --git a/clients/client-python/gravitino/utils/exceptions.py b/clients/client-python/gravitino/utils/exceptions.py deleted file mode 100644 index 147a1698362..00000000000 --- a/clients/client-python/gravitino/utils/exceptions.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -import json - - -class HTTPError(Exception): - """Base of all other errors""" - - def __init__(self, error): - self.status_code = error.code - self.reason = error.reason - self.body = error.read() - self.headers = error.hdrs - - def json(self): - """ - :return: object of response error from the API - """ - try: - return json.loads(self.body.decode("utf-8")) - except json.decoder.JSONDecodeError: - return {"exception": self.body.decode("utf-8")} - - def __str__(self): - return self.json().get("exception") - - -class BadRequestsError(HTTPError): - pass - - -class UnauthorizedError(HTTPError): - pass - - -class ForbiddenError(HTTPError): - pass - - -class NotFoundError(HTTPError): - pass - - -class MethodNotAllowedError(HTTPError): - pass - - -class PayloadTooLargeError(HTTPError): - pass - - -class UnsupportedMediaTypeError(HTTPError): - pass - - -class TooManyRequestsError(HTTPError): - pass - - -class InternalServerError(HTTPError): - pass - - -class ServiceUnavailableError(HTTPError): - pass - - -class GatewayTimeoutError(HTTPError): - pass - - -err_dict = { - 400: BadRequestsError, - 401: UnauthorizedError, - 403: ForbiddenError, - 404: NotFoundError, - 405: MethodNotAllowedError, - 413: PayloadTooLargeError, - 415: UnsupportedMediaTypeError, - 429: TooManyRequestsError, - 500: InternalServerError, - 503: ServiceUnavailableError, - 504: GatewayTimeoutError, -} - - -def handle_error(error): - try: - exc = err_dict[error.code](error) - except KeyError: - return HTTPError(error) - return exc diff --git a/clients/client-python/gravitino/utils/http_client.py b/clients/client-python/gravitino/utils/http_client.py deleted file mode 100644 index 870d2c2bc31..00000000000 --- a/clients/client-python/gravitino/utils/http_client.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from urllib.request import Request, build_opener -from urllib.parse import urlencode -from urllib.error import HTTPError -import json as _json - -from gravitino.typing import JSON_ro -from gravitino.utils.exceptions import handle_error -from gravitino.constants import TIMEOUT - - -class Response: - def __init__(self, response): - self._status_code = response.getcode() - self._body = response.read() - self._headers = response.info() - self._url = response.url - - @property - def status_code(self): - return self._status_code - - @property - def url(self): - return self._url - - @property - def body(self): - return self._body - - @property - def headers(self): - return self._headers - - def json(self): - if self.body: - return _json.loads(self.body.decode("utf-8")) - else: - return None - - -class HTTPClient: - def __init__( - self, - host, - *, - request_headers=None, - timeout=TIMEOUT, - is_debug=False, - ) -> None: - self.host = host - self.request_headers = request_headers or {} - self.timeout = timeout - self.is_debug = is_debug - - def _build_url(self, endpoint=None, params=None): - url = self.host - - if endpoint: - url = "{}/{}".format(url.rstrip("/"), endpoint.lstrip("/")) - - if params: - params = {k: v for k, v in params.items() if v is not None} - url_values = urlencode(sorted(params.items()), True) - url = "{}?{}".format(url, url_values) - - return url - - def _update_headers(self, request_headers): - self.request_headers.update(request_headers) - - def _mask_auth_headers(self, headers): - if self.is_debug: - return headers - - _headers = {} - for key, value in headers.items(): - if key.lower() == "authorization": - _headers[key] = "******" - else: - _headers[key] = value - return _headers - - def _make_request(self, opener, request, timeout=None): - timeout = timeout or self.timeout - try: - return opener.open(request, timeout=timeout) - except HTTPError as err: - exc = handle_error(err) - exc.__cause__ = None - raise exc - - def _request( - self, method, endpoint, params=None, json=None, headers=None, timeout=None - ): - method = method.upper() - request_data = None - - if headers: - self._update_headers(headers) - if json: - request_data = _json.dumps(json).encode("utf-8") - - opener = build_opener() - request = Request(self._build_url(endpoint, params), data=request_data) - if self.request_headers: - for key, value in self.request_headers.items(): - request.add_header(key, value) - if request_data and ("Content-Type" not in self.request_headers): - request.add_header("Content-Type", "application/json") - - request.get_method = lambda: method - return Response(self._make_request(opener, request, timeout=timeout)) - - def get(self, endpoint, params=None, **kwargs): - return self._request("get", endpoint, params=params, **kwargs) - - def delete(self, endpoint, **kwargs): - return self._request("delete", endpoint, **kwargs) - - def post(self, endpoint, json=None, **kwargs): - return self._request("post", endpoint, json=json, **kwargs) - - def put(self, endpoint, json=None, **kwargs): - return self._request("put", endpoint, json=json, **kwargs) - - -def unpack(path: str): - def decorator(func): - def wrapper(*args, **kwargs) -> JSON_ro: - resp = func(*args, **kwargs) - rv = resp.json() - for p in path.split("."): - if p not in rv: - raise KeyError(f"The path '{path}' can't find in dict") - rv = rv.get(p) - return rv - - return wrapper - - return decorator diff --git a/clients/client-python/gravitino/constants.py b/clients/client-python/gravitino_client/__init__.py similarity index 84% rename from clients/client-python/gravitino/constants.py rename to clients/client-python/gravitino_client/__init__.py index bca1d178c49..c064f0b0f0c 100644 --- a/clients/client-python/gravitino/constants.py +++ b/clients/client-python/gravitino_client/__init__.py @@ -1,6 +1,4 @@ """ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. -""" - -TIMEOUT = 10 +""" \ No newline at end of file diff --git a/clients/client-python/gravitino/utils/__init__.py b/clients/client-python/gravitino_client/core/__init__.py similarity index 58% rename from clients/client-python/gravitino/utils/__init__.py rename to clients/client-python/gravitino_client/core/__init__.py index 03295d89182..16d5ed38fd3 100644 --- a/clients/client-python/gravitino/utils/__init__.py +++ b/clients/client-python/gravitino_client/core/__init__.py @@ -2,5 +2,5 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ - -from gravitino.utils.http_client import Response, HTTPClient, unpack +from .gravitino_client import GravitinoClient +from .dto import VersionDTO \ No newline at end of file diff --git a/clients/client-python/gravitino_client/core/dto.py b/clients/client-python/gravitino_client/core/dto.py new file mode 100644 index 00000000000..f6fc0f52c44 --- /dev/null +++ b/clients/client-python/gravitino_client/core/dto.py @@ -0,0 +1,11 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from dataclasses import dataclass + +@dataclass +class VersionDTO: + version: str + compile_date: str + git_commit: str diff --git a/clients/client-python/gravitino_client/core/gravitino_client.py b/clients/client-python/gravitino_client/core/gravitino_client.py new file mode 100644 index 00000000000..eed39431b0e --- /dev/null +++ b/clients/client-python/gravitino_client/core/gravitino_client.py @@ -0,0 +1,56 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +import requests +from requests.exceptions import HTTPError +from gravitino_client.core.dto import VersionDTO + + +class GravitinoClient: + """ + Gravitino Client for interacting with the Gravitino API, allowing the client to list, load, + create, and alter Metalakes. + + Attributes: + base_url (str): The base URL of the Gravitino API to which the client will make requests. + + Args: + base_url (str): The base URL for the Gravitino API. + """ + + def __init__(self, base_url): + """ + Initializes a new instance of the GravitinoClient. + + Args: + base_url (str): The base URL for the Gravitino API where the client will send requests. + """ + self.base_url = base_url + + def getVersion(self) -> VersionDTO: + """ + Retrieves the version information from the Gravitino API. + + This method makes a GET request to the Gravitino API and extracts the version information from the response, + wrapping it into a VersionDTO object. + + Returns: + VersionDTO: An object containing the version details, including version, compile date, and git commit hash. + + Raises: + HTTPError: An error from the requests library if the HTTP request returned an unsuccessful status code. + """ + try: + response = requests.get(f"{self.base_url}/api/version") + response.raise_for_status() + version_data = response.json() + version_info = version_data.get("version") + + return VersionDTO( + version=version_info['version'], + compile_date=version_info['compileDate'], + git_commit=version_info['gitCommit'] + ) + except HTTPError as e: + raise HTTPError(f"Failed to retrieve version information: {e}") diff --git a/clients/client-python/requirements-dev.txt b/clients/client-python/requirements-dev.txt deleted file mode 100644 index 2cfa42afe23..00000000000 --- a/clients/client-python/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. \ No newline at end of file diff --git a/clients/client-python/requirements.txt b/clients/client-python/requirements.txt index 2cfa42afe23..5f118436582 100644 --- a/clients/client-python/requirements.txt +++ b/clients/client-python/requirements.txt @@ -1,2 +1,4 @@ # Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. \ No newline at end of file +# This software is licensed under the Apache License version 2. +pytest~=8.0.1 +requests~=2.31.0 \ No newline at end of file diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py deleted file mode 100644 index e130f2ed0fd..00000000000 --- a/clients/client-python/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from setuptools import find_packages, setup - - -setup( - name="gravitino", - description="project description TBD", - version="0.0.1", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - url="https://github.com/datastrato/gravitino", - author="datastrato", - python_requires=">=3.8", - packages=find_packages(include=["gravitino", ".*"]), - install_requires=open("requirements.txt").read(), - extras_require={ - "dev": open("requirements-dev.txt").read(), - }, -) diff --git a/clients/client-python/tests/__init__.py b/clients/client-python/tests/__init__.py index 5779a3ad252..c064f0b0f0c 100644 --- a/clients/client-python/tests/__init__.py +++ b/clients/client-python/tests/__init__.py @@ -1,4 +1,4 @@ """ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. -""" +""" \ No newline at end of file diff --git a/clients/client-python/tests/core/__init__.py b/clients/client-python/tests/core/__init__.py new file mode 100644 index 00000000000..c064f0b0f0c --- /dev/null +++ b/clients/client-python/tests/core/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" \ No newline at end of file diff --git a/clients/client-python/tests/core/test_gravitino_client.py b/clients/client-python/tests/core/test_gravitino_client.py new file mode 100644 index 00000000000..7d5ffba9eca --- /dev/null +++ b/clients/client-python/tests/core/test_gravitino_client.py @@ -0,0 +1,45 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +import pytest +import requests +from unittest.mock import MagicMock +from gravitino_client.core import GravitinoClient, VersionDTO + + +@pytest.fixture +def mock_get(monkeypatch): + mock = MagicMock() + monkeypatch.setattr("requests.get", mock) + return mock + + +def test_get_version_success(mock_get): + expected_version_dto = VersionDTO( + version="0.3.2-SNAPSHOT", + compile_date="25/01/2024 00:04:59", + git_commit="cb7a604bf19b6f992f00529e938cdd1d37af0187" + ) + mock_get.return_value.json.return_value = { + "code": 0, + "version": { + "version": "0.3.2-SNAPSHOT", + "compileDate": "25/01/2024 00:04:59", + "gitCommit": "cb7a604bf19b6f992f00529e938cdd1d37af0187" + } + } + + client = GravitinoClient(base_url="http://localhost:8090") + version_data = client.getVersion() + + assert version_data == expected_version_dto + + +def test_get_version_http_error(mock_get): + mock_get.side_effect = requests.exceptions.HTTPError + + client = GravitinoClient(base_url="http://localhost:8090") + + with pytest.raises(requests.exceptions.HTTPError): + client.getVersion() diff --git a/clients/client-python/tests/fixtures.py b/clients/client-python/tests/fixtures.py deleted file mode 100644 index dbf47969934..00000000000 --- a/clients/client-python/tests/fixtures.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -services_version = { - "version": "0.4.0", - "compileDate": "06/02/2024 08:37:11", - "gitCommit": "ae87dbdef5a749cdbed66d7f0e841cc809ad2510", -} -services_list_metalakes = [ - { - "name": "metalake_demo", - "comment": "comment", - "audit": {"creator": "anonymous", "createTime": "2024-03-30T13:49:53.382Z"}, - } -] -services_get_metalake = { - "name": "metalake_demo", - "comment": "comment", - "audit": {"creator": "anonymous", "createTime": "2024-03-30T13:49:53.382Z"}, -} -services_list_catalogs = [ - {"namespace": ["metalake_demo"], "name": "catalog_hive"}, - {"namespace": ["metalake_demo"], "name": "catalog_iceberg"}, - {"namespace": ["metalake_demo"], "name": "catalog_postgres"}, -] -services_get_catalog = {"namespace": ["metalake_demo"], "name": "catalog_hive"} -services_list_schemas = [ - {"namespace": ["metalake_demo", "catalog_hive"], "name": "default"}, - {"namespace": ["metalake_demo", "catalog_hive"], "name": "sales"}, -] -services_get_schema = {"namespace": ["metalake_demo", "catalog_hive"], "name": "sales"} -services_list_tables = [ - {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "categories"}, - {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "customers"}, - {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "products"}, - {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "sales"}, - {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "stores"}, -] -services_get_table = { - "name": "sales", - "comment": "", - "columns": [ - { - "name": "sale_id", - "type": "integer", - "nullable": True, - "autoIncrement": False, - }, - { - "name": "employee_id", - "type": "integer", - "nullable": True, - "autoIncrement": False, - }, - { - "name": "store_id", - "type": "integer", - "nullable": True, - "autoIncrement": False, - }, - { - "name": "product_id", - "type": "integer", - "nullable": True, - "autoIncrement": False, - }, - { - "name": "customer_id", - "type": "integer", - "nullable": True, - "autoIncrement": False, - }, - {"name": "sold", "type": "date", "nullable": True, "autoIncrement": False}, - { - "name": "quantity", - "type": "integer", - "nullable": True, - "autoIncrement": False, - }, - { - "name": "total_amount", - "type": "decimal(10,2)", - "nullable": True, - "autoIncrement": False, - }, - ], - "properties": { - "input-format": "org.apache.hadoop.mapred.TextInputFormat", - "transient_lastDdlTime": "1711806631", - "output-format": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "location": "hdfs://hive:9000/user/hive/warehouse/sales.db/sales", - "table-type": "MANAGED_TABLE", - "serde-lib": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", - "STATS_GENERATED_VIA_STATS_TASK": "workaround for potential lack of HIVE-12730", - "serde-name": "sales", - }, - "audit": {"creator": "anonymous", "createTime": "2024-03-30T13:50:31.289Z"}, - "distribution": {"strategy": "none", "number": 0, "funcArgs": []}, - "sortOrders": [], - "partitioning": [], - "indexes": [], -} diff --git a/clients/client-python/tests/test_gravitino_client.py b/clients/client-python/tests/test_gravitino_client.py deleted file mode 100644 index ad497b2ba7f..00000000000 --- a/clients/client-python/tests/test_gravitino_client.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -import unittest - -from gravitino import GravitinoClient, gravitino_metalake -from .utils import services_fixtures - - -@services_fixtures -class TestGravitinoClient(unittest.TestCase): - def setUp(self): - self.client = GravitinoClient("http://localhost:9000") - - def test_version(self, *args): - self.assertIn("version", list(self.client.version.keys())) - - def test_get_metalakes(self, *args): - metalakes = self.client.get_metalakes() - self.assertEqual(len(metalakes), 1) - self.assertEqual(metalakes[0].name, "metalake_demo") - - def test_get_metalake(self, *args): - metalake = self.client.get_metalake("metalake_demo") - self.assertEqual(metalake.name, "metalake_demo") - self.assertIn("catalog_hive", metalake) - - def test_get_catalog(self, *args): - catalog = self.client.get_metalake("metalake_demo").catalog_hive - self.assertEqual(catalog.name, "catalog_hive") - self.assertIn("sales", catalog) - - def test_get_schema(self, *args): - schema = self.client.get_metalake("metalake_demo").catalog_hive.sales - self.assertEqual(schema.name, "sales") - self.assertIn("sales", schema) - - def test_get_table(self, *args): - table = self.client.get_metalake("metalake_demo").catalog_hive.sales.sales - self.assertEqual(table.name, "sales") - self.assertEqual(table.info().get("name"), "sales") - - def test_dynamic_properties(self, *args): - metalake = self.client.get_metalake("metalake_demo") - self.assertIn("catalog_hive", dir(metalake)) - self.assertIn("catalog_iceberg", dir(metalake)) - self.assertIn("catalog_postgres", dir(metalake)) - self.assertEqual(metalake.catalog_hive.name, "catalog_hive") - self.assertEqual(metalake.catalog_hive.sales.name, "sales") - - -@services_fixtures -class TestGravitinoMetalake(unittest.TestCase): - def test_gravitino_metalake(self, *args): - metalake = gravitino_metalake("http://localhost:9000", "metalake_demo") - self.assertEqual(metalake.name, "metalake_demo") diff --git a/clients/client-python/tests/utils.py b/clients/client-python/tests/utils.py deleted file mode 100644 index 9953e929b71..00000000000 --- a/clients/client-python/tests/utils.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from unittest.mock import patch -from . import fixtures - - -def services_fixtures(cls): - @patch( - "gravitino.service._Service.get_version", return_value=fixtures.services_version - ) - @patch( - "gravitino.service._Service.list_metalakes", - return_value=fixtures.services_list_metalakes, - ) - @patch( - "gravitino.service._Service.get_metalake", - return_value=fixtures.services_get_metalake, - ) - @patch( - "gravitino.service._Service.list_catalogs", - return_value=fixtures.services_list_catalogs, - ) - @patch( - "gravitino.service._Service.get_catalog", - return_value=fixtures.services_get_catalog, - ) - @patch( - "gravitino.service._Service.list_schemas", - return_value=fixtures.services_list_schemas, - ) - @patch( - "gravitino.service._Service.get_schema", - return_value=fixtures.services_get_schema, - ) - @patch( - "gravitino.service._Service.list_tables", - return_value=fixtures.services_list_tables, - ) - @patch( - "gravitino.service._Service.get_table", - return_value=fixtures.services_get_table, - ) - class Wrapper(cls): - pass - - return Wrapper