From ae3579a807dd1851bf83c530aa2d6b15bb793173 Mon Sep 17 00:00:00 2001 From: Simon Schoonjans Date: Tue, 9 Jan 2024 17:26:36 +0100 Subject: [PATCH] fix: set correct httpx post arg based on content-type (#8) --- .github/workflows/ci.yml | 2 + Makefile | 2 + pyproject.toml | 5 +- requirements/requirements.all.3.11.txt | 12 +++-- requirements/requirements.dev.3.11.txt | 12 +++-- src/waylay/api/api_client.py | 63 +++----------------------- src/waylay/api/rest.py | 54 +++++++++++----------- 7 files changed, 59 insertions(+), 91 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eafe7e4..5584cdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Set up Git with token to access other private repositories + run: git config --global url."https://${{ secrets.OTOMATOR_PAT }}@github".insteadOf https://github - name: Setup Python 3.11 uses: actions/setup-python@v5 with: diff --git a/Makefile b/Makefile index 43febb0..e38cd04 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,8 @@ install-dependencies: pip install -r requirements/requirements.$$(bin/pyversion).txt install-dev-dependencies: +# need to install the `waylay` first, because the `waylay_` packages require it + pip install -e . --no-deps pip install -r requirements/requirements.$$(bin/pyversion).txt pip install -r requirements/requirements.dev.$$(bin/pyversion).txt diff --git a/pyproject.toml b/pyproject.toml index 30cf2e4..ead08f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,12 +37,13 @@ dev = [ 'types-python-dateutil', 'docformatter', 'waylay[services,services-types]', + 'python-magic', ] services = [ - "waylay_registry_api @ git+https://github.com/waylayio/waylay-py-services@feat/waylay_namespace#subdirectory=services/registry/waylay_registry_api" + "waylay_registry_api @ git+https://github.com/waylayio/waylay-py-services@feat/multipart_uploads#subdirectory=services/registry/waylay_registry_api" ] services-types = [ - "waylay_registry_types @ git+https://github.com/waylayio/waylay-py-services@feat/waylay_namespace#subdirectory=services/registry/waylay_registry_types" + "waylay_registry_types @ git+https://github.com/waylayio/waylay-py-services@feat/multipart_uploads#subdirectory=services/registry/waylay_registry_types" ] [tool.pytest.ini_options] env = [ diff --git a/requirements/requirements.all.3.11.txt b/requirements/requirements.all.3.11.txt index 7ab2f09..88cd379 100644 --- a/requirements/requirements.all.3.11.txt +++ b/requirements/requirements.all.3.11.txt @@ -1,3 +1,4 @@ +annotated-types==0.6.0 anyio==4.2.0 appdirs==1.4.4 astroid==3.0.2 @@ -21,20 +22,25 @@ platformdirs==4.1.0 pluggy==1.3.0 pyasn1==0.5.1 pycodestyle==2.11.1 +pydantic==2.5.3 +pydantic_core==2.14.6 pydocstyle==6.3.0 pylint==3.0.3 pytest==7.4.4 pytest-mock==3.12.0 python-dateutil==2.8.2 python-jose==3.3.0 +python-magic==0.4.27 rsa==4.9 six==1.16.0 sniffio==1.3.0 snowballstemmer==2.2.0 tomlkit==0.12.3 types-appdirs==1.4.3.5 -types-pyasn1==0.5.0.20231222 -types-python-dateutil==2.8.19.14 -types-python-jose==3.3.4.8 +types-pyasn1==0.5.0.20240106 +types-python-dateutil==2.8.19.20240106 +types-python-jose==3.3.4.20240106 typing_extensions==4.9.0 untokenize==0.1.1 +waylay_registry_api @ git+https://github.com/waylayio/waylay-py-services@7ab6b2cb2d5587d32bdddfa918b0bfb88beffd9c#subdirectory=services/registry/waylay_registry_api +waylay_registry_types @ git+https://github.com/waylayio/waylay-py-services@7ab6b2cb2d5587d32bdddfa918b0bfb88beffd9c#subdirectory=services/registry/waylay_registry_types diff --git a/requirements/requirements.dev.3.11.txt b/requirements/requirements.dev.3.11.txt index b132130..be9adec 100644 --- a/requirements/requirements.dev.3.11.txt +++ b/requirements/requirements.dev.3.11.txt @@ -1,3 +1,4 @@ +annotated-types==0.6.0 astroid==3.0.2 autopep8==2.0.4 charset-normalizer==3.3.2 @@ -12,14 +13,19 @@ packaging==23.2 platformdirs==4.1.0 pluggy==1.3.0 pycodestyle==2.11.1 +pydantic==2.5.3 +pydantic_core==2.14.6 pydocstyle==6.3.0 pylint==3.0.3 pytest-mock==3.12.0 pytest==7.4.4 +python-magic==0.4.27 snowballstemmer==2.2.0 tomlkit==0.12.3 types-appdirs==1.4.3.5 -types-pyasn1==0.5.0.20231222 -types-python-dateutil==2.8.19.14 -types-python-jose==3.3.4.8 +types-pyasn1==0.5.0.20240106 +types-python-dateutil==2.8.19.20240106 +types-python-jose==3.3.4.20240106 untokenize==0.1.1 +waylay_registry_api @ git+https://github.com/waylayio/waylay-py-services@7ab6b2cb2d5587d32bdddfa918b0bfb88beffd9c#subdirectory=services/registry/waylay_registry_api +waylay_registry_types @ git+https://github.com/waylayio/waylay-py-services@7ab6b2cb2d5587d32bdddfa918b0bfb88beffd9c#subdirectory=services/registry/waylay_registry_types diff --git a/src/waylay/api/api_client.py b/src/waylay/api/api_client.py index 130cd98..9552c5c 100644 --- a/src/waylay/api/api_client.py +++ b/src/waylay/api/api_client.py @@ -121,12 +121,8 @@ def param_serialize( # post parameters if files: - files = self.files_parameters(files) + files = self._sanitize_files_parameters(files) - # auth setting - # TODO ??? - - # body if body: body = self._sanitize_for_serialization(body) @@ -261,7 +257,10 @@ def _sanitize_for_serialization(self, obj): # and attributes which value is not None. # Convert attribute name to json key in # model definition for request. - obj_dict = obj.to_dict() + try: + obj_dict = obj.to_dict() + except AttributeError: + return obj return { key: self._sanitize_for_serialization(val) @@ -332,63 +331,15 @@ def __deserialize(self, data, klass): else: return data - def files_parameters(self, files=None): + def _sanitize_files_parameters(self, files=None): """Build form parameters. :param files: File parameters. :return: Form parameters with files. """ - params = {} - - if files: - for k, v in files.items(): - if not v: - continue - file_names = v if isinstance(v, list) else [v] - for n in file_names: - with open(n, 'rb') as f: - filename = os.path.basename(f.name) - filedata = f.read() - mimetype = ( - mimetypes.guess_type(filename)[0] - or 'application/octet-stream' - ) - params[k] = tuple([filename, filedata, mimetype]) - - return params - - def select_header_accept(self, accepts: List[str]) -> Optional[str]: - """Return `Accept` based on an array of accepts provided. - - :param accepts: List of headers. - :return: Accept (e.g. application/json). - - """ - if not accepts: - return None - - for accept in accepts: - if re.search('json', accept, re.IGNORECASE): - return accept - - return accepts[0] - - def select_header_content_type(self, content_types): - """Return `Content-Type` based on an array of content_types provided. - - :param content_types: List of content-types. - :return: Content-Type (e.g. application/json). - - """ - if not content_types: - return None - - for content_type in content_types: - if re.search('json', content_type, re.IGNORECASE): - return content_type - return content_types[0] + return files def __deserialize_file(self, response: rest.RESTResponse): """Deserializes body to file. diff --git a/src/waylay/api/rest.py b/src/waylay/api/rest.py index 02e71e5..57437fa 100644 --- a/src/waylay/api/rest.py +++ b/src/waylay/api/rest.py @@ -1,5 +1,6 @@ """REST client implementation.""" +from io import BufferedReader from typing import Any, Optional import httpx from waylay.api.api_config import ApiConfig @@ -74,34 +75,33 @@ async def request( ): timeout = _request_timeout + kwargs = { + 'method': method, + 'url': url, + 'params': query, + 'timeout': timeout, + 'headers': headers + } + # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: content_type = headers.get('Content-Type') - if content_type and content_type == 'multipart/form-data': - return await self.client.request( - method, - url, - params=query, - files=files, - timeout=timeout, - headers=headers - ) + if files or content_type and content_type == 'multipart/form-data': + kwargs.update({'files': files}) + elif isinstance(body, (bytes, bytearray, BufferedReader)): + if isinstance(body, BufferedReader): + body = body.read() + if not headers.get('content-type'): + try: + import magic + mime_type = magic.from_buffer(body) + except BaseException: + mime_type = 'application/octet-stream' + kwargs['headers'].update({'content-type': mime_type}) + kwargs.update({'content': body}) + elif content_type and content_type != 'application/json': + kwargs.update({'data': body}) else: - return await self.client.request( - method, - url, - params=query, - data=body, - timeout=timeout, - headers=headers - ) - - # For `GET`, `HEAD` - else: - return await self.client.request( - method, - url, - params=query, - timeout=timeout, - headers=headers - ) + kwargs.update({'json': body}) + + return await self.client.request(**kwargs)