From f2d1f1e14a0944191fa6aae65043173dbdbf974e Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 11 Dec 2023 15:20:57 -0500 Subject: [PATCH 1/5] feat: paginate cadt request --- app/crud/chia.py | 74 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/app/crud/chia.py b/app/crud/chia.py index 68e910a..090867d 100644 --- a/app/crud/chia.py +++ b/app/crud/chia.py @@ -36,31 +36,79 @@ def _headers(self) -> Dict[str, str]: return headers def get_climate_units(self, search: Dict[str, Any]) -> Any: + """ + Retrieves all climate units using pagination and given search parameters. + + Args: + search: A dictionary of search parameters. + + Returns: + A JSON object containing all the climate units. + """ + all_units = [] # List to store all units + page = 1 + limit = 10 + try: - params = urlencode(search) - url = urlparse(self.url + "/v1/units") + while True: + # Update search parameters with current page and limit + search_with_pagination = {**search, "page": page, "limit": limit, "hasMarketplaceIdentifier": True} + params = urlencode(search_with_pagination) - r = requests.get(url.geturl(), params=params, headers=self._headers()) - if r.status_code != requests.codes.ok: - logger.error(f"Request Url: {r.url} Error Message: {r.text}") - raise error_code.internal_server_error(message="Call Climate API Failure") + # Construct the URL + url = urlparse(f"{self.url}/v1/units?{params}") - return r.json() + response = requests.get(url.geturl(), headers=self._headers()) + if response.status_code != requests.codes.ok: + logger.error(f"Request Url: {response.url} Error Message: {response.text}") + raise error_code.internal_server_error(message="Call Climate API Failure") + + data = response.json() + + all_units.extend(data['data']) # Add the units from the current page + + if page >= data['pageCount']: + break # Exit loop if all pages have been processed + + page += 1 + + return all_units except TimeoutError as e: logger.error("Call Climate API Timeout, ErrorMessage: " + str(e)) raise error_code.internal_server_error("Call Climate API Timeout") def get_climate_projects(self) -> Any: + """ + Retrieves all climate projects using pagination. + + Returns: + A JSON object containing all the climate projects. + """ + all_projects = [] + page = 1 + limit = 10 + try: - url = urlparse(self.url + "/v1/projects") + while True: + # Construct the URL with page and limit parameters + url = urlparse(f"{self.url}/v1/projects?page={page}&limit={limit}&onlyMarketplaceProjects=true") - r = requests.get(url.geturl(), headers=self._headers()) - if r.status_code != requests.codes.ok: - logger.error(f"Request Url: {r.url} Error Message: {r.text}") - raise error_code.internal_server_error(message="Call Climate API Failure") + response = requests.get(url.geturl(), headers=self._headers()) + if response.status_code != requests.codes.ok: + logger.error(f"Request Url: {response.url} Error Message: {response.text}") + raise error_code.internal_server_error(message="Call Climate API Failure") - return r.json() + data = response.json() + + all_projects.extend(data['data']) # Add the projects from the current page + + if page >= data['pageCount']: + break # Exit loop if all pages have been processed + + page += 1 + + return all_projects except TimeoutError as e: logger.error("Call Climate API Timeout, ErrorMessage: " + str(e)) From 6538a97cace034094de816a907e6079dc69d85ff Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 11 Dec 2023 16:31:22 -0500 Subject: [PATCH 2/5] refactor: consolidate common functionality --- app/crud/chia.py | 76 ++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/app/crud/chia.py b/app/crud/chia.py index 090867d..4f9ba54 100644 --- a/app/crud/chia.py +++ b/app/crud/chia.py @@ -34,49 +34,63 @@ def _headers(self) -> Dict[str, str]: headers["x-api-key"] = self.api_key return headers - - def get_climate_units(self, search: Dict[str, Any]) -> Any: + + def _get_paginated_data(self, path: str, search_params: Dict[str, Any]) -> List[Any]: """ - Retrieves all climate units using pagination and given search parameters. - + Generic function to retrieve paginated data from a given path. + Args: - search: A dictionary of search parameters. - + path: API endpoint path. + search_params: A dictionary of search parameters including pagination. + Returns: - A JSON object containing all the climate units. + A list of all data retrieved from the paginated API. """ - all_units = [] # List to store all units + all_data = [] page = 1 limit = 10 try: while True: # Update search parameters with current page and limit - search_with_pagination = {**search, "page": page, "limit": limit, "hasMarketplaceIdentifier": True} - params = urlencode(search_with_pagination) + params = {**search_params, "page": page, "limit": limit} + encoded_params = urlencode(params) # Construct the URL - url = urlparse(f"{self.url}/v1/units?{params}") + url = urlparse(f"{self.url}{path}?{encoded_params}") response = requests.get(url.geturl(), headers=self._headers()) if response.status_code != requests.codes.ok: logger.error(f"Request Url: {response.url} Error Message: {response.text}") - raise error_code.internal_server_error(message="Call Climate API Failure") + raise error_code.internal_server_error(message="API Call Failure") data = response.json() - all_units.extend(data['data']) # Add the units from the current page + all_data.extend(data['data']) # Add data from the current page if page >= data['pageCount']: break # Exit loop if all pages have been processed page += 1 - return all_units + return all_data except TimeoutError as e: - logger.error("Call Climate API Timeout, ErrorMessage: " + str(e)) - raise error_code.internal_server_error("Call Climate API Timeout") + logger.error("API Call Timeout, ErrorMessage: " + str(e)) + raise error_code.internal_server_error("API Call Timeout") + + def get_climate_units(self, search: Dict[str, Any]) -> Any: + """ + Retrieves all climate units using pagination and given search parameters. + + Args: + search: A dictionary of search parameters. + + Returns: + A JSON object containing all the climate units. + """ + search_with_marketplace = {**search, "hasMarketplaceIdentifier": True} + return self._get_paginated_data("/v1/units", search_with_marketplace) def get_climate_projects(self) -> Any: """ @@ -85,34 +99,8 @@ def get_climate_projects(self) -> Any: Returns: A JSON object containing all the climate projects. """ - all_projects = [] - page = 1 - limit = 10 - - try: - while True: - # Construct the URL with page and limit parameters - url = urlparse(f"{self.url}/v1/projects?page={page}&limit={limit}&onlyMarketplaceProjects=true") - - response = requests.get(url.geturl(), headers=self._headers()) - if response.status_code != requests.codes.ok: - logger.error(f"Request Url: {response.url} Error Message: {response.text}") - raise error_code.internal_server_error(message="Call Climate API Failure") - - data = response.json() - - all_projects.extend(data['data']) # Add the projects from the current page - - if page >= data['pageCount']: - break # Exit loop if all pages have been processed - - page += 1 - - return all_projects - - except TimeoutError as e: - logger.error("Call Climate API Timeout, ErrorMessage: " + str(e)) - raise error_code.internal_server_error("Call Climate API Timeout") + search_params = {"onlyMarketplaceProjects": True} + return self._get_paginated_data("/v1/projects", search_params) def get_climate_organizations(self) -> Any: try: From 59b6833f1ad97a5945788704fc4d825a76a7b7e0 Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Mon, 11 Dec 2023 16:53:16 -0800 Subject: [PATCH 3/5] Catch KeyError and TypeError when parsing units --- app/crud/chia.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/crud/chia.py b/app/crud/chia.py index 4f9ba54..62d4084 100644 --- a/app/crud/chia.py +++ b/app/crud/chia.py @@ -34,7 +34,7 @@ def _headers(self) -> Dict[str, str]: headers["x-api-key"] = self.api_key return headers - + def _get_paginated_data(self, path: str, search_params: Dict[str, Any]) -> List[Any]: """ Generic function to retrieve paginated data from a given path. @@ -66,9 +66,9 @@ def _get_paginated_data(self, path: str, search_params: Dict[str, Any]) -> List[ data = response.json() - all_data.extend(data['data']) # Add data from the current page + all_data.extend(data["data"]) # Add data from the current page - if page >= data['pageCount']: + if page >= data["pageCount"]: break # Exit loop if all pages have been processed page += 1 @@ -82,10 +82,10 @@ def _get_paginated_data(self, path: str, search_params: Dict[str, Any]) -> List[ def get_climate_units(self, search: Dict[str, Any]) -> Any: """ Retrieves all climate units using pagination and given search parameters. - + Args: search: A dictionary of search parameters. - + Returns: A JSON object containing all the climate units. """ @@ -95,7 +95,7 @@ def get_climate_units(self, search: Dict[str, Any]) -> Any: def get_climate_projects(self) -> Any: """ Retrieves all climate projects using pagination. - + Returns: A JSON object containing all the climate projects. """ @@ -178,7 +178,7 @@ def combine_climate_units_and_metadata(self, search: Dict[str, Any]) -> List[Dic try: warehouse_project_id = unit["issuance"]["warehouseProjectId"] project = project_by_id[warehouse_project_id] - except KeyError: + except (KeyError, TypeError): logger.warning(f"Can not get project by warehouse_project_id: {warehouse_project_id}") continue From 31e7d1417f0147f2230b3f93436a9173a04d5eb0 Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Mon, 11 Dec 2023 17:11:43 -0800 Subject: [PATCH 4/5] Mock up call to not use production server --- tests/test_activities_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_activities_api.py b/tests/test_activities_api.py index f71fec6..4270eb7 100644 --- a/tests/test_activities_api.py +++ b/tests/test_activities_api.py @@ -65,6 +65,7 @@ def test_activities_with_empty_db_then_success( fastapi_client.portal = portal # workaround anyio 4.0.0 incompat with TextClient m.setattr(crud.BlockChainCrud, "get_challenge", mock_get_challenge) m.setattr(crud.DBCrud, "select_activity_with_pagination", mock_db_data) + m.setattr(crud.ClimateWareHouseCrud, "combine_climate_units_and_metadata", mock.MagicMock(return_value={})) params = urlencode({}) response = fastapi_client.get("v1/activities/", params=params) From 752147137de2d25344dfdac59852bcb1c3bbab6d Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Tue, 12 Dec 2023 10:44:00 -0800 Subject: [PATCH 5/5] use importlib to get version instead of parsing pyproject.toml --- app/api/v1/cron.py | 2 +- app/logger.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index c78b2c3..277a692 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -132,7 +132,7 @@ async def _scan_token_activity( # This is causing logging for benign errors, so commenting out for now # except json.JSONDecodeError as e: - # logger.error(f"Failed to parse JSON for key {key} in organization {org_name}: {str(e)}") + # logger.error(f"Failed to parse JSON for key {key} in organization {org_name}: {str(e)}") except Exception as e: logger.error(f"An error occurred for organization {org_name} under key {key}: {str(e)}") diff --git a/app/logger.py b/app/logger.py index ce50155..97b9e9c 100644 --- a/app/logger.py +++ b/app/logger.py @@ -1,14 +1,13 @@ from __future__ import annotations +import importlib.metadata import logging + import uvicorn -import toml + from app.config import settings -# Parse pyproject.toml to get the version -with open('pyproject.toml', 'r') as toml_file: - pyproject = toml.load(toml_file) -version = pyproject['tool']['poetry']['version'] +version = importlib.metadata.version("Chia Climate Token Driver") # Define the log format with version log_format = f"%(asctime)s,%(msecs)d {version} %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s"