From 81d3b02426299a19405bd18c287a6b96bad59386 Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Wed, 24 Jul 2024 18:22:22 -0700 Subject: [PATCH] fix: update to new api domain, remove API 'version' config --- .../destination_glide/destination.py | 15 +++++------ .../destination_glide/glide.py | 15 +++-------- .../GlideBigTableRestStrategy_int_test.py | 27 +++++++++++++------ .../destination-glide/scripts/test-unit.sh | 1 + .../GlideBigTableRestStrategy_test.py | 7 +++-- 5 files changed, 33 insertions(+), 32 deletions(-) diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py index 0f7dcabe4a0e..34f798d044e7 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/destination.py @@ -22,8 +22,8 @@ from typing import Any, Iterable, Mapping import uuid -CONFIG_GLIDE_API_VERSION_DEFAULT = "tables" -CONFIG_GLIDE_API_HOST_DEFAULT = "https://api.glideapp.io" +CONFIG_GLIDE_API_HOST_DEFAULT = "https://api.glideapps.com" +CONFIG_GLIDE_API_PATH_ROOT_DEFAULT = "" logger = logging.getLogger(__name__) logger.setLevel(LOG_LEVEL_DEFAULT) @@ -73,19 +73,16 @@ def write( """ # load user-specified config: api_host = config.get('api_host', CONFIG_GLIDE_API_HOST_DEFAULT) - api_path_root = config['api_path_root'] - api_key = config['api_key'] - glide_api_version = config.get( - 'glide_api_version', CONFIG_GLIDE_API_VERSION_DEFAULT) + api_path_root = config.get('api_path_root', CONFIG_GLIDE_API_PATH_ROOT_DEFAULT) + api_key = config.get('api_key') # configure the table based on the stream catalog: # choose a strategy based on config: def create_table_client_for_stream(stream_name): # TODO: sanitize stream_name chars and length for GBT name - glide = GlideBigTableFactory.create(glide_api_version) - logger.debug(f"Using glide api strategy '{glide.__class__.__name__}' for glide_api_version '{glide_api_version}'.") # nopep8 - glide.init(api_host, api_key, api_path_root, stream_name) + glide = GlideBigTableFactory.create() + glide.init(api_key, stream_name, api_host, api_path_root) return glide table_clients = {} diff --git a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py index a60849edc995..d4dee296c4b7 100644 --- a/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py +++ b/airbyte-integrations/connectors/destination-glide/destination_glide/glide.py @@ -18,7 +18,6 @@ "json", ] - class Column(dict): """ Represents a Column in the glide API. @@ -54,7 +53,7 @@ def headers(self) -> Dict[str, str]: } def url(self, path: str) -> str: - return f"{self.api_host}/{self.api_path_root}/{path}" + return f"{self.api_host}/{self.api_path_root + '/' if self.api_path_root != '' else ''}{path}" """ An API client for interacting with a Glide Big Table. The intention is to @@ -64,7 +63,7 @@ def url(self, path: str) -> str: The protocol is to call `init`, `set_schema`, `add_rows` one or more times, and `commit` in that order. """ - def init(self, api_host, api_key, api_path_root, table_name): + def init(self, api_key, table_name, api_host="https://api.glideapps.com", api_path_root=""): """ Sets the connection information for the table. """ @@ -102,17 +101,11 @@ class GlideBigTableFactory: Factory for creating a GlideBigTableBase API client. """ @classmethod - def create(cls, strategy: str) -> GlideBigTableBase: + def create(cls) -> GlideBigTableBase: """ Creates a new instance of the default implementation for the GlideBigTable API client. """ - implementation_map = { - "tables": lambda: GlideBigTableRestStrategy() - } - if strategy not in implementation_map: - raise ValueError(f"Strategy '{strategy}' not found. Expected one of '{implmap.keys()}'.") # nopep8 - return implementation_map[strategy]() - + return GlideBigTableRestStrategy() class GlideBigTableRestStrategy(GlideBigTableBase): def reset(self): diff --git a/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py b/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py index f5ffd39298ab..9b0942d0fd19 100644 --- a/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py +++ b/airbyte-integrations/connectors/destination-glide/integration_tests/GlideBigTableRestStrategy_int_test.py @@ -16,9 +16,20 @@ class TestGlideBigTableRestStrategy(unittest.TestCase): Tests against a working Glide /tables API endpoint rather than being mocked like the one in unit tests. ''' - api_host = "https://functions.prod.internal.glideapps.com" api_key = None - api_path_root = "api" + + env = "prod" + if (env == "poc"): + api_host = "https://functions.prod.internal.glideapps.com" + api_path_root = "api" + elif env=="staging": + api_host = "https://api.staging.glideapps.com" + api_path_root = "" + elif env=="prod": + api_host = "https://api.glideapps.com" + api_path_root = "" + else: + raise Exception(f"Unknown env: {env}") def setUp(self): @@ -31,10 +42,10 @@ def setUp(self): def test_new_table(self): # init - gbt = GlideBigTableFactory().create("tables") + gbt = GlideBigTableFactory().create() table_name = f"test-table-{str(uuid.uuid4())}" - gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) # set_schema test_columns = [ @@ -71,8 +82,8 @@ def test_updating_table(self): # init table_name = f"test-table-{str(uuid.uuid4())}" - gbt = GlideBigTableFactory().create("tables") - gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + gbt = GlideBigTableFactory().create() + gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) # set_schema test_columns = [ @@ -96,8 +107,8 @@ def test_updating_table(self): ##### NOW update the existing table we just created: # now do the update the second table now: - gbt = GlideBigTableFactory().create("tables") - gbt.init(self.api_host, self.api_key, self.api_path_root, table_name) + gbt = GlideBigTableFactory().create() + gbt.init(self.api_key, table_name, self.api_host, self.api_path_root) gbt.set_schema(test_columns) now = datetime.now() diff --git a/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh b/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh index 52bcb42674cb..8f55f0e395b7 100755 --- a/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh +++ b/airbyte-integrations/connectors/destination-glide/scripts/test-unit.sh @@ -2,4 +2,5 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory this_script=$(basename $0) +# NOTE: -k EXPRESSION Only run tests which match the given substring expression. An expression is a Python evaluable expression where all names are substring-matched against test names and their parent classes. Example: -k 'test_method or test_other' matches all test functions and classes whose name contains 'test_method' or 'test_other'... poetry run pytest unit_tests "$@" diff --git a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py index ade04df3cf81..d0e4721c2cba 100644 --- a/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py +++ b/airbyte-integrations/connectors/destination-glide/unit_tests/GlideBigTableRestStrategy_test.py @@ -24,8 +24,7 @@ def setUp(self): self.table_name = f"test-table-name-{str(uuid.uuid4())}" self.stash_id = f"stash-id-{str(uuid.uuid4())}" self.gbt = GlideBigTableRestStrategy() - self.gbt.init(self.api_host, self.api_key, - self.api_path_root, self.table_name) + self.gbt.init(self.api_key, self.table_name, self.api_host, self.api_path_root) def mock_post_for_set_schema(self, mock_post): mock_post.return_value.status_code = 200 @@ -71,7 +70,7 @@ def test_add_rows(self, mock_post): mock_post.assert_called_once() self.assertEqual( - mock_post.call_args.kwargs["json"]["data"], test_rows) + mock_post.call_args.kwargs["json"], test_rows) @patch.object(requests, "post") def test_add_rows_batching(self, mock_post): @@ -90,7 +89,7 @@ def test_add_rows_batching(self, mock_post): self.assertEqual(5, mock_post.call_count) # validate that the last row is what we expect: - self.assertEqual(mock_post.call_args.kwargs["json"]["data"], + self.assertEqual(mock_post.call_args.kwargs["json"], [ {"test-str": f"one {TEST_ROW_COUNT-1}", "test-num": TEST_ROW_COUNT-1} ])