ref(source-google-ads): raise exception on missing stream (#46543)

Signed-off-by: Artem Inzhyyants <[email protected]>
airbytehq · Oct 8, 2024 · 32297ad · 32297ad
1 parent bff4613
commit 32297ad
Show file tree

Hide file tree

Showing 9 changed files with 344 additions and 145 deletions.
diff --git a/airbyte-integrations/connectors/source-google-ads/metadata.yaml b/airbyte-integrations/connectors/source-google-ads/metadata.yaml
@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
-  dockerImageTag: 3.7.5
+  dockerImageTag: 3.7.6
   dockerRepository: airbyte/source-google-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads
   githubIssueLabel: source-google-ads

diff --git a/airbyte-integrations/connectors/source-google-ads/poetry.lock b/airbyte-integrations/connectors/source-google-ads/poetry.lock
diff --git a/airbyte-integrations/connectors/source-google-ads/pyproject.toml b/airbyte-integrations/connectors/source-google-ads/pyproject.toml
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.7.5"
+version = "3.7.6"
 name = "source-google-ads"
 description = "Source implementation for Google Ads."
 authors = [ "Airbyte <[email protected]>",]
@@ -19,7 +19,7 @@ include = "source_google_ads"
 python = "^3.10,<3.12"
 google-ads = "==25.0.0"
 protobuf = "==4.25.2"
-airbyte-cdk = "^4"
+airbyte-cdk = "^5"
 
 [tool.poetry.scripts]
 source-google-ads = "source_google_ads.run:run"
@@ -28,4 +28,4 @@ source-google-ads = "source_google_ads.run:run"
 pytest-mock = "^3.12.0"
 requests-mock = "^1.11.0"
 freezegun = "^1.4.0"
-pytest = "^6.1"
+pytest = "^7"
diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py
@@ -5,7 +5,7 @@
 
 from typing import Any, List, Mapping
 
-from airbyte_cdk.config_observation import create_connector_config_control_message
+from airbyte_cdk.config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources import Source
@@ -96,18 +96,6 @@ def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str,
         source.write_config(migrated_config, config_path)
         return migrated_config
 
-    @classmethod
-    def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None:
-        """
-        Emits the control messages related to configuration migration.
-
-        Args:
-        - migrated_config (Mapping[str, Any]): The migrated configuration.
-        """
-        cls.message_repository.emit_message(create_connector_config_control_message(migrated_config))
-        for message in cls.message_repository._message_queue:
-            print(message.json(exclude_unset=True))
-
     @classmethod
     def migrate(cls, args: List[str], source: Source) -> None:
         """
@@ -125,4 +113,4 @@ def migrate(cls, args: List[str], source: Source) -> None:
         if config_path:
             config = source.read_config(config_path)
             if cls.should_migrate(config):
-                cls.emit_control_message(cls.modify_and_save(config_path, source, config))
+                emit_configuration_as_airbyte_control_message(cls.modify_and_save(config_path, source, config))
diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py
@@ -49,8 +49,8 @@
 
 
 class SourceGoogleAds(AbstractSource):
-    # Skip exceptions on missing streams
-    raise_exception_on_missing_stream = False
+    # Raise exceptions on missing streams
+    raise_exception_on_missing_stream = True
 
     @staticmethod
     def _validate_and_transform(config: Mapping[str, Any]):

diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py
@@ -8,11 +8,10 @@
 
 import backoff
 import pendulum
-from airbyte_cdk.models import SyncMode
+from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources.streams import CheckpointMixin, Stream
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 from airbyte_cdk.utils import AirbyteTracedException
-from airbyte_protocol.models import FailureType
 from google.ads.googleads.errors import GoogleAdsException
 from google.ads.googleads.v17.services.services.google_ads_service.pagers import SearchPager
 from google.ads.googleads.v17.services.types.google_ads_service import SearchGoogleAdsResponse

diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py
@@ -34,7 +34,7 @@ def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None:
             updated_config.write(config)
 
 
-def test_migrate_config():
+def test_migrate_config(capsys):
     migration_instance = MigrateCustomQuery()
     original_config = load_config()
     original_config_queries = original_config["custom_queries"].copy()
@@ -54,12 +54,13 @@ def test_migrate_config():
     # load the old custom reports VS migrated
     new_config_queries = test_migrated_config["custom_queries_array"].copy()
     new_config_queries[0]["query"] = new_config_queries[0]["query"].replace(", segments.date", "")
-    print(f"{original_config=} \n {test_migrated_config=}")
     assert original_config["custom_queries"] == new_config_queries
     # test CONTROL MESSAGE was emitted
-    control_msg = migration_instance.message_repository._message_queue[0]
-    assert control_msg.type == Type.CONTROL
-    assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG
+    what = capsys.readouterr().out
+    control_msg = json.loads(what)
+    # control_msg = migration_instance.message_repository._message_queue[0]
+    assert control_msg["type"] == Type.CONTROL.value
+    assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value
     # revert the test_config to the starting point
     revert_migration()
 

diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py
@@ -10,7 +10,8 @@
 
 import pendulum
 import pytest
-from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode
+from airbyte_cdk import AirbyteTracedException
+from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, FailureType, SyncMode
 from pendulum import duration, today
 from source_google_ads.custom_query_stream import IncrementalCustomQuery
 from source_google_ads.google_ads import GoogleAds
@@ -139,10 +140,9 @@ def test_read_missing_stream(config, mock_get_customers):
         ]
     )
 
-    try:
+    with pytest.raises(AirbyteTracedException) as e:
         list(source.read(logging.getLogger("airbyte"), config=config, catalog=catalog))
-    except KeyError as error:
-        pytest.fail(str(error))
+    assert e.value.failure_type == FailureType.config_error
 
 
 @pytest.mark.parametrize(

diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md
@@ -60,9 +60,8 @@ Each Google Ads API developer token is assigned an access level and "permissible
 <!-- /env:oss -->
 
 <!-- env:cloud -->
-<HideInUI>
+
 #### For Airbyte Cloud:
-</HideInUI>
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
 2. Click Sources and then click + New source.
@@ -119,6 +118,7 @@ If you are accessing your account through a Google Ads Manager account, you must
 
 <!-- /env:oss -->
 <HideInUI>
+
 ## Supported sync modes
 
 The Google Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):
@@ -335,11 +335,12 @@ Due to a limitation in the Google Ads API which does not allow getting performan
 
 | Version   | Date       | Pull Request                                             | Subject                                                                                                                              |
 |:----------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------|
-| 3.7.5 | 2024-09-21 | [45801](https://github.com/airbytehq/airbyte/pull/45801) | Update dependencies |
-| 3.7.4 | 2024-09-20 | [44600](https://github.com/airbytehq/airbyte/pull/44600) | Update API documentation URLs |
-| 3.7.3 | 2024-09-14 | [45497](https://github.com/airbytehq/airbyte/pull/45497) | Update dependencies |
-| 3.7.2 | 2024-09-07 | [45263](https://github.com/airbytehq/airbyte/pull/45263) | Update dependencies |
-| 3.7.1 | 2024-08-31 | [44326](https://github.com/airbytehq/airbyte/pull/44326) | Update dependencies |
+| 3.7.6     | 2024-09-21 | [46543](https://github.com/airbytehq/airbyte/pull/46543) | Raise exception on missing stream                                                                                                    |
+| 3.7.5     | 2024-09-21 | [45801](https://github.com/airbytehq/airbyte/pull/45801) | Update dependencies                                                                                                                  |
+| 3.7.4     | 2024-09-20 | [44600](https://github.com/airbytehq/airbyte/pull/44600) | Update API documentation URLs                                                                                                        |
+| 3.7.3     | 2024-09-14 | [45497](https://github.com/airbytehq/airbyte/pull/45497) | Update dependencies                                                                                                                  |
+| 3.7.2     | 2024-09-07 | [45263](https://github.com/airbytehq/airbyte/pull/45263) | Update dependencies                                                                                                                  |
+| 3.7.1     | 2024-08-31 | [44326](https://github.com/airbytehq/airbyte/pull/44326) | Update dependencies                                                                                                                  |
 | `3.7.0  ` | 2024-08-15 | [44095](https://github.com/airbytehq/airbyte/pull/44095) | Migrate to google-ads v17                                                                                                            |
 | `3.6.5  ` | 2024-08-12 | [43882](https://github.com/airbytehq/airbyte/pull/43882) | Update dependencies                                                                                                                  |
 | `3.6.4`   | 2024-08-10 | [43628](https://github.com/airbytehq/airbyte/pull/43628) | Update dependencies                                                                                                                  |