feat(source-gcs): updated support level, logging, cat for oauth (#45923)

airbytehq · Oct 29, 2024 · fdca55a · fdca55a
1 parent 6002cf8
commit fdca55a
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 6 deletions.
diff --git a/airbyte-integrations/connectors/source-gcs/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gcs/acceptance-test-config.yml
@@ -16,6 +16,8 @@ acceptance_tests:
         status: exception
       - config_path: "secrets/config_glob.json"
         status: succeed
+      - config_path: "secrets/config_oauth.json"
+        status: succeed
   discovery:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml
@@ -1,7 +1,7 @@
 data:
   ab_internal:
     ql: 200
-    sl: 100
+    sl: 300
   allowedHosts:
     hosts:
       - storage.googleapis.com
@@ -11,7 +11,7 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820
-  dockerImageTag: 0.8.0
+  dockerImageTag: 0.8.1
   dockerRepository: airbyte/source-gcs
   documentationUrl: https://docs.airbyte.com/integrations/sources/gcs
   githubIssueLabel: source-gcs
@@ -29,7 +29,7 @@ data:
     oss:
       enabled: true
   releaseStage: alpha
-  supportLevel: community
+  supportLevel: certified
   tags:
     - language:python
     - cdk:python-file-based
@@ -64,4 +64,9 @@ data:
           secretStore:
             type: GSM
             alias: airbyte-connector-testing-secret-store
+        - name: SECRET_SOURCE-GCS_OAUTH
+          fileName: config_oauth.json
+          secretStore:
+            type: GSM
+            alias: airbyte-connector-testing-secret-store
 metadataSpecVersion: "1.0"
diff --git a/airbyte-integrations/connectors/source-gcs/pyproject.toml b/airbyte-integrations/connectors/source-gcs/pyproject.toml
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.8.0"
+version = "0.8.1"
 name = "source-gcs"
 description = "Source implementation for Gcs."
 authors = [ "Airbyte <[email protected]>",]

diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py
@@ -4,7 +4,7 @@
 from typing import Literal, Union
 
 from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
-from pydantic import BaseModel, Field
+from pydantic.v1 import BaseModel, Field
 
 
 class OAuthCredentials(BaseModel):

diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py
@@ -19,6 +19,9 @@
 from source_gcs.helpers import GCSRemoteFile
 from source_gcs.zip_helper import ZipHelper
 
+# google can raise warnings for end user credentials, wrapping it to Logger
+logging.captureWarnings(True)
+
 ERROR_MESSAGE_ACCESS = (
     "We don't have access to {uri}. The file appears to have become unreachable during sync."
     "Check whether key {uri} exists in `{bucket}` bucket and/or has proper ACL permissions"
@@ -50,7 +53,9 @@ def _initialize_gcs_client(self):
             raise ValueError("Source config is missing; cannot create the GCS client.")
         if self._gcs_client is None:
             credentials = self._get_credentials()
-            self._gcs_client = storage.Client(credentials=credentials)
+            # using default project to avoid getting project from env, applies only for OAuth creds
+            project = getattr(credentials, "project_id", "default")
+            self._gcs_client = storage.Client(project=project, credentials=credentials)
         return self._gcs_client
 
     def _get_credentials(self):

diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md
@@ -236,6 +236,7 @@ Google Cloud Storage (GCS) supports following file formats:
 
 | Version | Date       | Pull Request                                             | Subject                                                                 |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------|
+| 0.8.1   | 2024-10-28 | [45923](https://github.com/airbytehq/airbyte/pull/45923) | Update logging                                                          |
 | 0.8.0   | 2024-10-28 | [45414](https://github.com/airbytehq/airbyte/pull/45414) | Add support for OAuth authentication                                    |
 | 0.7.4   | 2024-10-12 | [46858](https://github.com/airbytehq/airbyte/pull/46858) | Update dependencies                                                     |
 | 0.7.3   | 2024-10-05 | [46458](https://github.com/airbytehq/airbyte/pull/46458) | Update dependencies                                                     |