Merge branch 'master' into composite_pk_fix

airbyte-cdk/python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.51.1
+current_version = 0.51.2
 commit = False
 
 [bumpversion:file:setup.py]

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.51.2
+Check config against spec in embedded sources and remove list endpoint from connector builder module
+
 ## 0.51.1
 low-code: allow formatting datetime as milliseconds since unix epoch
 

airbyte-cdk/python/Dockerfile

@@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \
     && apk --no-cache add tzdata build-base
 
 # install airbyte-cdk
-RUN pip install --prefix=/install airbyte-cdk==0.51.1
+RUN pip install --prefix=/install airbyte-cdk==0.51.2
 
 # build a clean environment
 FROM base
@@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 # needs to be the same as CDK
-LABEL io.airbyte.version=0.51.1
+LABEL io.airbyte.version=0.51.2
 LABEL io.airbyte.name=airbyte/source-declarative-manifest

airbyte-cdk/python/airbyte_cdk/connector_builder/README.md

@@ -11,17 +11,17 @@ python main.py read --config path/to/config --catalog path/to/catalog
 ```
 
 Note:
-- Requires the keys `__injected_declarative_manifest` and `__command` in its config, where `__injected_declarative_manifest` is a JSON manifest and `__command` is one of the commands handled by the ConnectorBuilderHandler (`stream_read`, `list_streams`, or `resolve_manifest`), i.e.
+- Requires the keys `__injected_declarative_manifest` and `__command` in its config, where `__injected_declarative_manifest` is a JSON manifest and `__command` is one of the commands handled by the ConnectorBuilderHandler (`stream_read` or `resolve_manifest`), i.e.
 ```
 {
   "config": <normal config>,
   "__injected_declarative_manifest": {...},
-  "__command": <"resolve_manifest" | "list_streams" | "test_read">
+  "__command": <"resolve_manifest" | "test_read">
 }
 ```
 *See [ConnectionSpecification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#actor-specification) for details on the `"config"` key if needed.
 
-- When the `__command` is `list_streams` or `resolve_manifest`, the argument to `catalog` should be an empty string.
+- When the `__command` is `resolve_manifest`, the argument to `catalog` should be an empty string.
 
 ### Locally running the docker image
 

airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -4,18 +4,15 @@
 
 import dataclasses
 from datetime import datetime
-from typing import Any, Dict, List, Mapping
-from urllib.parse import urljoin
+from typing import Any, Mapping
 
 from airbyte_cdk.connector_builder.message_grouper import MessageGrouper
 from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog
 from airbyte_cdk.models import Type
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
-from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
-from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
@@ -89,44 +86,5 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
         return error.as_airbyte_message()
 
 
-def list_streams(source: ManifestDeclarativeSource, config: Dict[str, Any]) -> AirbyteMessage:
-    try:
-        streams = [
-            {
-                "name": http_stream.name,
-                "url": urljoin(
-                    http_stream.requester.get_url_base(),
-                    http_stream.requester.get_path(stream_state=None, stream_slice=None, next_page_token=None),
-                ),
-            }
-            for http_stream in _get_http_streams(source, config)
-        ]
-        return AirbyteMessage(
-            type=Type.RECORD,
-            record=AirbyteRecordMessage(
-                data={"streams": streams},
-                emitted_at=_emitted_at(),
-                stream="list_streams",
-            ),
-        )
-    except Exception as exc:
-        return AirbyteTracedException.from_exception(exc, message=f"Error listing streams: {str(exc)}").as_airbyte_message()
-
-
-def _get_http_streams(source: ManifestDeclarativeSource, config: Dict[str, Any]) -> List[SimpleRetriever]:
-    http_streams = []
-    for stream in source.streams(config=config):
-        if isinstance(stream, DeclarativeStream):
-            if isinstance(stream.retriever, SimpleRetriever):
-                http_streams.append(stream.retriever)
-            else:
-                raise TypeError(
-                    f"A declarative stream should only have a retriever of type SimpleRetriever, but received: {stream.retriever.__class__}"
-                )
-        else:
-            raise TypeError(f"A declarative source should only contain streams of type DeclarativeStream, but received: {stream.__class__}")
-    return http_streams
-
-
 def _emitted_at() -> int:
     return int(datetime.now().timestamp()) * 1000

airbyte-cdk/python/airbyte_cdk/connector_builder/main.py

@@ -7,16 +7,9 @@
 from typing import Any, List, Mapping, Optional, Tuple
 
 from airbyte_cdk.connector import BaseConnector
-from airbyte_cdk.connector_builder.connector_builder_handler import (
-    TestReadLimits,
-    create_source,
-    get_limits,
-    list_streams,
-    read_stream,
-    resolve_manifest,
-)
+from airbyte_cdk.connector_builder.connector_builder_handler import TestReadLimits, create_source, get_limits, read_stream, resolve_manifest
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
-from airbyte_cdk.models import ConfiguredAirbyteCatalog
+from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
@@ -50,19 +43,17 @@ def get_config_and_catalog_from_args(args: List[str]) -> Tuple[str, Mapping[str,
 
 def handle_connector_builder_request(
     source: ManifestDeclarativeSource, command: str, config: Mapping[str, Any], catalog: Optional[ConfiguredAirbyteCatalog], limits: TestReadLimits
-):
+) -> AirbyteMessage:
     if command == "resolve_manifest":
         return resolve_manifest(source)
     elif command == "test_read":
         assert catalog is not None, "`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None."
         return read_stream(source, config, catalog, limits)
-    elif command == "list_streams":
-        return list_streams(source, config)
     else:
         raise ValueError(f"Unrecognized command {command}.")
 
 
-def handle_request(args: List[str]):
+def handle_request(args: List[str]) -> AirbyteMessage:
     command, config, catalog = get_config_and_catalog_from_args(args)
     limits = get_limits(config)
     source = create_source(config, limits)

airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py

@@ -9,13 +9,16 @@
 from airbyte_cdk.sources.embedded.catalog import create_configured_catalog, get_stream, get_stream_names
 from airbyte_cdk.sources.embedded.runner import SourceRunner
 from airbyte_cdk.sources.embedded.tools import get_defined_id
+from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
 from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type
 
 TOutput = TypeVar("TOutput")
 
 
 class BaseEmbeddedIntegration(ABC, Generic[TConfig, TOutput]):
     def __init__(self, runner: SourceRunner[TConfig], config: TConfig):
+        check_config_against_spec_or_exit(config, runner.spec())
+
         self.source = runner
         self.config = config
 

airbyte-cdk/python/airbyte_cdk/sources/embedded/runner.py

@@ -8,11 +8,15 @@
 from typing import Generic, Iterable, Optional
 
 from airbyte_cdk.connector import TConfig
-from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog
+from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification
 from airbyte_cdk.sources.source import Source
 
 
 class SourceRunner(ABC, Generic[TConfig]):
+    @abstractmethod
+    def spec(self) -> ConnectorSpecification:
+        pass
+
     @abstractmethod
     def discover(self, config: TConfig) -> AirbyteCatalog:
         pass
@@ -27,6 +31,9 @@ def __init__(self, source: Source, name: str):
         self._source = source
         self._logger = logging.getLogger(name)
 
+    def spec(self) -> ConnectorSpecification:
+        return self._source.spec(self._logger)
+
     def discover(self, config: TConfig) -> AirbyteCatalog:
         return self._source.discover(self._logger, config)
 

airbyte-cdk/python/setup.py

@@ -21,7 +21,7 @@
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.51.1",
+    version="0.51.2",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py

@@ -20,7 +20,6 @@
     TestReadLimits,
     create_source,
     get_limits,
-    list_streams,
     resolve_manifest,
 )
 from airbyte_cdk.connector_builder.main import handle_connector_builder_request, handle_request, read_stream
@@ -43,7 +42,6 @@
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.sources.declarative.retrievers import SimpleRetrieverTestReadDecorator
 from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
-from airbyte_cdk.sources.streams.core import Stream
 from unit_tests.connector_builder.utils import create_configured_catalog
 
 _stream_name = "stream_with_custom_requester"
@@ -537,7 +535,7 @@ def check_config_against_spec(self):
 )
 def test_invalid_protocol_command(command, valid_resolve_manifest_config_file):
     config = copy.deepcopy(RESOLVE_MANIFEST_CONFIG)
-    config["__command"] = "list_streams"
+    config["__command"] = "resolve_manifest"
     with pytest.raises(SystemExit):
         handle_request([command, "--config", str(valid_resolve_manifest_config_file), "--catalog", ""])
 
@@ -567,71 +565,6 @@ def manifest_declarative_source():
     return mock.Mock(spec=ManifestDeclarativeSource, autospec=True)
 
 
-def test_list_streams(manifest_declarative_source):
-    manifest_declarative_source.streams.return_value = [
-        create_mock_declarative_stream(create_mock_retriever("a name", "https://a-url-base.com", "a-path")),
-        create_mock_declarative_stream(create_mock_retriever("another name", "https://another-url-base.com", "another-path")),
-    ]
-
-    result = list_streams(manifest_declarative_source, {})
-
-    assert result.type == MessageType.RECORD
-    assert result.record.stream == "list_streams"
-    assert result.record.data == {
-        "streams": [
-            {"name": "a name", "url": "https://a-url-base.com/a-path"},
-            {"name": "another name", "url": "https://another-url-base.com/another-path"},
-        ]
-    }
-
-
-def test_given_stream_is_not_declarative_stream_when_list_streams_then_return_exception_message(manifest_declarative_source):
-    manifest_declarative_source.streams.return_value = [mock.Mock(spec=Stream)]
-
-    error_message = list_streams(manifest_declarative_source, {})
-
-    assert error_message.type == MessageType.TRACE
-    assert error_message.trace.error.message.startswith("Error listing streams")
-    assert "A declarative source should only contain streams of type DeclarativeStream" in error_message.trace.error.internal_message
-
-
-def test_given_declarative_stream_retriever_is_not_http_when_list_streams_then_return_exception_message(manifest_declarative_source):
-    declarative_stream = mock.Mock(spec=DeclarativeStream)
-    # `spec=DeclarativeStream` is needed for `isinstance` work but `spec` does not expose dataclasses fields, so we create one ourselves
-    declarative_stream.retriever = mock.Mock()
-    manifest_declarative_source.streams.return_value = [declarative_stream]
-
-    error_message = list_streams(manifest_declarative_source, {})
-
-    assert error_message.type == MessageType.TRACE
-    assert error_message.trace.error.message.startswith("Error listing streams")
-    assert "A declarative stream should only have a retriever of type SimpleRetriever" in error_message.trace.error.internal_message
-
-
-def test_given_unexpected_error_when_list_streams_then_return_exception_message(manifest_declarative_source):
-    manifest_declarative_source.streams.side_effect = Exception("unexpected error")
-
-    error_message = list_streams(manifest_declarative_source, {})
-
-    assert error_message.type == MessageType.TRACE
-    assert error_message.trace.error.message.startswith("Error listing streams")
-    assert "unexpected error" == error_message.trace.error.internal_message
-
-
-def test_list_streams_integration_test():
-    config = copy.deepcopy(RESOLVE_MANIFEST_CONFIG)
-    command = "list_streams"
-    config["__command"] = command
-    source = ManifestDeclarativeSource(MANIFEST)
-    limits = TestReadLimits()
-
-    list_streams = handle_connector_builder_request(source, command, config, None, limits)
-
-    assert list_streams.record.data == {
-        "streams": [{"name": "stream_with_custom_requester", "url": "https://api.sendgrid.com/v3/marketing/lists"}]
-    }
-
-
 def create_mock_retriever(name, url_base, path):
     http_stream = mock.Mock(spec=SimpleRetriever, autospec=True)
     http_stream.name = name

airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py

@@ -7,6 +7,7 @@
 from unittest.mock import MagicMock
 
 from airbyte_cdk.sources.embedded.base_integration import BaseEmbeddedIntegration
+from airbyte_cdk.utils import AirbyteTracedException
 from airbyte_protocol.models import (
     AirbyteCatalog,
     AirbyteLogMessage,
@@ -16,6 +17,7 @@
     AirbyteStream,
     ConfiguredAirbyteCatalog,
     ConfiguredAirbyteStream,
+    ConnectorSpecification,
     DestinationSyncMode,
     Level,
     SyncMode,
@@ -33,7 +35,14 @@ def setUp(self):
         self.source_class = MagicMock()
         self.source = MagicMock()
         self.source_class.return_value = self.source
-        self.config = MagicMock()
+        self.source.spec.return_value = ConnectorSpecification(connectionSpecification={
+            "properties": {
+                "test": {
+                    "type": "string",
+                }
+            }
+        })
+        self.config = {"test": "abc"}
         self.integration = TestIntegration(self.source, self.config)
         self.stream1 = AirbyteStream(
             name="test",
@@ -76,6 +85,12 @@ def test_integration(self):
             None,
         )
 
+    def test_failed_check(self):
+        self.config = {"test": 123}
+        with self.assertRaises(AirbyteTracedException) as error:
+            TestIntegration(self.source, self.config)
+        assert str(error.exception) == "123 is not of type 'string'"
+
     def test_state(self):
         state = AirbyteStateMessage(data={})
         self.source.read.return_value = [

airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py

@@ -98,7 +98,7 @@ def validate_all_tags_are_keyvalue_pairs(
 def is_major_version(version: str) -> bool:
     """Check whether the version is of format N.0.0"""
     semver_version = semver.Version.parse(version)
-    return semver_version.minor == 0 and semver_version.patch == 0
+    return semver_version.minor == 0 and semver_version.patch == 0 and semver_version.prerelease is None
 
 
 def validate_major_version_bump_has_breaking_change_entry(

airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_change_prerelease.yaml

@@ -0,0 +1,19 @@
+metadataSpecVersion: 1.0
+data:
+  name: AlloyDB for PostgreSQL
+  definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
+  connectorType: source
+  dockerRepository: airbyte/image-exists-1
+  githubIssueLabel: source-alloydb-strict-encrypt
+  dockerImageTag: 2.0.0-dev.cf3628ccf3
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  connectorSubtype: database
+  releaseStage: generally_available
+  license: MIT
+  releases:
+    breakingChanges:
+      2.0.0:
+        upgradeDeadline: 2023-08-22
+        message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)."
+  tags:
+    - language:java