From 48bf520d87341d03c8d3475d1003c186392eea8b Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc Date: Thu, 27 Jul 2023 10:05:35 -0400 Subject: [PATCH 01/10] Fix stream read given stream doesn't have any slice (#28746) * Fix stream read given stream doesn't have any slice * Not return slices if there are none * Fix test --- .../connector_builder/message_grouper.py | 19 +++-------- .../test_connector_builder_handler.py | 4 +-- .../connector_builder/test_message_grouper.py | 32 +++++++------------ 3 files changed, 18 insertions(+), 37 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py index dd6298c2630b..b787fe5d43c9 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py @@ -31,7 +31,6 @@ AirbyteMessage, AirbyteTraceMessage, ConfiguredAirbyteCatalog, - Level, OrchestratorType, TraceType, ) @@ -126,7 +125,6 @@ def _get_message_groups( current_slice_pages: List[StreamReadPages] = [] current_page_request: Optional[HttpRequest] = None current_page_response: Optional[HttpResponse] = None - had_error = False while records_count < limit and (message := next(messages, None)): json_object = self._parse_json(message.log) if message.type == MessageType.LOG else None @@ -134,7 +132,7 @@ def _get_message_groups( raise ValueError(f"Expected log message to be a dict, got {json_object} of type {type(json_object)}") json_message: Optional[Dict[str, JsonType]] = json_object if self._need_to_close_page(at_least_one_page_in_group, message, json_message): - self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, True) + self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records) current_page_request = None current_page_response = None @@ -172,12 +170,9 @@ def _get_message_groups( current_page_request = self._create_request_from_log_message(json_message) current_page_response = self._create_response_from_log_message(json_message) else: - if message.log.level == Level.ERROR: - had_error = True yield message.log elif message.type == MessageType.TRACE: if message.trace.type == TraceType.ERROR: - had_error = True yield message.trace elif message.type == MessageType.RECORD: current_page_records.append(message.record.data) @@ -187,8 +182,9 @@ def _get_message_groups( elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG: yield message.control else: - self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, validate_page_complete=not had_error) - yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor) + if current_page_request or current_page_response or current_page_records: + self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records) + yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor) @staticmethod def _need_to_close_page(at_least_one_page_in_group: bool, message: AirbyteMessage, json_message: Optional[Dict[str, Any]]) -> bool: @@ -224,15 +220,10 @@ def _is_auxiliary_http_request(message: Optional[Dict[str, Any]]) -> bool: return is_http and message.get("http", {}).get("is_auxiliary", False) @staticmethod - def _close_page(current_page_request: Optional[HttpRequest], current_page_response: Optional[HttpResponse], current_slice_pages: List[StreamReadPages], current_page_records: List[Mapping[str, Any]], validate_page_complete: bool) -> None: + def _close_page(current_page_request: Optional[HttpRequest], current_page_response: Optional[HttpResponse], current_slice_pages: List[StreamReadPages], current_page_records: List[Mapping[str, Any]]) -> None: """ Close a page when parsing message groups - @param validate_page_complete: in some cases, we expect the CDK to not return a response. As of today, this will only happen before - an uncaught exception and therefore, the assumption is that `validate_page_complete=True` only on the last page that is being closed """ - if validate_page_complete and (not current_page_request or not current_page_response): - raise ValueError("Every message grouping should have at least one request and response") - current_slice_pages.append( StreamReadPages(request=current_page_request, response=current_page_response, records=deepcopy(current_page_records)) # type: ignore ) diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py index 4245ccc9d129..48d5884d03d0 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py @@ -510,9 +510,7 @@ def check_config_against_spec(self): response = read_stream(source, TEST_READ_CONFIG, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), limits) expected_stream_read = StreamRead(logs=[LogMessage("error_message - a stack trace", "ERROR")], - slices=[StreamReadSlices( - pages=[StreamReadPages(records=[], request=None, response=None)], - slice_descriptor=None, state=None)], + slices=[], test_read_limit_reached=False, auxiliary_requests=[], inferred_schema=None, diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py index 015863c7e87d..67d437dfac91 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py @@ -367,25 +367,6 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: assert actual_page == expected_pages[i] -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_invalid_group_format(mock_entrypoint_read: Mock) -> None: - response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}'} - - mock_source = make_mock_source(mock_entrypoint_read, iter( - [ - response_log_message(response), - record_message("hashiras", {"name": "Shinobu Kocho"}), - record_message("hashiras", {"name": "Muichiro Tokito"}), - ] - ) - ) - - api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) - - with pytest.raises(ValueError): - api.get_message_groups(source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")) - - @pytest.mark.parametrize( "log_message, expected_response", [ @@ -588,7 +569,7 @@ def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_ha @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_given_auxiliary_requests_then_return_global_request(mock_entrypoint_read: Mock) -> None: +def test_given_auxiliary_requests_then_return_auxiliary_request(mock_entrypoint_read: Mock) -> None: mock_source = make_mock_source(mock_entrypoint_read, iter( any_request_and_response_with_a_record() + [ @@ -603,6 +584,17 @@ def test_given_auxiliary_requests_then_return_global_request(mock_entrypoint_rea assert len(stream_read.auxiliary_requests) == 1 +@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +def test_given_no_slices_then_return_empty_slices(mock_entrypoint_read: Mock) -> None: + mock_source = make_mock_source(mock_entrypoint_read, iter([auxiliary_request_log_message()])) + connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) + stream_read: StreamRead = connector_builder_handler.get_message_groups( + source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") + ) + + assert len(stream_read.slices) == 0 + + def make_mock_source(mock_entrypoint_read: Mock, return_value: Iterator[AirbyteMessage]) -> MagicMock: mock_source = MagicMock() mock_entrypoint_read.return_value = return_value From 71437385cbd3802b7ce946b3bde5517350dd946e Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:36:50 +0530 Subject: [PATCH 02/10] =?UTF-8?q?=E2=9C=A8=20Source=20surveycto=20:=20adde?= =?UTF-8?q?d=20check=20connection=20function=20(#28512)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * check connection function implement * failed msg * Changes Suggested * removed unused base_encode Signed-off-by: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> * removed unused code * checking all form ids * exception modify Co-authored-by: Marcos Marx * unit testing for check connection * docs and metadata * docker file update --------- Signed-off-by: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Co-authored-by: Marcos Marx --- .../connectors/source-surveycto/Dockerfile | 2 +- .../connectors/source-surveycto/metadata.yaml | 2 +- .../source_surveycto/source.py | 23 +++++++- .../source_surveycto/spec.yaml | 2 +- .../unit_tests/test_source.py | 59 +++++++++++-------- docs/integrations/sources/surveycto.md | 1 + 6 files changed, 58 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/source-surveycto/Dockerfile b/airbyte-integrations/connectors/source-surveycto/Dockerfile index 3ea0fac5aad8..a98c8726003a 100644 --- a/airbyte-integrations/connectors/source-surveycto/Dockerfile +++ b/airbyte-integrations/connectors/source-surveycto/Dockerfile @@ -37,6 +37,6 @@ COPY source_surveycto ./source_surveycto ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-surveycto diff --git a/airbyte-integrations/connectors/source-surveycto/metadata.yaml b/airbyte-integrations/connectors/source-surveycto/metadata.yaml index 5e0563ff88ef..521eae1b9801 100644 --- a/airbyte-integrations/connectors/source-surveycto/metadata.yaml +++ b/airbyte-integrations/connectors/source-surveycto/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: dd4632f4-15e0-4649-9b71-41719fb1fdee - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/source-surveycto githubIssueLabel: source-surveycto icon: surveycto.svg diff --git a/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py b/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py index e83dcbfb5c70..ce5a22fd4e20 100644 --- a/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py +++ b/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py @@ -12,7 +12,7 @@ from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer - +from airbyte_cdk.models import SyncMode from .helpers import Helpers @@ -109,9 +109,26 @@ def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: # Source class SourceSurveycto(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - return True, None + + def check_connection(self, logger, config) -> Tuple[bool, Any]: + + form_ids = config["form_id"] + + try: + for form_id in form_ids: + schema = Helpers.call_survey_cto(config, form_id) + filter_data = Helpers.get_filter_data(schema) + schema_res = Helpers.get_json_schema(filter_data) + stream = SurveyctoStream(config=config, form_id=form_id, schema=schema_res) + next(stream.read_records(sync_mode=SyncMode.full_refresh)) + + return True, None + + except Exception as error: + return False, f"Unable to connect - {(error)}" + + def generate_streams(self, config: str) -> List[Stream]: forms = config.get("form_id", []) streams = [] diff --git a/airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml b/airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml index a86d4f46d808..f9ab7aed5d6d 100644 --- a/airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml +++ b/airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml @@ -27,7 +27,7 @@ connectionSpecification: order: 2 form_id: type: array - title: Form's Id + title: Form Id's description: Unique identifier for one of your forms order: 3 start_date: diff --git a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py index 343043787611..505a5c38db8c 100644 --- a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py @@ -1,36 +1,45 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - import pytest -from airbyte_cdk.models import ConnectorSpecification -from source_surveycto.helpers import Helpers -from source_surveycto.source import SourceSurveycto - +from unittest.mock import MagicMock, patch +from source_surveycto.source import SourceSurveycto, SurveyctoStream @pytest.fixture(name='config') def config_fixture(): - return {'server_name': 'server_name', 'form_id': 'form_id', 'start_date': 'Jan 09, 2022 00:00:00 AM', 'password': 'password', 'username': 'username'} - - -def test_spec(): - source = SourceSurveycto() + return { + 'server_name': 'server_name', + 'form_id': ['form_id_1', 'form_id_2'], + 'start_date': 'Jan 09, 2022 00:00:00 AM', + 'password': 'password', + 'username': 'username' + } + +@pytest.fixture(name='source') +def source_fixture(): + return SourceSurveycto() + +@pytest.fixture(name='mock_survey_cto') +def mock_survey_cto_fixture(): + with patch('source_surveycto.source.Helpers.call_survey_cto', return_value="value") as mock_call_survey_cto, \ + patch('source_surveycto.source.Helpers.get_filter_data', return_value="value") as mock_filter_data, \ + patch('source_surveycto.source.Helpers.get_json_schema', return_value="value") as mock_json_schema: + yield mock_call_survey_cto, mock_filter_data, mock_json_schema + +def test_check_connection_valid(mock_survey_cto, source, config): logger_mock = MagicMock() - spec = source.spec(logger_mock) - assert source.check_connection(spec, ConnectorSpecification) + records = iter(["record1", "record2"]) + with patch.object(SurveyctoStream, 'read_records', return_value=records): + assert source.check_connection(logger_mock, config) == (True, None) -@patch("requests.get") -def test_check_connection(config): - source = SourceSurveycto() +def test_check_connection_failure(mock_survey_cto, source, config): logger_mock = MagicMock() - assert source.check_connection(logger_mock, config) == (True, None) + expected_outcome = 'Unable to connect - 400 Client Error: 400 for url: https://server_name.surveycto.com/api/v2/forms/data/wide/json/form_id_1?date=Jan+09%2C+2022+00%3A00%3A00+AM' + assert source.check_connection(logger_mock, config) == (False, expected_outcome) +def test_generate_streams(mock_survey_cto, source, config): + streams = source.generate_streams(config) + assert len(streams) == 2 -def test_streams(config): - source = SourceSurveycto() - Helpers.call_survey_cto = MagicMock() +@patch('source_surveycto.source.SourceSurveycto.generate_streams', return_value=['stream_1', 'stream2']) +def test_streams(mock_generate_streams, source, config): streams = source.streams(config) - assert len(streams) == 7 + assert len(streams) == 2 diff --git a/docs/integrations/sources/surveycto.md b/docs/integrations/sources/surveycto.md index b04fafb0597b..1e399fec4bf7 100644 --- a/docs/integrations/sources/surveycto.md +++ b/docs/integrations/sources/surveycto.md @@ -46,5 +46,6 @@ The SurveyCTO source connector supports the following streams: ## Changelog | Version | Date | Pull Request | Subject | +| 0.1.2 | 2023-07-27 | [28512](https://github.com/airbytehq/airbyte/pull/28512) | Added Check Connection | | 0.1.1 | 2023-04-25 | [24784](https://github.com/airbytehq/airbyte/pull/24784) | Fix incremental sync | | 0.1.0 | 2022-11-16 | [19371](https://github.com/airbytehq/airbyte/pull/19371) | SurveyCTO Source Connector | From 292530c536733159113d6f4fa1e4c6c31e4943c7 Mon Sep 17 00:00:00 2001 From: maxi297 Date: Thu, 27 Jul 2023 14:13:49 +0000 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=A4=96=20Bump=20patch=20version=20o?= =?UTF-8?q?f=20Airbyte=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/Dockerfile | 4 ++-- airbyte-cdk/python/setup.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 88820e85cf90..a048edb680e8 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.47.2 +current_version = 0.47.3 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index fb2308401b29..ffd6346cc2df 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.47.3 +Connector Builder: Ensure we return when there are no slices + ## 0.47.2 low-code: deduplicate query params if they are already encoded in the URL diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index 4202f7cabf0f..e47c6e45d191 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.47.2 +RUN pip install --prefix=/install airbyte-cdk==0.47.3 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.47.2 +LABEL io.airbyte.version=0.47.3 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 61db44dada46..23983cd995df 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -21,7 +21,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.47.2", + version="0.47.3", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 137abbf164e7e177ede2a09b53a96a293891658f Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Thu, 27 Jul 2023 10:51:56 -0400 Subject: [PATCH 04/10] add sentry reporting to connector publish pipelines (#28748) * add sentry * use env var * add click context * Automated Commit - Format and Process Resources Changes * add more ctx * add missing file * only publish for now * use GitHub secret for sentry dsn * Update airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py Co-authored-by: Augustin * fix syntax err * set release as package version * set global tags * add max duration to context * format --------- Co-authored-by: pedroslopez Co-authored-by: alafanechere Co-authored-by: Augustin --- .../actions/run-dagger-pipeline/action.yml | 4 ++ .github/workflows/publish_connectors.yml | 2 + .../pipelines/pipelines/__init__.py | 11 ++- .../connectors/pipelines/pipelines/bases.py | 3 + .../pipelines/commands/airbyte_ci.py | 1 - .../pipelines/pipelines/sentry_utils.py | 71 +++++++++++++++++++ .../connectors/pipelines/pipelines/utils.py | 3 +- airbyte-ci/connectors/pipelines/poetry.lock | 44 +++++++++++- .../connectors/pipelines/pyproject.toml | 1 + 9 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 9df42434b1ca..1c27d1bbd830 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -47,6 +47,9 @@ inputs: description: "Bucket name for metadata service" required: false default: "prod-airbyte-cloud-connector-metadata-service" + sentry_dsn: + description: "Sentry DSN" + required: false spec_cache_bucket_name: description: "Bucket name for GCS spec cache" required: false @@ -110,6 +113,7 @@ runs: METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }} PRODUCTION: ${{ inputs.production }} PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + SENTRY_DSN: ${{ inputs.sentry_dsn }} SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }} SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }} SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }} diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index fb92c4fef519..23a8ae233113 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -37,6 +37,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} subcommand: "connectors --concurrency=1 --execute-timeout=3600 --modified publish --main-release" @@ -53,6 +54,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" diff --git a/airbyte-ci/connectors/pipelines/pipelines/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/__init__.py index 47035d5627aa..371bafaa1370 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/__init__.py @@ -8,6 +8,10 @@ from rich.logging import RichHandler +from . import sentry_utils + +sentry_utils.initialize() + logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("httpx").setLevel(logging.WARNING) @@ -16,6 +20,11 @@ # RichHandler does not work great in the CI logging_handlers = [logging.StreamHandler()] -logging.basicConfig(level=logging.INFO, format="%(name)s: %(message)s", datefmt="[%X]", handlers=logging_handlers) +logging.basicConfig( + level=logging.INFO, + format="%(name)s: %(message)s", + datefmt="[%X]", + handlers=logging_handlers, +) main_logger = logging.getLogger(__name__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/bases.py b/airbyte-ci/connectors/pipelines/pipelines/bases.py index c69e6176f4a8..124c9d813cc4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/bases.py +++ b/airbyte-ci/connectors/pipelines/pipelines/bases.py @@ -18,6 +18,8 @@ import anyio import asyncer from anyio import Path +from pipelines import sentry_utils + from connector_ops.utils import console from dagger import Container, DaggerError, QueryError from jinja2 import Environment, PackageLoader, select_autoescape @@ -150,6 +152,7 @@ async def run_with_completion(self, completion_event: anyio.Event, *args, **kwar completion_event.set() return self._get_timed_out_step_result() + @sentry_utils.with_step_context async def run(self, *args, **kwargs) -> StepResult: """Public method to run the step. It output a step result. diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py index e5ca07d4e837..0157ed370493 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py @@ -133,7 +133,6 @@ def airbyte_ci( main_logger.info(f"Pipeline Start Timestamp: {pipeline_start_timestamp}") main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") - airbyte_ci.add_command(connectors) airbyte_ci.add_command(metadata) diff --git a/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py b/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py new file mode 100644 index 000000000000..9d768767bf08 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py @@ -0,0 +1,71 @@ +import os +import sentry_sdk +import importlib.metadata +from connector_ops.utils import Connector + + +def initialize(): + if "SENTRY_DSN" in os.environ: + sentry_sdk.init( + dsn=os.environ.get("SENTRY_DSN"), + release=f"pipelines@{importlib.metadata.version('pipelines')}", + ) + set_global_tags() + + +def set_global_tags(): + sentry_sdk.set_tag("ci_branch", os.environ.get("CI_GIT_BRANCH", "unknown")) + sentry_sdk.set_tag("ci_job", os.environ.get("CI_JOB_KEY", "unknown")) + sentry_sdk.set_tag("pull_request", os.environ.get("PULL_REQUEST_NUMBER", "unknown")) + + +def with_step_context(func): + def wrapper(self, *args, **kwargs): + with sentry_sdk.configure_scope() as scope: + step_name = self.__class__.__name__ + scope.set_tag("pipeline_step", step_name) + scope.set_context( + "Pipeline Step", + { + "name": step_name, + "step_title": self.title, + "max_retries": self.max_retries, + "max_duration": self.max_duration, + "retry_count": self.retry_count, + }, + ) + + if hasattr(self.context, "connector"): + connector: Connector = self.context.connector + scope.set_tag("connector", connector.technical_name) + scope.set_context( + "Connector", + { + "name": connector.name, + "technical_name": connector.technical_name, + "language": connector.language, + "version": connector.version, + "release_stage": connector.release_stage, + }, + ) + + return func(self, *args, **kwargs) + + return wrapper + + +def with_command_context(func): + def wrapper(self, ctx, *args, **kwargs): + with sentry_sdk.configure_scope() as scope: + scope.set_tag("pipeline_command", self.name) + scope.set_context( + "Pipeline Command", + { + "name": self.name, + "params": self.params, + }, + ) + scope.set_context("Click Context", ctx.obj) + return func(self, ctx, *args, **kwargs) + + return wrapper diff --git a/airbyte-ci/connectors/pipelines/pipelines/utils.py b/airbyte-ci/connectors/pipelines/pipelines/utils.py index 88e6e97bdbd9..ba397fb9e4f0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/utils.py @@ -22,7 +22,7 @@ import asyncer import click import git -from pipelines import consts, main_logger +from pipelines import consts, main_logger, sentry_utils from pipelines.consts import GCS_PUBLIC_DOMAIN from connector_ops.utils import get_all_released_connectors, get_changed_connectors from dagger import Client, Config, Connection, Container, DaggerError, ExecError, File, ImageLayerCompression, QueryError, Secret @@ -446,6 +446,7 @@ def create_and_open_file(file_path: Path) -> TextIOWrapper: class DaggerPipelineCommand(click.Command): + @sentry_utils.with_command_context def invoke(self, ctx: click.Context) -> Any: """Wrap parent invoke in a try catch suited to handle pipeline failures. Args: diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index edabca25578f..34e7dc67fbad 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1473,6 +1473,48 @@ files = [ {file = "semver-3.0.1.tar.gz", hash = "sha256:9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1"}, ] +[[package]] +name = "sentry-sdk" +version = "1.28.1" +description = "Python client for Sentry (https://sentry.io)" +optional = false +python-versions = "*" +files = [ + {file = "sentry-sdk-1.28.1.tar.gz", hash = "sha256:dcd88c68aa64dae715311b5ede6502fd684f70d00a7cd4858118f0ba3153a3ae"}, + {file = "sentry_sdk-1.28.1-py2.py3-none-any.whl", hash = "sha256:6bdb25bd9092478d3a817cb0d01fa99e296aea34d404eac3ca0037faa5c2aa0a"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""} + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +arq = ["arq (>=0.23)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +chalice = ["chalice (>=1.16.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +fastapi = ["fastapi (>=0.79.0)"] +flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"] +grpcio = ["grpcio (>=1.21.1)"] +httpx = ["httpx (>=0.16.0)"] +huey = ["huey (>=2)"] +loguru = ["loguru (>=0.5)"] +opentelemetry = ["opentelemetry-distro (>=0.35b0)"] +pure-eval = ["asttokens", "executing", "pure-eval"] +pymongo = ["pymongo (>=3.1)"] +pyspark = ["pyspark (>=2.4.4)"] +quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +starlette = ["starlette (>=0.19.1)"] +starlite = ["starlite (>=1.48)"] +tornado = ["tornado (>=5)"] + [[package]] name = "six" version = "1.16.0" @@ -1748,4 +1790,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "86aa1d5023a1c416a242fe29c915d106cad0faaba5d4ee157c8959963302b921" +content-hash = "3b0e434fb2cff3e3f3be2addac1fe06487730d61badb6ea3e18a562c8faa9649" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 34337c88c421..19f930143a35 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -22,6 +22,7 @@ jinja2 = "^3.0.2" requests = "^2.28.2" connector-ops = {path = "../connector_ops"} toml = "^0.10.2" +sentry-sdk = "^1.28.1" [tool.poetry.group.test.dependencies] pytest = "^6.2.5" From 4ad6deda2642961e748d1c0c4c9c6c556e6d7472 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Thu, 27 Jul 2023 17:54:31 +0300 Subject: [PATCH 05/10] :sparkles: :sparkles: Source Stripe: add missing fields (#28776) * Connector health: source hubspot, gitlab, snapchat-marketing: fix builds * #24672 source Stripe: add missing fields to stream schemas * #24672 upd changelog * fix schema error * fix schema error --- .../connectors/source-stripe/Dockerfile | 2 +- .../source-stripe/acceptance-test-config.yml | 1 - .../connectors/source-stripe/metadata.yaml | 2 +- .../source_stripe/schemas/accounts.json | 83 +++++ .../source_stripe/schemas/charges.json | 78 +++++ .../source_stripe/schemas/credit_notes.json | 3 + .../source_stripe/schemas/disputes.json | 6 + .../source_stripe/schemas/invoice_items.json | 18 + .../schemas/invoice_line_items.json | 74 ++++ .../source_stripe/schemas/invoices.json | 328 ++++++++++++++++++ .../source_stripe/schemas/payouts.json | 12 + .../source_stripe/schemas/plans.json | 3 + .../source_stripe/schemas/products.json | 6 + .../schemas/promotion_codes.json | 3 + .../source_stripe/schemas/setup_attempts.json | 6 + .../schemas/shared/balance_transactions.json | 3 + .../schemas/shared/cardholder.json | 6 + .../schemas/shared/customer.json | 3 + .../schemas/shared/payment_intent.json | 54 +++ .../source_stripe/schemas/shared/price.json | 104 ++++++ .../schemas/shared/setup_intent.json | 14 + .../schemas/shared/tax_rates.json | 53 +++ .../schemas/subscription_items.json | 14 + .../schemas/subscription_schedule.json | 3 + .../source_stripe/schemas/subscriptions.json | 195 +++++++++++ docs/integrations/sources/stripe.md | 1 + 26 files changed, 1072 insertions(+), 3 deletions(-) create mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/price.json create mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index c8044b8218ba..7322cdbfeff7 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=3.15.0 +LABEL io.airbyte.version=3.16.0 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index 128dce92d4b6..66ffbca8941c 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -55,7 +55,6 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes - fail_on_extra_columns: false ignored_fields: invoices: - name: invoice_pdf diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index 94e5044e5b73..6a8f3ac84d04 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 3.15.0 + dockerImageTag: 3.16.0 dockerRepository: airbyte/source-stripe githubIssueLabel: source-stripe icon: stripe.svg diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json index 2c9256ccea4e..b1a68dbcb95d 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json @@ -739,6 +739,89 @@ "type": { "enum": ["custom", "express", "standard"], "type": ["null", "string"] + }, + "future_requirements": { + "type": ["null", "object"], + "properties": { + "alternatives": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "alternative_fields_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "original_fields_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + }, + "current_deadline": { + "type": ["null", "integer"] + }, + "currently_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "disabled_reason": { + "type": ["null", "string"] + }, + "errors": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "code": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "requirement": { + "type": ["null", "string"] + } + } + } + }, + "eventually_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "past_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "pending_verification": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, + "controller": { + "type": ["null", "object"], + "properties": { + "is_controller": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json index 8d40077b0a4f..b347163b58bc 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json @@ -1018,6 +1018,84 @@ }, "description": { "type": ["null", "string"] + }, + "statement_descriptor_suffix": { + "type": ["null", "string"] + }, + "calculated_statement_descriptor": { + "type": ["null", "string"] + }, + "receipt_url": { + "type": ["null", "string"] + }, + "transfer_data": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "destination": { + "type": ["null", "string"] + } + } + }, + "billing_details": { + "type": ["null", "object"], + "properties": { + "address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "failure_balance_transaction": { + "type": ["null", "string"] + }, + "amount_captured": { + "type": ["null", "integer"] + }, + "application_fee_amount": { + "type": ["null", "integer"] + }, + "amount_updates": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "payment_method": { + "type": ["null", "string"] + }, + "disputed": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json index f5025045b582..7ea99e6353be 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json @@ -632,6 +632,9 @@ "null", "integer" ] + }, + "effective_at": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json index 4c0f997ed4a3..b627b72dd7e0 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json @@ -148,6 +148,12 @@ }, "status": { "type": ["null", "string"] + }, + "payment_intent": { + "type": ["null", "string"] + }, + "balance_transaction": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json index e18a689d4228..0c2f07854952 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json @@ -150,6 +150,24 @@ }, "subscription_item": { "type": ["null", "string"] + }, + "price": { + "$ref": "price.json" + }, + "test_clock": { + "type": ["null", "string"] + }, + "discounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "tax_rates": { + "$ref": "tax_rates.json" + }, + "unit_amount_decimal": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json index c09aea04ba6f..3ade45cbe340 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json @@ -149,6 +149,80 @@ }, "currency": { "type": ["null", "string"] + }, + "amount_excluding_tax": { + "type": ["null", "integer"] + }, + "unit_amount_excluding_tax": { + "type": ["null", "string"] + }, + "proration_details": { + "type": ["null", "object"], + "properties": { + "credited_items": { + "type": ["null", "object"], + "properties": { + "invoice": { + "type": ["null", "string"] + }, + "invoice_line_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + } + }, + "price": { + "$ref": "price.json" + }, + "discount_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "discount": { + "type": ["null", "string"] + } + } + } + }, + "discounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "tax_rates": { + "$ref": "tax_rates.json" + }, + "tax_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "tax_rate": { + "type": ["null", "string"] + }, + "taxability_reason": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "integer"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json index 8c30c5f37a15..f650fa4ec0f8 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json @@ -255,6 +255,334 @@ "type": ["null", "integer"] } } + }, + "post_payment_credit_notes_amount": { + "type": ["null", "integer"] + }, + "paid_out_of_band": { + "type": ["null", "boolean"] + }, + "total_discount_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "discount": { + "type": ["null", "string"] + } + } + } + }, + "customer_name": { + "type": ["null", "string"] + }, + "shipping_cost": { + "type": ["null", "object"], + "properties": { + "amount_subtotal": { + "type": ["null", "integer"] + }, + "amount_tax": { + "type": ["null", "integer"] + }, + "amount_total": { + "type": ["null", "integer"] + }, + "shipping_rate": { + "type": ["null", "string"] + }, + "taxes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "transfer_data": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "destination": { + "type": ["null", "string"] + } + } + }, + "application_fee_amount": { + "type": ["null", "integer"] + }, + "customer_shipping": { + "type": ["null", "object"], + "address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + }, + "application": { + "type": ["null", "string"] + }, + "amount_shipping": { + "type": ["null", "integer"] + }, + "from_invoice": { + "type": ["null", "object"], + "properties": { + "actions": { + "type": ["null", "string"] + }, + "invoice": { + "type": ["null", "string"] + } + } + }, + "customer_tax_exempt": { + "type": ["null", "string"] + }, + "total_tax_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "tax_rate": { + "type": ["null", "string"] + }, + "taxability_reason": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "integer"] + } + } + } + }, + "footer": { + "type": ["null", "string"] + }, + "test_clock": { + "type": ["null", "string"] + }, + "automatic_tax": { + "type": ["null", "object"], + "properties": { + "enabled": { + "type": ["null", "boolean"] + }, + "status": { + "type": ["null", "string"] + } + } + }, + "payment_settings": { + "type": ["null", "object"], + "properties": { + "default_mandate": { + "type": ["null", "string"] + }, + "payment_method_options": { + "type": ["null", "object"] + }, + "payment_method_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, + "default_source": { + "type": ["null", "string"] + }, + "payment_intent": { + "type": ["null", "string"] + }, + "default_payment_method": { + "type": ["null", "string"] + }, + "shipping_details": { + "type": ["null", "object"], + "properties": { + "address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "collection_method": { + "type": ["null", "string"] + }, + "effective_at": { + "type": ["null", "integer"] + }, + "default_tax_rates": { + "type": ["null", "array"], + "items": { + "$ref": "tax_rates.json" + } + }, + "total_excluding_tax": { + "type": ["null", "integer"] + }, + "subtotal_excluding_tax": { + "type": ["null", "integer"] + }, + "last_finalization_error": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "doc_url": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "param": { + "type": ["null", "string"] + }, + "payment_method_type": { + "type": ["null", "string"] + } + } + }, + "latest_revision": { + "type": ["null", "string"] + }, + "rendering_options": { + "type": ["null", "object"], + "properties": { + "amount_tax_display": { + "type": ["null", "string"] + } + } + }, + "quote": { + "type": ["null", "string"] + }, + "pre_payment_credit_notes_amount": { + "type": ["null", "integer"] + }, + "customer_phone": { + "type": ["null", "string"] + }, + "on_behalf_of": { + "type": ["null", "string"] + }, + "account_tax_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "customer_email": { + "type": ["null", "string"] + }, + "customer_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "account_name": { + "type": ["null", "string"] + }, + "account_country": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json index 49806969dc1b..a057a81ec4fb 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json @@ -123,6 +123,18 @@ }, "source_transaction": { "type": ["null", "string"] + }, + "original_payout": { + "type": ["null", "string"] + }, + "reconciliation_status": { + "type": ["null", "string"] + }, + "source_balance": { + "type": ["null", "string"] + }, + "reversed_by": { + "type": ["null", "string"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json index 3af88a962ca0..2b46c1435c85 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json @@ -87,6 +87,9 @@ "metadata": { "type": ["null", "object"], "properties": {} + }, + "amount_decimal": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json index 42558f33e9b4..b7416f78a356 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json @@ -81,6 +81,12 @@ }, "url": { "type": ["null", "string"] + }, + "default_price": { + "type": ["null", "string"] + }, + "tax_code": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json index 030254e5a0ab..c602c4398f13 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json @@ -54,6 +54,9 @@ "minimum_amount": { "type": ["null", "integer"] }, "minimum_amount_currency": { "type": ["null", "string"] } } + }, + "times_redeemed": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json index c60fefbc85da..cb7e1ff4829a 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json @@ -211,6 +211,12 @@ }, "usage": { "type": ["null", "string"] + }, + "flow_directions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/balance_transactions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/balance_transactions.json index 432a3a37ce62..f544551605e9 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/balance_transactions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/balance_transactions.json @@ -66,6 +66,9 @@ }, "amount": { "type": ["null", "integer"] + }, + "reporting_category": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json index 22f4304c7687..66934d4cfda7 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json @@ -105,6 +105,12 @@ }, "type": { "type": ["null", "string"] + }, + "preferred_locales": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json index 8c41df3c3e1d..9d7850a5d562 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json @@ -835,6 +835,9 @@ }, "tax_info": { "type": ["null", "string"] + }, + "test_clock": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json index dda01df7d270..cfc7ea3a5c0f 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json @@ -833,6 +833,60 @@ }, "transfer_group": { "type": ["null", "string"] + }, + "latest_charge": { + "type": ["null", "string"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "tip": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + } + } + } + } + }, + "processing": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "customer_notification": { + "type": ["null", "object"], + "properties": { + "approval_requested": { + "type": ["null", "boolean"] + }, + "completes_at": { + "type": ["null", "integer"] + } + } + } + } + } + } + }, + "automatic_payment_methods": { + "type": ["null", "object"], + "properties": { + "allow_redirects": { + "type": ["null", "string"] + }, + "enabled": { + "type": ["null", "boolean"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/price.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/price.json new file mode 100644 index 000000000000..f8e72578526e --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/price.json @@ -0,0 +1,104 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "billing_scheme": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "currency_options": { + "type": ["null", "string"] + }, + "custom_unit_amount": { + "type": ["null", "object"], + "properties": { + "maximum": { + "type": ["null", "integer"] + }, + "minimum": { + "type": ["null", "integer"] + }, + "preset": { + "type": ["null", "integer"] + } + } + }, + "livemode": { + "type": ["null", "boolean"] + }, + "lookup_key": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"] + }, + "nickname": { + "type": ["null", "string"] + }, + "product": { + "type": ["null", "string"] + }, + "recurring": { + "type": ["null", "object"], + "properties": { + "aggregate_usage": { + "type": ["null", "string"] + }, + "interval": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + }, + "usage_type": { + "type": ["null", "string"] + } + } + }, + "tax_behavior": { + "type": ["null", "string"] + }, + "tiers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "tiers_mode": { + "type": ["null", "string"] + }, + "transform_quantity": { + "type": ["null", "object"], + "properties": { + "divide_by": { + "type": ["null", "integer"] + }, + "round": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "unit_amount_decimal": { + "type": ["null", "string"] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json index 51c9e2535b52..ffb7220e60e3 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json @@ -72,6 +72,20 @@ "usage": { "type": ["string"], "enum": ["on_session", "off_session"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "automatic_payment_methods": { + "type": ["null", "object"], + "properties": { + "allow_redirects": { + "type": ["null", "string"] + }, + "enabled": { + "type": ["null", "boolean"] + } + } } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json new file mode 100644 index 000000000000..6c4c193e2ace --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json @@ -0,0 +1,53 @@ +{ + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "country": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "display_name": { + "type": ["null", "string"] + }, + "effective_percentage": { + "type": ["null", "number"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "jurisdiction": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"] + }, + "percentage": { + "type": ["null", "number"] + }, + "state": { + "type": ["null", "string"] + }, + "tax_type": { + "type": ["null", "string"] + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json index 04ca778c8da8..c6d36caab561 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json @@ -153,6 +153,20 @@ }, "trial_end": { "type": ["null", "number"] + }, + "billing_thresholds": { + "type": ["null", "object"], + "properties": { + "usage_gte": { + "type": ["null", "integer"] + } + } + }, + "tax_rates": { + "$ref": "tax_rates.json" + }, + "price": { + "$ref": "price.json" } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json index 6afdd775bd21..e8729bfefab4 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json @@ -258,6 +258,9 @@ }, "test_clock": { "type": ["null", "string"] + }, + "renewal_interval": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json index 5d7e5a3713ff..405647bd85cc 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json @@ -247,6 +247,201 @@ }, "object": { "type": ["null", "string"] + }, + "pending_setup_intent": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "transfer_data": { + "type": ["null", "object"], + "properties": { + "amount_percent": { + "type": ["null", "number"] + }, + "destination": { + "type": ["null", "string"] + } + } + }, + "application": { + "type": ["null", "string"] + }, + "test_clock": { + "type": ["null", "string"] + }, + "automatic_tax": { + "type": ["null", "object"], + "properties": { + "enabled": { + "type": ["null", "boolean"] + } + } + }, + "payment_settings": { + "type": ["null", "object"], + "properties": { + "payment_method_options": { + "type": ["null", "object"] + }, + "payment_method_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "save_default_payment_method": { + "type": ["null", "string"] + } + } + }, + "next_pending_invoice_item_invoice": { + "type": ["null", "integer"] + }, + "default_source": { + "type": ["null", "string"] + }, + "default_payment_method": { + "type": ["null", "string"] + }, + "collection_method": { + "type": ["null", "string"] + }, + "pending_invoice_item_interval": { + "type": ["null", "object"], + "properties": { + "interval": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + } + } + }, + "default_tax_rates": { + "type": ["null", "array"], + "items": { + "$ref": "tax_rates.json" + } + }, + "pause_collection": { + "type": ["null", "object"], + "properties": { + "behavior": { + "type": ["null", "string"] + }, + "resumes_at": { + "type": ["null", "integer"] + } + } + }, + "cancellation_details": { + "type": ["null", "object"], + "properties": { + "comment": { + "type": ["null", "string"] + }, + "feedback": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + } + } + }, + "latest_invoice": { + "type": ["null", "string"] + }, + "pending_update": { + "type": ["null", "object"], + "properties": { + "billing_cycle_anchor": { + "type": ["null", "integer"] + }, + "expires_at": { + "type": ["null", "integer"] + }, + "subscription_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "billing_thresholds": { + "type": ["null", "object"], + "properties": { + "usage_gte": { + "type": ["null", "integer"] + } + } + }, + "created": { + "type": ["null", "integer"] + }, + "metadata": { + "type": ["null", "object"] + }, + "price": { + "$ref": "price.json" + }, + "quantity": { + "type": ["null", "integer"] + }, + "subscription": { + "type": ["null", "string"] + }, + "tax_rates": { + "$ref": "tax_rates.json" + } + } + } + }, + "trial_end": { + "type": ["null", "integer"] + }, + "trial_from_plan": { + "type": ["null", "boolean"] + } + } + }, + "description": { + "type": ["null", "string"] + }, + "schedule": { + "type": ["null", "string"] + }, + "trial_settings": { + "type": ["null", "object"], + "properties": { + "end_behavior": { + "type": ["null", "object"], + "properties": { + "missing_payment_method": { + "type": ["null", "string"] + } + } + } + } + }, + "on_behalf_of": { + "type": ["null", "string"] + }, + "billing_thresholds": { + "type": ["null", "object"], + "properties": { + "amount_gte": { + "type": ["null", "integer"] + }, + "reset_billing_cycle_anchor": { + "type": ["null", "boolean"] + } + } } } } diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 0625e3ef816d..dc44d58efe02 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -103,6 +103,7 @@ The Stripe connector should not run into Stripe API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.16.0 | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas | | 3.15.0 | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams | | 3.14.0 | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream | | 3.13.0 | 2023-07-18 | [28466](https://github.com/airbytehq/airbyte/pull/28466) | Pin source API version | From 36affa62534b211f1c12802684a5236d0cf3ad00 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Thu, 27 Jul 2023 11:00:34 -0400 Subject: [PATCH 06/10] Docs: Source Salesforce docs update (#28751) * update setup instructions * update setup steps in inapp * add please * remove whitespace * small fix * Update salesforce.md * typecase --- docs/integrations/sources/salesforce.inapp.md | 67 ++++++++++++++----- docs/integrations/sources/salesforce.md | 64 +++++++++--------- 2 files changed, 82 insertions(+), 49 deletions(-) diff --git a/docs/integrations/sources/salesforce.inapp.md b/docs/integrations/sources/salesforce.inapp.md index 1416134893dc..d60a552e465a 100644 --- a/docs/integrations/sources/salesforce.inapp.md +++ b/docs/integrations/sources/salesforce.inapp.md @@ -1,24 +1,20 @@ ## Prerequisites -* [Salesforce Account](https://login.salesforce.com/) with Enterprise access or API quota purchased -* (Optional) Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US) +- [Salesforce Account](https://login.salesforce.com/) with Enterprise access or API quota purchased +- (Optional, Recommended) Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US) + +- (For Airbyte Open Source) Salesforce [OAuth](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_tokens_scopes.htm&type=5) credentials + ## Setup guide -1. Enter a name for the Salesforce connector. -2. Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Make sure you are logged into the right account. We recommend creating a dedicated read-only Salesforce user (see below for instructions). -3. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. -4. (Optional) Enter the **Start Date** in YYYY-MM-DDT00:00:00Z format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate the data for last two years. -5. (Optional) In the Salesforce Object filtering criteria section, click **Add**. From the Search criteria dropdown, select the criteria relevant to you. For Search value, add the search terms relevant to you. If this field is blank, Airbyte will scan for all objects. You can also filter which objects you want to sync later on when setting up your connection. -9. Click **Set up source**. - -### (Optional) Create a read-only Salesforce user +### Step 1: (Optional, Recommended) Create a read-only Salesforce user While you can set up the Salesforce connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the data Airbyte can read. To create a dedicated read only Salesforce user: -1. [Log into Salesforce](https://login.salesforce.com/) with an admin account. +1. [Log in to Salesforce](https://login.salesforce.com/) with an admin account. 2. On the top right of the screen, click the gear icon and then click **Setup**. 3. In the left navigation bar, under Administration, click **Users** > **Profiles**. The Profiles page is displayed. Click **New profile**. 4. For Existing Profile, select **Read only**. For Profile Name, enter **Airbyte Read Only User**. @@ -27,13 +23,49 @@ To create a dedicated read only Salesforce user: 7. Scroll to the top and click **Save**. 8. On the left side, under Administration, click **Users** > **Users**. The All Users page is displayed. Click **New User**. 9. Fill out the required fields: - 1. For License, select **Salesforce**. - 2. For Profile, select **Airbyte Read Only User**. - 3. For Email, make sure to use an email address that you can access. + 1. For License, select **Salesforce**. + 2. For Profile, select **Airbyte Read Only User**. + 3. For Email, make sure to use an email address that you can access. 10. Click **Save**. 11. Copy the Username and keep it accessible. 12. Log into the email you used above and verify your new Salesforce account user. You'll need to set a password as part of this process. Keep this password accessible. + + +### For Airbyte Open Source only: Obtain Salesforce OAuth credentials + +If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate: + +- Client ID +- Client Secret +- Refresh Token + +To obtain these credentials, follow [this walkthrough](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) with the following modifications: + + 1. If your Salesforce URL is not in the `X.salesforce.com` format, use your Salesforce domain name. For example, if your Salesforce URL is `awesomecompany.force.com` then use that instead of `awesomecompany.salesforce.com`. + 2. When running a curl command, run it with the `-L` option to follow any redirects. + 3. If you [created a read-only user](https://docs.google.com/document/d/1wZR8pz4MRdc2zUculc9IqoF8JxN87U40IqVnTtcqdrI/edit#heading=h.w5v6h7b2a9y4), use the user credentials when logging in to generate OAuth tokens. + + + +### Step 2: Set up the Salesforce connector in Airbyte + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. Find and select **Salesforce** from the list of available sources. +4. Enter a **Source name** of your choosing to help you identify this source. +5. To authenticate: + +**For Airbyte Cloud**: Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Please make sure you are logged into the right account. + + +**For Airbyte Open Source**: Enter your Client ID, Client Secret, and Refresh Token. + +6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. +7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/). +8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data. +9. Click **Set up source** and wait for the tests to complete. + ### Supported Objects The Salesforce connector supports reading both Standard Objects and Custom Objects from Salesforce. Each object is read as a separate stream. See a list of all Salesforce Standard Objects [here](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_list.htm). @@ -42,14 +74,15 @@ Airbyte fetches and handles all the possible and available streams dynamically b * If the authenticated Salesforce user has the Role and Permissions to read and fetch objects -* If the object has the queryable property set to true. Airbyte can fetch only queryable streams via the API. If you don’t see your object available via Airbyte, check if it is API-accessible to the Salesforce user you authenticated with in Step 2. +* If the object has the queryable property set to true. Airbyte can fetch only queryable streams via the API. If you don’t see your object available via Airbyte, check if it is API-accessible to the Salesforce user you authenticated with. ### Incremental Deletes -The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with the field `isDeleted=true` value. +The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with the `isDeleted=true` value in the respective field. ### Syncing Formula Fields The Salesforce connector syncs formula field outputs from Salesforce. If the formula of a field changes in Salesforce and no other field on the record is updated, you will need to reset the stream to pull in all the updated values of the field. -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Salesforce](https://docs.airbyte.com/integrations/sources/google-analytics-v4). +For detailed information on supported sync modes, supported streams and performance considerations, refer to the +[full documentation for Salesforce](https://docs.airbyte.com/integrations/sources/google-analytics-v4). diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index c960f149f59a..7a98220ba50d 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -1,13 +1,11 @@ # Salesforce -Setting up the Salesforce source connector involves creating a read-only Salesforce user and configuring the Salesforce connector through the Airbyte UI. - -This page guides you through the process of setting up the Salesforce source connector. +This page contains the setup guide and reference information for the Salesforce source connector. ## Prerequisites - [Salesforce Account](https://login.salesforce.com/) with Enterprise access or API quota purchased -- Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US) (optional) +- (Optional, Recommended) Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US) - (For Airbyte Open Source) Salesforce [OAuth](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_tokens_scopes.htm&type=5) credentials @@ -20,7 +18,7 @@ While you can set up the Salesforce connector using any Salesforce user with rea To create a dedicated read only Salesforce user: -1. [Log into Salesforce](https://login.salesforce.com/) with an admin account. +1. [Log in to Salesforce](https://login.salesforce.com/) with an admin account. 2. On the top right of the screen, click the gear icon and then click **Setup**. 3. In the left navigation bar, under Administration, click **Users** > **Profiles**. The Profiles page is displayed. Click **New profile**. 4. For Existing Profile, select **Read only**. For Profile Name, enter **Airbyte Read Only User**. @@ -36,40 +34,42 @@ To create a dedicated read only Salesforce user: 11. Copy the Username and keep it accessible. 12. Log into the email you used above and verify your new Salesforce account user. You'll need to set a password as part of this process. Keep this password accessible. -### Step 2: Set up Salesforce as a Source in Airbyte - - - -**For Airbyte Cloud:** - -To set up Salesforce as a source in Airbyte Cloud: - -1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. -3. On the Set up the source page, select **Salesforce** from the **Source type** dropdown. -4. For Name, enter a name for the Salesforce connector. -5. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. -6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate the data for last two years. -7. (Optional) In the Salesforce Object filtering criteria section, click **Add**. From the Search criteria dropdown, select the criteria relevant to you. For Search value, add the search terms relevant to you. If this field is blank, Airbyte will replicate all data. -8. Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Make sure you are logged into the right account. -9. Click **Set up source**. - - -**For Airbyte Open Source:** +### For Airbyte Open Source only: Obtain Salesforce OAuth credentials -To set up Salesforce as a source in Airbyte Open Source: +If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate: -1. Follow this [walkthrough](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) with the following modifications: +- Client ID +- Client Secret +- Refresh Token - 1. If your Salesforce URL’s is not in the `X.salesforce.com` format, use your Salesforce domain name. For example, if your Salesforce URL is `awesomecompany.force.com` then use that instead of `awesomecompany.salesforce.com`. +To obtain these credentials, follow [this walkthrough](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) with the following modifications: + + 1. If your Salesforce URL is not in the `X.salesforce.com` format, use your Salesforce domain name. For example, if your Salesforce URL is `awesomecompany.force.com` then use that instead of `awesomecompany.salesforce.com`. 2. When running a curl command, run it with the `-L` option to follow any redirects. 3. If you [created a read-only user](https://docs.google.com/document/d/1wZR8pz4MRdc2zUculc9IqoF8JxN87U40IqVnTtcqdrI/edit#heading=h.w5v6h7b2a9y4), use the user credentials when logging in to generate OAuth tokens. -2. Navigate to the Airbute Open Source dashboard and follow the same steps as [setting up Salesforce as a source in Airbyte Cloud](#for-airbyte-cloud). +### Step 2: Set up the Salesforce connector in Airbyte + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. Find and select **Salesforce** from the list of available sources. +4. Enter a **Source name** of your choosing to help you identify this source. +5. To authenticate: + +**For Airbyte Cloud**: Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Please make sure you are logged into the right account. + + +**For Airbyte Open Source**: Enter your Client ID, Client Secret, and Refresh Token. + +6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. +7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/). +8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data. +9. Click **Set up source** and wait for the tests to complete. + ## Supported sync modes The Salesforce source connector supports the following sync modes: @@ -79,9 +79,9 @@ The Salesforce source connector supports the following sync modes: - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) - (Recommended)[ Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) -### Incremental Deletes Sync +### Incremental Deletes sync -The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with the field `isDeleted=true` value. +The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with the `isDeleted=true` value in the respective field. ## Performance considerations @@ -95,7 +95,7 @@ Airbyte fetches and handles all the possible and available streams dynamically b - If the authenticated Salesforce user has the Role and Permissions to read and fetch objects -- If the stream has the queryable property set to true. Airbyte can fetch only queryable streams via the API. If you don’t see your object available via Airbyte, check if it is API-accessible to the Salesforce user you authenticated with in Step 2. +- If the stream has the queryable property set to true. Airbyte can fetch only queryable streams via the API. If you don’t see your object available via Airbyte, check if it is API-accessible to the Salesforce user you authenticated with. **Note:** [BULK API](https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/asynch_api_intro.htm) cannot be used to receive data from the following streams due to Salesforce API limitations. The Salesforce connector syncs them using the REST API which will occasionally cost more of your API quota: From 7fcf8f05ab56ad5bfc3d7b44db5d3e5036170141 Mon Sep 17 00:00:00 2001 From: Augustin Date: Thu, 27 Jul 2023 17:31:13 +0200 Subject: [PATCH 07/10] connectors-ci: fix full dagger python connector dagger build (#28783) --- .../pipelines/pipelines/actions/environments.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py index 1aa33e889838..a60ea74167f4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py @@ -8,25 +8,25 @@ import importlib.util import json -import toml import re import uuid from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Callable, List, Optional +import toml import yaml +from dagger import CacheVolume, Client, Container, DaggerError, Directory, File, Platform, Secret +from dagger.engine._version import CLI_VERSION as dagger_engine_version from pipelines import consts from pipelines.consts import ( - CONNECTOR_OPS_SOURCE_PATHSOURCE_PATH, CI_CREDENTIALS_SOURCE_PATH, + CONNECTOR_OPS_SOURCE_PATHSOURCE_PATH, CONNECTOR_TESTING_REQUIREMENTS, LICENSE_SHORT_FILE_PATH, PYPROJECT_TOML_FILE_PATH, ) from pipelines.utils import get_file_contents -from dagger import CacheVolume, Client, Container, DaggerError, Directory, File, Platform, Secret -from dagger.engine._version import CLI_VERSION as dagger_engine_version if TYPE_CHECKING: from pipelines.contexts import ConnectorContext, PipelineContext @@ -980,7 +980,7 @@ async def with_airbyte_python_connector_full_dagger(context: ConnectorContext, b .with_mounted_cache("/root/.cache/pip", pip_cache) .with_exec(["pip", "install", "--upgrade", "pip"]) .with_exec(["apt-get", "install", "-y", "tzdata"]) - .with_file("setup.py", await context.get_connector_dir(include="setup.py").file("setup.py")) + .with_file("setup.py", (await context.get_connector_dir(include="setup.py")).file("setup.py")) ) for dependency_path in setup_dependencies_to_mount: @@ -995,8 +995,8 @@ async def with_airbyte_python_connector_full_dagger(context: ConnectorContext, b .with_file("/usr/localtime", builder.file("/usr/share/zoneinfo/Etc/UTC")) .with_new_file("/etc/timezone", "Etc/UTC") .with_exec(["apt-get", "install", "-y", "bash"]) - .with_file("main.py", await context.get_connector_dir(include="main.py").file("main.py")) - .with_directory(snake_case_name, await context.get_connector_dir(include=snake_case_name).directory(snake_case_name)) + .with_file("main.py", (await context.get_connector_dir(include="main.py")).file("main.py")) + .with_directory(snake_case_name, (await context.get_connector_dir(include=snake_case_name)).directory(snake_case_name)) .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint)) .with_entrypoint(entrypoint) .with_label("io.airbyte.version", context.metadata["dockerImageTag"]) From 91eff3af38c05fc80840cdef3db256831a0a674c Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Thu, 27 Jul 2023 12:08:28 -0400 Subject: [PATCH 08/10] fix reported sentry tags (#28784) * fix tags * dont set globals --- .../pipelines/pipelines/sentry_utils.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py b/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py index 9d768767bf08..394663ec8ec0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py @@ -1,6 +1,10 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import importlib.metadata import os + import sentry_sdk -import importlib.metadata from connector_ops.utils import Connector @@ -10,13 +14,6 @@ def initialize(): dsn=os.environ.get("SENTRY_DSN"), release=f"pipelines@{importlib.metadata.version('pipelines')}", ) - set_global_tags() - - -def set_global_tags(): - sentry_sdk.set_tag("ci_branch", os.environ.get("CI_GIT_BRANCH", "unknown")) - sentry_sdk.set_tag("ci_job", os.environ.get("CI_JOB_KEY", "unknown")) - sentry_sdk.set_tag("pull_request", os.environ.get("PULL_REQUEST_NUMBER", "unknown")) def with_step_context(func): @@ -65,7 +62,11 @@ def wrapper(self, ctx, *args, **kwargs): "params": self.params, }, ) + scope.set_context("Click Context", ctx.obj) + scope.set_tag("git_branch", ctx.obj.get("git_branch", "unknown")) + scope.set_tag("git_revision", ctx.obj.get("git_revision", "unknown")) + return func(self, ctx, *args, **kwargs) return wrapper From 1d1ed472d62d01686d7d5872ebdab83dc1102241 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 27 Jul 2023 10:29:15 -0600 Subject: [PATCH 09/10] Run ensure path (#28792) --- .github/actions/run-dagger-pipeline/action.yml | 1 + .github/workflows/connector-performance-command.yml | 1 + .github/workflows/connector_metadata_checks.yml | 1 + .github/workflows/connector_teams_review_requirements.yml | 1 + .github/workflows/legacy-publish-command.yml | 1 + .github/workflows/legacy-test-command.yml | 1 + .github/workflows/test-performance-command.yml | 1 + .../bases/connector-acceptance-test/acceptance-test-docker.sh | 1 + 8 files changed, 8 insertions(+) diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 1c27d1bbd830..5db0e9a2d7ec 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -92,6 +92,7 @@ runs: shell: bash run: | pip install pipx + pipx ensurepath pipx install airbyte-ci/connectors/pipelines/ - name: Run airbyte-ci shell: bash diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index c9bdcf07a985..17216d62f12b 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -113,6 +113,7 @@ jobs: - name: Install CI scripts run: | pip install pipx + pipx ensurepath pipx install airbyte-ci/connectors/ci_credentials pipx install airbyte-ci/connectors/connector_ops - name: Source or Destination harness diff --git a/.github/workflows/connector_metadata_checks.yml b/.github/workflows/connector_metadata_checks.yml index 98411d18fab2..2af441a2bbe9 100644 --- a/.github/workflows/connector_metadata_checks.yml +++ b/.github/workflows/connector_metadata_checks.yml @@ -20,6 +20,7 @@ jobs: - name: Install ci-connector-ops package run: | pip install pipx + pipx ensurepath pipx install airbyte-ci/connectors/connector_ops/ - name: Check test strictness level run: check-test-strictness-level diff --git a/.github/workflows/connector_teams_review_requirements.yml b/.github/workflows/connector_teams_review_requirements.yml index 97e023277f49..f4765eef488e 100644 --- a/.github/workflows/connector_teams_review_requirements.yml +++ b/.github/workflows/connector_teams_review_requirements.yml @@ -25,6 +25,7 @@ jobs: - name: Install ci-connector-ops package run: | pip install pipx + pipx ensurepath pipx install airbyte-ci/connectors/connector_ops - name: Write review requirements file id: write-review-requirements-file diff --git a/.github/workflows/legacy-publish-command.yml b/.github/workflows/legacy-publish-command.yml index 3e08b48b2b42..3a89f2f08341 100644 --- a/.github/workflows/legacy-publish-command.yml +++ b/.github/workflows/legacy-publish-command.yml @@ -252,6 +252,7 @@ jobs: - name: Install CI scripts run: | pip install pipx + pipx ensurepath pipx install airbyte-ci/connectors/ci_credentials pipx install airbyte-ci/connectors/connector_ops - name: Write Integration Test Credentials for ${{ matrix.connector }} diff --git a/.github/workflows/legacy-test-command.yml b/.github/workflows/legacy-test-command.yml index b1a0b82b4277..2634062b3aa6 100644 --- a/.github/workflows/legacy-test-command.yml +++ b/.github/workflows/legacy-test-command.yml @@ -103,6 +103,7 @@ jobs: - name: Install CI scripts run: | pip install pipx + pipx ensurepath pipx install airbyte-ci/connectors/ci_credentials pipx install airbyte-ci/connectors/connector_ops - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index d986d4fd8171..0fa936b646b8 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -92,6 +92,7 @@ jobs: - name: Install CI scripts run: | pip install pipx + pipx ensurepath pipx install airbyte-ci/connectors/ci_credentials - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} run: | diff --git a/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh b/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh index 750c965094de..e599e8549b5a 100755 --- a/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh +++ b/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh @@ -17,6 +17,7 @@ CONNECTOR_DIR="$ROOT_DIR/airbyte-integrations/connectors/$CONNECTOR_NAME" if [ -n "$FETCH_SECRETS" ]; then cd $ROOT_DIR pip install pipx + pipx ensurepath pipx install airbyte-ci/connectors/ci_credentials VERSION=dev ci_credentials $CONNECTOR_NAME write-to-storage || true cd - From 1658ecc1439afe691ebc157f9d547769c808b6d5 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 27 Jul 2023 10:45:32 -0600 Subject: [PATCH 10/10] Add shell bash (#28794) --- .github/workflows/legacy-publish-command.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/legacy-publish-command.yml b/.github/workflows/legacy-publish-command.yml index 3a89f2f08341..2c8b7775076a 100644 --- a/.github/workflows/legacy-publish-command.yml +++ b/.github/workflows/legacy-publish-command.yml @@ -250,12 +250,14 @@ jobs: with: python-version: "3.10" - name: Install CI scripts + shell: bash run: | pip install pipx pipx ensurepath pipx install airbyte-ci/connectors/ci_credentials pipx install airbyte-ci/connectors/connector_ops - name: Write Integration Test Credentials for ${{ matrix.connector }} + shell: bash run: | ci_credentials ${{ matrix.connector }} write-to-storage # normalization also runs destination-specific tests, so fetch their creds also @@ -268,6 +270,7 @@ jobs: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Set Name and Version Environment Vars if: startsWith(matrix.connector, 'connectors') + shell: bash run: | source tools/lib/lib.sh DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile @@ -283,6 +286,7 @@ jobs: - name: Run QA checks for ${{ matrix.connector }} id: qa_checks if: always() + shell: bash run: | run-qa-checks ${{ matrix.connector }} - name: Publish ${{ matrix.connector }} @@ -301,6 +305,7 @@ jobs: attempt_delay: 5000 in # ms - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} if: always() + shell: bash run: | ci_credentials ${{ matrix.connector }} update-secrets # normalization also runs destination-specific tests, so fetch their creds also