diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py index 772bc5dd0bb0..fc31d199aee7 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py @@ -98,7 +98,7 @@ def validate_all_tags_are_keyvalue_pairs( def is_major_version(version: str) -> bool: """Check whether the version is of format N.0.0""" semver_version = semver.Version.parse(version) - return semver_version.minor == 0 and semver_version.patch == 0 + return semver_version.minor == 0 and semver_version.patch == 0 and semver_version.prerelease is None def validate_major_version_bump_has_breaking_change_entry( diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_change_prerelease.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_change_prerelease.yaml new file mode 100644 index 000000000000..9dadab96b112 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_change_prerelease.yaml @@ -0,0 +1,19 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 2.0.0-dev.cf3628ccf3 + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 2.0.0: + upgradeDeadline: 2023-08-22 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + tags: + - language:java diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index ce2f101f1b11..080a5b84fe05 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -36,5 +36,5 @@ COPY source_google_sheets ./source_google_sheets ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.4 +LABEL io.airbyte.version=0.3.5 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml index ab5dc26dd814..ca0cbc207383 100644 --- a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: file connectorType: source definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 - dockerImageTag: 0.3.4 + dockerImageTag: 0.3.5 dockerRepository: airbyte/source-google-sheets githubIssueLabel: source-google-sheets icon: google-sheets.svg diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py index 8e0edca55e3c..c111964a44ad 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py @@ -149,7 +149,7 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: ) from err raise Exception(f"Could not run discovery: {reason}") - def read( + def _read( self, logger: AirbyteLogger, config: json, @@ -206,7 +206,22 @@ def read( else: logger.info(f"Skipping syncing sheet {sheet}: {reason}") - logger.info(f"Finished syncing spreadsheet {spreadsheet_id}") + def read( + self, + logger: AirbyteLogger, + config: json, + catalog: ConfiguredAirbyteCatalog, + state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, + ) -> Generator[AirbyteMessage, None, None]: + try: + yield from self._read(logger, config, catalog, state) + except errors.HttpError as e: + if e.status_code == 429: + logger.info(f"Stopped syncing process due to rate limits. {e.reason}") + else: + logger.info(f"{e.status_code}: {e.reason}") + finally: + logger.info(f"Finished syncing spreadsheet {Helpers.get_spreadsheet_id(config['spreadsheet_id'])}") @staticmethod def get_credentials(config): diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py index ab87f0fff9df..626f0ef84f14 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py @@ -2,13 +2,22 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging + import pytest import requests +from airbyte_cdk.models.airbyte_protocol import ( + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, +) from airbyte_cdk.utils import AirbyteTracedException from apiclient import errors from source_google_sheets import SourceGoogleSheets from source_google_sheets.client import GoogleSheetsClient -from source_google_sheets.helpers import SCOPES +from source_google_sheets.helpers import SCOPES, Helpers def test_invalid_credentials_error_message(invalid_config): @@ -58,3 +67,29 @@ def test_discover_403_error(mocker, invalid_config): expected_message = ("Forbidden when requesting spreadsheet with id invalid_spreadsheet_id. The caller does not have right permissions. " "See docs for more details here: https://cloud.google.com/service-infrastructure/docs/service-control/reference/rpc/google.api/servicecontrol.v1#code") assert e.value.args[0] == expected_message + + +def test_read_429_error(mocker, invalid_config, caplog): + source = SourceGoogleSheets() + resp = requests.Response() + resp.status = 429 + resp.reason = "Request a higher quota limit" + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=mocker.Mock) + mocker.patch.object(Helpers, "get_sheets_in_spreadsheet", side_effect=errors.HttpError(resp=resp, content=b'')) + + sheet1 = "soccer_team" + sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) + sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=sheet1, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) + assert [] == records + assert "Stopped syncing process due to rate limits. Request a higher quota limit" in caplog.text diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl index a08553bb736f..643ca48f835d 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl @@ -33,9 +33,9 @@ {"stream": "forms", "data": {"id": "03e69987-1dcb-4d55-9cb6-d3812ac00ee6", "name": "New form 93", "createdAt": "2023-02-13T16:56:33.108Z", "updatedAt": "2023-02-13T16:56:33.108Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195424} {"stream": "forms", "data": {"id": "0a7fd84f-471e-444a-a4e0-ca36d39f8af7", "name": "New form 27", "createdAt": "2023-02-13T16:45:22.640Z", "updatedAt": "2023-02-13T16:45:22.640Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195425} {"stream": "forms", "data": {"id": "0bf0c00f-e68d-4de2-8cd9-d9b04e41072f", "name": "New form 55", "createdAt": "2023-02-13T16:50:27.345Z", "updatedAt": "2023-02-13T16:50:27.345Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195425} -{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": null, "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-08-01T00:45:14.864000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-01T00:45:14.864Z", "archived": false}, "emitted_at": 1691139359639} -{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": null, "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-04-10T22:31:22.345000+00:00", "hs_lastmodifieddate": "2023-07-18T15:45:26.602000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "pending", "hs_status_display_order": 5, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-07-18T15:45:26.602Z", "archived": false}, "emitted_at": 1691139359640} -{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": null, "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-04T00:38:43.574000+00:00", "hs_lastmodifieddate": "2023-08-04T00:38:43.617000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "in_progress", "hs_status_display_order": 1, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-04T00:38:43.617Z", "archived": false}, "emitted_at": 1691139359641} +{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-08-18T14:59:25.726000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-18T14:59:25.726Z", "archived": false}, "emitted_at": 1692530575531} +{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-04-10T22:31:22.345000+00:00", "hs_lastmodifieddate": "2023-08-18T14:59:25.726000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "pending", "hs_status_display_order": 5, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-18T14:59:25.726Z", "archived": false}, "emitted_at": 1692530575532} +{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-19T22:11:13.040000+00:00", "hs_lastmodifieddate": "2023-08-19T22:11:13.080000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "in_progress", "hs_status_display_order": 1, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-19T22:11:13.080Z", "archived": false}, "emitted_at": 1692530575532} {"stream": "line_items", "data": {"id": "4617680695", "properties": {"amount": 34.0, "createdate": "2023-01-31T00:31:29.812000+00:00", "description": null, "discount": null, "hs_acv": 34.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-01-31T00:31:29.812000+00:00", "hs_line_item_currency_code": null, "hs_margin": 34.0, "hs_margin_acv": 34.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 34.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 4617680695, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 34.0, "hs_product_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 34.0, "hs_term_in_months": null, "hs_total_discount": 0.0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "test", "price": 34, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-01-31T00:31:29.812Z", "updatedAt": "2023-01-31T00:31:29.812Z", "archived": false}, "emitted_at": 1689697250135} {"stream": "line_items", "data": {"id": "5153237390", "properties": {"amount": 95.0, "createdate": "2023-04-04T21:28:36.663000+00:00", "description": "Baseball hat, medium", "discount": 5, "hs_acv": 95.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-04-04T21:28:36.663000+00:00", "hs_line_item_currency_code": null, "hs_margin": 90.0, "hs_margin_acv": 90.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 90.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 5153237390, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 100.0, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 95.0, "hs_term_in_months": null, "hs_total_discount": 5.0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 100, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-04-04T21:28:36.663Z", "updatedAt": "2023-04-04T21:28:36.663Z", "archived": false}, "emitted_at": 1689697250136} {"stream":"marketing_emails","data":{"ab":false,"abHoursToWait":4,"abSampleSizeDefault":null,"abSamplingDefault":null,"abSuccessMetric":null,"abTestPercentage":50,"abVariation":false,"absoluteUrl":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","allEmailCampaignIds":[243851494],"analyticsPageId":"100523515217","analyticsPageType":"email","archivedAt":0,"archivedInDashboard":false,"audienceAccess":"PUBLIC","author":"integration-test@airbyte.io","authorName":"Team-1 Airbyte","blogRssSettings":null,"canSpamSettingsId":36765207029,"categoryId":2,"contentAccessRuleIds":[],"contentAccessRuleTypes":[],"contentTypeCategory":2,"createPage":false,"created":1675121582718,"createdById":12282590,"currentState":"PUBLISHED","currentlyPublished":true,"customReplyTo":"","customReplyToEnabled":false,"domain":"","emailBody":"{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}","emailNote":"","emailTemplateMode":"DRAG_AND_DROP","emailType":"BATCH_EMAIL","emailbodyPlaintext":"","feedbackSurveyId":null,"flexAreas":{"main":{"boxed":false,"isSingleColumnFullWidth":false,"sections":[{"columns":[{"id":"column-0-0","widgets":["module-0-0-0"],"width":12}],"id":"section-0","style":{"backgroundColor":"#eaf0f6","backgroundType":"CONTENT","paddingBottom":"10px","paddingTop":"10px"}},{"columns":[{"id":"column-1-0","widgets":["module-1-0-0"],"width":12}],"id":"section-1","style":{"backgroundType":"CONTENT","paddingBottom":"30px","paddingTop":"30px"}},{"columns":[{"id":"column-2-0","widgets":["module-2-0-0"],"width":12}],"id":"section-2","style":{"backgroundColor":"","backgroundType":"CONTENT","paddingBottom":"20px","paddingTop":"20px"}}]}},"freezeDate":1675121645993,"fromName":"Team Airbyte","hasContentAccessRules":false,"htmlTitle":"","id":100523515217,"isCreatedFomSandboxSync":false,"isGraymailSuppressionEnabled":true,"isInstanceLayoutPage":false,"isPublished":true,"isRecipientFatigueSuppressionEnabled":null,"language":"en","layoutSections":{},"liveDomain":"integrationtest-dev-8727216-8727216.hs-sites.com","mailingListsExcluded":[],"mailingListsIncluded":[],"maxRssEntries":5,"metaDescription":"","name":"test","pageExpiryEnabled":false,"pageRedirected":false,"pastMabExperimentIds":[],"portalId":8727216,"previewKey":"nlkwziGL","primaryEmailCampaignId":243851494,"processingStatus":"PUBLISHED","publishDate":1675121645997,"publishImmediately":true,"publishedAt":1675121646297,"publishedByEmail":"integration-test@airbyte.io","publishedById":12282590,"publishedByName":"Team-1 Airbyte","publishedUrl":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","replyTo":"integration-test@airbyte.io","resolvedDomain":"integrationtest-dev-8727216-8727216.hs-sites.com","rssEmailByText":"By","rssEmailClickThroughText":"Read more »","rssEmailCommentText":"Comment »","rssEmailEntryTemplateEnabled":false,"rssEmailImageMaxWidth":0,"rssEmailUrl":"","sections":{},"securityState":"NONE","selected":0,"slug":"-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","smartEmailFields":{},"state":"PUBLISHED","stats":{"counters":{"sent":0,"open":0,"delivered":0,"bounce":0,"unsubscribed":0,"click":0,"reply":0,"dropped":1,"selected":1,"spamreport":0,"suppressed":0,"hardbounced":0,"softbounced":0,"pending":0,"contactslost":0,"notsent":1},"deviceBreakdown":{"open_device_type":{"computer":0,"mobile":0,"unknown":0},"click_device_type":{"computer":0,"mobile":0,"unknown":0}},"failedToLoad":false,"qualifierStats":{},"ratios":{"clickratio":0,"clickthroughratio":0,"deliveredratio":0,"openratio":0,"replyratio":0,"unsubscribedratio":0,"spamreportratio":0,"bounceratio":0,"hardbounceratio":0,"softbounceratio":0,"contactslostratio":0,"pendingratio":0,"notsentratio":100}},"styleSettings":{"background_color":"#EAF0F6","background_image":null,"background_image_type":null,"body_border_color":"#EAF0F6","body_border_color_choice":"BORDER_MANUAL","body_border_width":"1","body_color":"#ffffff","color_picker_favorite1":null,"color_picker_favorite2":null,"color_picker_favorite3":null,"color_picker_favorite4":null,"color_picker_favorite5":null,"color_picker_favorite6":null,"email_body_padding":null,"email_body_width":null,"heading_one_font":{"bold":null,"color":null,"font":null,"font_style":{},"italic":null,"size":"28","underline":null},"heading_two_font":{"bold":null,"color":null,"font":null,"font_style":{},"italic":null,"size":"22","underline":null},"links_font":{"bold":false,"color":"#00a4bd","font":null,"font_style":{},"italic":false,"size":null,"underline":true},"primary_accent_color":null,"primary_font":"Arial, sans-serif","primary_font_color":"#23496d","primary_font_line_height":null,"primary_font_size":"15","secondary_accent_color":null,"secondary_font":"Arial, sans-serif","secondary_font_color":"#23496d","secondary_font_line_height":null,"secondary_font_size":"12","use_email_client_default_settings":false,"user_module_defaults":{"button_email":{"background_color":"#00a4bd","corner_radius":8,"font":"Arial, sans-serif","font_color":"#ffffff","font_size":16,"font_style":{"color":"#ffffff","font":"Arial, sans-serif","size":{"units":"px","value":16},"styles":{"bold":false,"italic":false,"underline":false}}},"email_divider":{"color":{"color":"#23496d","opacity":100},"height":1,"line_type":"solid"}}},"subcategory":"batch","subject":"test","subscription":23704464,"subscriptionName":"Test sub","teamPerms":[],"templatePath":"@hubspot/email/dnd/welcome.html","transactional":false,"translations":{},"unpublishedAt":0,"updated":1675121702583,"updatedById":12282590,"url":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","useRssHeadlineAsSubject":false,"userPerms":[],"vidsExcluded":[],"vidsIncluded":[2501],"visibleToAll":true},"emitted_at":1688060624527} @@ -43,7 +43,7 @@ {"stream": "products", "data": {"id": "1783898388", "properties": {"amount": null, "createdate": "2023-01-31T00:08:27.149000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-01-31T00:28:58.829000+00:00", "hs_merged_object_ids": null, "hs_object_id": 1783898388, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_type": "inventory", "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "test", "price": 1, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-01-31T00:08:27.149Z", "updatedAt": "2023-01-31T00:28:58.829Z", "archived": false}, "emitted_at": 1689697253487} {"stream": "property_history", "data": {"value": "sh", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1673944973763, "selected": false, "property": "firstname", "vid": 151}, "emitted_at": 1685387222765} {"stream": "subscription_changes", "data": {"timestamp": 1675123491624, "recipient": "testingapicontact_0@hubspot.com", "portalId": 8727216, "normalizedEmailId": "6b59e963-cabc-4bf8-baec-feab401bdd98", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1675123491624, "change": "SUBSCRIBED", "portalId": 8727216, "subscriptionId": 23704464, "causedByEvent": {"id": "cd276838-3925-4649-9a38-2b61761362c4", "created": 1675123491624}, "changeType": "SUBSCRIPTION_STATUS"}]}, "emitted_at": 1685387223625} -{"stream": "tickets", "data": {"id": "1401690016", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2023-01-30T23:52:42.464000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2023-01-30T23:52:42.464000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2023-01-30T23:52:42.464000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2023-01-30T23:52:43.939000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_times_contacted": 0, "hs_object_id": 1401690016, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 1401690016, "hs_ticket_priority": null, "hs_time_in_1": 14574501432, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:52:42.464000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": null, "subject": "test", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2023-01-30T23:52:42.464Z", "updatedAt": "2023-01-30T23:52:43.939Z", "archived": false}, "emitted_at": 1689697263984} +{"stream": "tickets", "data": {"id": "1401690016", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2023-01-30T23:52:42.464000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2023-01-30T23:52:42.464000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2023-01-30T23:52:42.464000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2023-01-30T23:52:43.939000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_times_contacted": 0, "hs_object_id": 1401690016, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 1401690016, "hs_ticket_priority": null, "hs_time_in_1": 17411822718, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:52:42.464000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": null, "subject": "test", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2023-01-30T23:52:42.464Z", "updatedAt": "2023-01-30T23:52:43.939Z", "archived": false}, "emitted_at": 1692534585461} {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 40032127, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 321690519}, "name": "Unnamed workflow - Mon Mar 15 2021 12:58:03 GMT+0200 (cloned)", "id": 40032127, "type": "DRIP_DELAY", "enabled": true, "creationSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-userweb"}, "createdByUser": {"userId": 12282590, "userEmail": "integration-test@airbyte.io"}, "clonedFromWorkflowId": 23314874, "createdAt": 1675124258186}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-userweb"}, "updatedByUser": {"userId": 12282590, "userEmail": "integration-test@airbyte.io"}, "updatedAt": 1675124308226}, "originalAuthorUserId": 12282590, "contactListIds": {"enrolled": 167, "active": 168, "completed": 169, "succeeded": 170}, "personaTagIds": [], "lastUpdatedByUserId": 12282590, "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1675124258190, "updatedAt": 1675124308226, "description": ""}, "emitted_at": 1685387227678} {"stream": "pets", "data": {"id": "5936415312", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:08:50.632000+00:00", "hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5936415312, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Marcos Pet", "pet_type": "Dog"}, "createdAt": "2023-04-12T17:08:50.632Z", "updatedAt": "2023-04-12T17:08:50.632Z", "archived": false}, "emitted_at": 1689697266624} {"stream": "pets", "data": {"id": "5938880054", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:53:12.692000+00:00", "hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880054, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Integration Test Pet", "pet_type": "Unknown"}, "createdAt": "2023-04-12T17:53:12.692Z", "updatedAt": "2023-04-12T17:53:12.692Z", "archived": false}, "emitted_at": 1689697266625} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml index a9da94f8248d..b303c49e4115 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml @@ -27,6 +27,10 @@ acceptance_tests: path: "integration_tests/expected_records.jsonl" fail_on_extra_columns: true timeout_seconds: 3600 + ignored_fields: + campaign_groups: + - name: "lastModified" + bypass_reason: "Volatile data" empty_streams: - name: ad_member_company_size_analytics bypass_reason: "Empty stream; Retention period is 2y" diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index 36cc0272e398..7038660a56b2 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=3.17.3 +LABEL io.airbyte.version=3.17.4 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index a8a194d90585..e46be4d24e8f 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 3.17.3 + dockerImageTag: 3.17.4 dockerRepository: airbyte/source-stripe githubIssueLabel: source-stripe icon: stripe.svg diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py index 41189ed0952c..3d8caf860ee5 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py @@ -8,6 +8,35 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy +from requests import HTTPError + +STRIPE_ERROR_CODES = { + "more_permissions_required": "This is most likely due to insufficient permissions on the credentials in use. " + "Try to grant required permissions/scopes or re-authenticate", + "account_invalid": "The card, or account the card is connected to, is invalid. You need to contact your card issuer " + "to check that the card is working correctly.", + "oauth_not_supported": "Please use a different authentication method.", +} + + +class StripeAvailabilityStrategy(HttpAvailabilityStrategy): + def handle_http_error( + self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError + ) -> Tuple[bool, Optional[str]]: + status_code = error.response.status_code + if status_code not in [400, 403]: + raise error + parsed_error = error.response.json() + error_code = parsed_error.get("error", {}).get("code") + error_message = STRIPE_ERROR_CODES.get(error_code, parsed_error.get("error", {}).get("message")) + if not error_message: + raise error + doc_ref = self._visit_docs_message(logger, source) + reason = f"The endpoint {error.response.url} returned {status_code}: {error.response.reason}. {error_message}. {doc_ref} " + response_error_message = stream.parse_response_error_message(error.response) + if response_error_message: + reason += response_error_message + return False, reason class StripeSubStreamAvailabilityStrategy(HttpAvailabilityStrategy): diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json index b347163b58bc..0a6bb196269d 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json @@ -666,27 +666,7 @@ "type": ["null", "string"] }, "shipping_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" } } }, @@ -701,27 +681,7 @@ "type": ["null", "object"], "properties": { "billing_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" }, "email": { "type": ["null", "string"] @@ -730,27 +690,7 @@ "type": ["null", "string"] }, "shipping_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index d26f3bfc7b4b..224a33a84ee1 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -13,14 +13,8 @@ from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from source_stripe.availability_strategy import StripeSubStreamAvailabilityStrategy - -STRIPE_ERROR_CODES: List = [ - # stream requires additional permissions - "more_permissions_required", - # account_id doesn't have the access to the stream - "account_invalid", -] +from source_stripe.availability_strategy import StripeAvailabilityStrategy, StripeSubStreamAvailabilityStrategy + STRIPE_API_VERSION = "2022-11-15" @@ -30,6 +24,10 @@ class StripeStream(HttpStream, ABC): DEFAULT_SLICE_RANGE = 365 transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeAvailabilityStrategy() + def __init__(self, start_date: int, account_id: str, slice_range: int = DEFAULT_SLICE_RANGE, **kwargs): super().__init__(**kwargs) self.account_id = account_id @@ -66,27 +64,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp response_json = response.json() yield from response_json.get("data", []) # Stripe puts records in a container array "data" - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - try: - yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) - except requests.exceptions.HTTPError as e: - status_code = e.response.status_code - parsed_error = e.response.json() - error_code = parsed_error.get("error", {}).get("code") - error_message = parsed_error.get("message") - # if the API Key doesn't have required permissions to particular stream, this stream will be skipped - if status_code == 403 and error_code in STRIPE_ERROR_CODES: - self.logger.warn(f"Stream {self.name} is skipped, due to {error_code}. Full message: {error_message}") - pass - else: - self.logger.error(f"Syncing stream {self.name} is failed, due to {error_code}. Full message: {error_message}") - class BasePaginationStripeStream(StripeStream, ABC): def request_params( diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py new file mode 100644 index 000000000000..f72068c051d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +@pytest.fixture(autouse=True) +def disable_cache(mocker): + mocker.patch( + "source_stripe.streams.Customers.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + mocker.patch( + "source_stripe.streams.Transfers.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + mocker.patch( + "source_stripe.streams.Subscriptions.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + mocker.patch( + "source_stripe.streams.SubscriptionItems.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + + +@pytest.fixture(name="config") +def config_fixture(): + config = {"client_secret": "sk_test(live)_", + "account_id": "", "start_date": "2020-05-01T00:00:00Z"} + return config + + +@pytest.fixture(name="stream_args") +def stream_args_fixture(): + authenticator = TokenAuthenticator("sk_test(live)_") + args = { + "authenticator": authenticator, + "account_id": "", + "start_date": 1588315041, + "slice_range": 365, + } + return args diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index 8646d83321e1..f5ea6c77b31c 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -45,30 +45,16 @@ def test_source_streams(): assert len(streams) == 46 -@pytest.fixture(name="config") -def config_fixture(): - config = {"client_secret": "sk_test(live)_", - "account_id": "", "start_date": "2020-05-01T00:00:00Z"} - return config - - -@pytest.fixture(name="logger_mock") -def logger_mock_fixture(): - return patch("source_tiktok_marketing.source.logger") - - @patch.object(source_stripe.source, "stripe") -def test_source_check_connection_ok(mocked_client, config, logger_mock): - assert SourceStripe().check_connection( - logger_mock, config=config) == (True, None) +def test_source_check_connection_ok(mocked_client, config): + assert SourceStripe().check_connection(None, config=config) == (True, None) @patch.object(source_stripe.source, "stripe") -def test_source_check_connection_failure(mocked_client, config, logger_mock): +def test_source_check_connection_failure(mocked_client, config): exception = Exception("Test") mocked_client.Account.retrieve = Mock(side_effect=exception) - assert SourceStripe().check_connection( - logger_mock, config=config) == (False, exception) + assert SourceStripe().check_connection(None, config=config) == (False, exception) @patch.object(source_stripe.source, "stripe") diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index e51b491e7ad6..1fd42de179e9 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -2,9 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging + import pendulum import pytest from airbyte_cdk.models import SyncMode +from source_stripe.availability_strategy import STRIPE_ERROR_CODES from source_stripe.streams import ( ApplicationFees, ApplicationFeesRefunds, @@ -152,12 +155,6 @@ def test_sub_stream(requests_mock): ] -@pytest.fixture(name="config") -def config_fixture(): - config = {"authenticator": "authenticator", "account_id": "", "start_date": 1596466368} - return config - - @pytest.mark.parametrize( "stream_cls, kwargs, expected", [ @@ -198,9 +195,9 @@ def test_path_and_headers( stream_cls, kwargs, expected, - config, + stream_args, ): - stream = stream_cls(**config) + stream = stream_cls(**stream_args) assert stream.path(**kwargs) == expected headers = stream.request_headers(**kwargs) assert headers["Stripe-Version"] == "2022-11-15" @@ -241,6 +238,44 @@ def test_request_params( stream, kwargs, expected, - config, + stream_args, ): - assert stream(**config).request_params(**kwargs) == expected + assert stream(**stream_args).request_params(**kwargs) == expected + + +@pytest.mark.parametrize( + "stream_cls", + ( + ApplicationFees, + Customers, + BalanceTransactions, + Charges, + Coupons, + Disputes, + Events, + Invoices, + InvoiceItems, + Payouts, + Plans, + Prices, + Products, + Subscriptions, + SubscriptionSchedule, + Transfers, + Refunds, + PaymentIntents, + CheckoutSessions, + PromotionCodes, + ExternalAccount, + SetupIntents, + ShippingRates + ) +) +def test_403_error_handling(stream_args, stream_cls, requests_mock): + stream = stream_cls(**stream_args) + logger = logging.getLogger("airbyte") + for error_code in STRIPE_ERROR_CODES: + requests_mock.get(f"{stream.url_base}{stream.path()}", status_code=403, json={"error": {"code": f"{error_code}"}}) + available, message = stream.check_availability(logger) + assert not available + assert STRIPE_ERROR_CODES[error_code] in message diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md index a525293c4bf7..78cb0fa74648 100644 --- a/docs/integrations/sources/google-sheets.md +++ b/docs/integrations/sources/google-sheets.md @@ -76,7 +76,8 @@ The [Google API rate limit](https://developers.google.com/sheets/api/limits) is | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|-----------------------------------------------------------------------------------| -| 0.3.4 | 2023-05-15 | [29453](https://github.com/airbytehq/airbyte/pull/29453) | Update spec descriptions | +| 0.3.5 | 2023-08-16 | [29427](https://github.com/airbytehq/airbyte/pull/29427) | Add stop reading in case of 429 error | +| 0.3.4 | 2023-05-15 | [29453](https://github.com/airbytehq/airbyte/pull/29453) | Update spec descriptions | | 0.3.3 | 2023-08-10 | [29327](https://github.com/airbytehq/airbyte/pull/29327) | Add user-friendly error message for 404 and 403 error while discover | | 0.3.2 | 2023-08-09 | [29246](https://github.com/airbytehq/airbyte/pull/29246) | Add checking while reading to skip modified sheets | | 0.3.1 | 2023-07-06 | [28033](https://github.com/airbytehq/airbyte/pull/28033) | Fixed several reported vulnerabilities (25 total), CVE-2022-37434, CVE-2022-42898 | diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index c9835cd13be5..9a69d906d11a 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -1,33 +1,50 @@ # Stripe -:::warning -Stripe API Restriction: Access to the events endpoint is [guaranteed only for the last 30 days](https://stripe.com/docs/api/events). Using the full-refresh-overwrite sync from Airbyte will delete the events data older than 30 days from your target destination. -::: - -This page guides you through the process of setting up the Stripe source connector. +This page contains the setup guide and reference information for the Stripe source connector. ## Prerequisites -- Your [Stripe `Account ID`](https://dashboard.stripe.com/settings/account) -- Your [Stripe `Secret Key`](https://dashboard.stripe.com/apikeys) +- Access to the Stripe account containing the data you wish to replicate + +## Setup Guide + +To authenticate the Stripe connector, you need to use a Stripe API key. Although you may use an existing key, we recommend that you create a new restricted key specifically for Airbyte and grant it **Read** privileges only. We also recommend granting **Read** privileges to all available permissions, and configuring the specific data you would like to replicate within Airbyte. + +### Create a Stripe Secret Key + +1. Log in to your [Stripe account](https://dashboard.stripe.com/login). +2. In the top navigation bar, click **Developers**. +3. In the top-left corner, click **API keys**. +4. Click **+ Create restricted key**. +5. Choose a **Key name**, and select **Read** for all available permissions. +6. Click **Create key**. You may be prompted to enter a confirmation code sent to your email address. + +For more information on Stripe API Keys, see the [Stripe documentation](https://stripe.com/docs/keys). + +### Set up the Stripe source connector in Airbyte + +1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. Find and select **Stripe** from the list of available sources. +4. For **Source name**, enter a name to help you identify this source. +5. For **Account ID**, enter your Stripe Account ID. This ID begins with `acct_`, and can be found in the top-right corner of your Stripe [account settings page](https://dashboard.stripe.com/settings/account). +6. For **Secret Key**, enter the restricted key you created for the connection. +7. For **Replication Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. +8. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments (such as tips, refunds, chargebacks, etc). -## Set up the Stripe source connector + - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced. + - Setting the **Lookback Window** to 1 means Airbyte will re-export data from the past day, capturing any changes made in the last 24 hours. + - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week. -1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. -2. Click **Sources** and then click **+ New source**. -3. On the Set up the source page, select **Stripe** from the Source type dropdown. -4. Enter a name for your source. -5. For **Account ID**, enter your [Stripe `Account ID`](https://dashboard.stripe.com/settings/account). -6. For **Secret Key**, enter your [Stripe `Secret Key`](https://dashboard.stripe.com/apikeys) +9. (Optional) For **Data Request Window**, you may specify the time window in days used by the connector when requesting data from the Stripe API. This window defines the span of time covered in each request, with larger values encompassing more days in a single request. Generally speaking, the lack of overhead from making fewer requests means a larger window is faster to sync. However, this also means the state of the sync will persist less frequently. If an issue occurs or the sync is interrupted, a larger window means more data will need to be resynced, potentially causing a delay in the overall process. - We recommend creating a secret key specifically for Airbyte to control which resources Airbyte can access. For ease of use, we recommend granting read permission to all resources and configuring which resource to replicate in the Airbyte UI. You can also use the API keys for the [test mode](https://stripe.com/docs/keys#obtain-api-keys) to try out the Stripe integration with Airbyte. + For example, if you are replicating three years worth of data: -7. For **Replication start date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. -8. For **Lookback Window in days (Optional)**, select the number of days the value in days prior to the start date that you to sync your data with. If your data is updated after setting up this connector, you can use the this option to reload data from the past N days. Example: If the Replication start date is set to `2021-01-01T00:00:00Z`, then: - - If you leave the Lookback Window in days parameter to its the default value of 0, Airbyte will sync data from the Replication start date `2021-01-01T00:00:00Z` - - If the Lookback Window in days value is set to 1, Airbyte will consider the Replication start date to be `2020-12-31T00:00:00Z` - - If the Lookback Window in days value is set to 7, Airbyte will sync data from `2020-12-25T00:00:00Z` -9. Click **Set up source**. + - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted. + - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs. + + If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days. +10. Click **Set up source** and wait for the tests to complete. ## Supported sync modes @@ -40,10 +57,11 @@ The Stripe source connector supports the following [sync modes](https://docs.air Since the Stripe API does not allow querying objects which were updated since the last sync, the Stripe connector uses the `created` field to query for new data in your Stripe account. ::: -## Supported Streams +## Supported streams The Stripe source connector supports the following streams: +- [Accounts](https://stripe.com/docs/api/accounts/list) \(Incremental\) - [Application Fees](https://stripe.com/docs/api/application_fees) \(Incremental\) - [Application Fee Refunds](https://stripe.com/docs/api/fee_refunds/list) - [Authorizations](https://stripe.com/docs/api/issuing/authorizations/list) \(Incremental\) @@ -52,18 +70,23 @@ The Stripe source connector supports the following streams: - [Cardholders](https://stripe.com/docs/api/issuing/cardholders/list) \(Incremental\) - [Cards](https://stripe.com/docs/api/issuing/cards/list) \(Incremental\) - [Charges](https://stripe.com/docs/api/charges/list) \(Incremental\) - - The `amount` column defaults to the smallest currency unit. (See [charge object](https://stripe.com/docs/api/charges/object) for more details) + :::note + The `amount` column defaults to the smallest currency unit. Check [the Stripe docs](https://stripe.com/docs/api/charges/object) for more details. + ::: - [Checkout Sessions](https://stripe.com/docs/api/checkout/sessions/list) - [Checkout Sessions Line Items](https://stripe.com/docs/api/checkout/sessions/line_items) - [Coupons](https://stripe.com/docs/api/coupons/list) \(Incremental\) -- [CreditNotes](https://stripe.com/docs/api/credit_notes/list) \(Full Refresh\) +- [Credit Notes](https://stripe.com/docs/api/credit_notes/list) \(Full Refresh\) - [Customer Balance Transactions](https://stripe.com/docs/api/customer_balance_transactions/list) - [Customers](https://stripe.com/docs/api/customers/list) \(Incremental\) - - This endpoint does not include deleted customers + :::note + This endpoint does _not_ include deleted customers + ::: - [Disputes](https://stripe.com/docs/api/disputes/list) \(Incremental\) - [Early Fraud Warnings](https://stripe.com/docs/api/radar/early_fraud_warnings/list) \(Incremental\) - [Events](https://stripe.com/docs/api/events/list) \(Incremental\) - - The Stripe API does not guarantee access to events older than 30 days, so this stream will only pull events created from the 30 days prior to the initial sync and not from the Replication start date. +- [File Links](https://stripe.com/docs/api/file_links/list) \(Incremental\) +- [Files](https://stripe.com/docs/api/files/list) \(Incremental\) - [Invoice Items](https://stripe.com/docs/api/invoiceitems/list) \(Incremental\) - [Invoice Line Items](https://stripe.com/docs/api/invoices/invoice_lines) - [Invoices](https://stripe.com/docs/api/invoices/list) \(Incremental\) @@ -77,24 +100,25 @@ The Stripe source connector supports the following streams: - [Products](https://stripe.com/docs/api/products/list) \(Incremental\) - [Refunds](https://stripe.com/docs/api/refunds/list) \(Incremental\) - [Reviews](https://stripe.com/docs/api/radar/reviews/list) \(Incremental\) -- [SetupIntents](https://stripe.com/docs/api/setup_intents/list) \(Incremental\) -- [ShippingRates](https://stripe.com/docs/api/shipping_rates/list) \(Incremental\) +- [Setup Attempts](https://stripe.com/docs/api/setup_attempts/list) \(Incremental\) +- [Setup Intents](https://stripe.com/docs/api/setup_intents/list) \(Incremental\) +- [Shipping Rates](https://stripe.com/docs/api/shipping_rates/list) \(Incremental\) - [Subscription Items](https://stripe.com/docs/api/subscription_items/list) - [Subscription Schedule](https://stripe.com/docs/api/subscription_schedules) \(Incremental\) - [Subscriptions](https://stripe.com/docs/api/subscriptions/list) \(Incremental\) +- [Top Ups](https://stripe.com/docs/api/topups/list) \(Incremental\) - [Transactions](https://stripe.com/docs/api/transfers/list) \(Incremental\) - [Transfers](https://stripe.com/docs/api/transfers/list) \(Incremental\) - [Transfer Reversals](https://stripe.com/docs/api/transfer_reversals/list) -- [Accounts](https://stripe.com/docs/api/accounts/list) \(Incremental\) -- [Setup Attempts](https://stripe.com/docs/api/setup_attempts/list) \(Incremental\) - [Usage Records](https://stripe.com/docs/api/usage_records/subscription_item_summary_list) -- [TopUps](https://stripe.com/docs/api/topups/list) \(Incremental\) -- [Files](https://stripe.com/docs/api/files/list) \(Incremental\) -- [FileLinks](https://stripe.com/docs/api/file_links/list) \(Incremental\) + +:::warning +**Stripe API Restriction on Events Data**: Access to the events endpoint is [guaranteed only for the last 30 days](https://stripe.com/docs/api/events) by Stripe. If you use the Full Refresh Overwrite sync, be aware that any events data older than 30 days will be **deleted** from your target destination and replaced with the data from the last 30 days only. Use an Append sync mode to ensure historical data is retained. +::: ### Data type mapping -The [Stripe API](https://stripe.com/docs/api) uses the same [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions are performed for the Stripe connector. +The [Stripe API](https://stripe.com/docs/api) uses the same [JSON Schema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions are performed for the Stripe connector. ### Performance considerations @@ -103,8 +127,9 @@ The Stripe connector should not run into Stripe API limitations under normal usa ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------| :------------------------------------------------------- |:-----------------------------------------------------------------------------------------------------------------------------------------------------| -| 3.17.3 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.17.4 | 2023-08-15 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Revert 3.17.3 | +| 3.17.3 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property | | 3.17.2 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Fix stream schemas, remove custom 403 error handling | | 3.17.1 | 2023-08-01 | [28887](https://github.com/airbytehq/airbyte/pull/28887) | Fix `Invoices` schema | | 3.17.0 | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream |