From 7cab7c509d4bc0b832151fa7b0b69749835349ba Mon Sep 17 00:00:00 2001 From: midavadim Date: Mon, 23 Oct 2023 17:14:58 +0300 Subject: [PATCH] :tada: Source klaviyo - update schema for events stream (#31565) Co-authored-by: midavadim --- .../connectors/source-klaviyo/Dockerfile | 38 ------------------ .../source-klaviyo/acceptance-test-config.yml | 3 +- .../connectors/source-klaviyo/metadata.yaml | 9 ++++- .../source_klaviyo/schemas/events.json | 16 +++++++- .../sources/klaviyo-migrations.md | 6 +++ docs/integrations/sources/klaviyo.md | 39 ++++++++++--------- 6 files changed, 50 insertions(+), 61 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-klaviyo/Dockerfile create mode 100644 docs/integrations/sources/klaviyo-migrations.md diff --git a/airbyte-integrations/connectors/source-klaviyo/Dockerfile b/airbyte-integrations/connectors/source-klaviyo/Dockerfile deleted file mode 100644 index 78f277b48018..000000000000 --- a/airbyte-integrations/connectors/source-klaviyo/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_klaviyo ./source_klaviyo - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.5.0 -LABEL io.airbyte.name=airbyte/source-klaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml index 1b498c9196ff..f8ad8f3e3985 100644 --- a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml @@ -19,7 +19,8 @@ acceptance_tests: tests: - config_path: secrets/config.json backward_compatibility_tests_config: - disable_for_version: 0.4.0 + disable_for_version: "1.0.0" + previous_connector_version: "0.5.0" full_refresh: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml index 697c24b3eeaf..844e4840d221 100644 --- a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml +++ b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml @@ -6,7 +6,9 @@ data: connectorSubtype: api connectorType: source definitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde - dockerImageTag: 0.5.0 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-klaviyo githubIssueLabel: source-klaviyo icon: klaviyo.svg @@ -18,6 +20,11 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 1.0.0: + message: In this release, for 'events' stream changed type of 'event_properties/items/quantity' field from integer to number. Users will need to refresh the source schema and reset events streams after upgrading. + upgradeDeadline: "2023-11-30" documentationUrl: https://docs.airbyte.com/integrations/sources/klaviyo tags: - language:python diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json index 7adc5ec96495..d8adc26d6f39 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json @@ -21,11 +21,23 @@ "name": { "type": "string" }, "sku": { "type": "string" }, "price": { "type": "number" }, - "quantity": { "type": "integer" } + "quantity": { "type": "number" } } } } - } + }, + "$event_id": { "type": ["null", "string"] }, + "$flow ": { "type": ["null", "string"] }, + "$message": { "type": ["null", "string"] }, + "Campaign Name": { "type": ["null", "string"] }, + "Client Canonical": { "type": ["null", "string"] }, + "Client Name": { "type": ["null", "string"] }, + "Client OS": { "type": ["null", "string"] }, + "Client OS Family": { "type": ["null", "string"] }, + "Client Type": { "type": ["null", "string"] }, + "Email Domain": { "type": ["null", "string"] }, + "Subject": { "type": ["null", "string"] }, + "URL": { "type": ["null", "string"] } }, "person": { "type": "object", diff --git a/docs/integrations/sources/klaviyo-migrations.md b/docs/integrations/sources/klaviyo-migrations.md new file mode 100644 index 000000000000..3bae687b326a --- /dev/null +++ b/docs/integrations/sources/klaviyo-migrations.md @@ -0,0 +1,6 @@ +# Klaviyo Migration Guide + +## Upgrading to 1.0.0 + +`event_properties/items/quantity` for `Events` stream is changed from `integer` to `number`. +For a smooth migration, data reset and schema refresh are needed. \ No newline at end of file diff --git a/docs/integrations/sources/klaviyo.md b/docs/integrations/sources/klaviyo.md index aa299dd04984..b6408c4ffa99 100644 --- a/docs/integrations/sources/klaviyo.md +++ b/docs/integrations/sources/klaviyo.md @@ -52,22 +52,23 @@ The Klaviyo connector should not run into Klaviyo API limitations under normal u ## Changelog -| Version | Date | Pull Request | Subject | -|:---------|:-----------| :--------------------------------------------------------- |:------------------------------------------------------------------------------------------| -| `0.5.0` | 2023-10-19 | [31611](https://github.com/airbytehq/airbyte/pull/31611) | Add `date-time` format for `datetime` field in `Events` stream | -| `0.4.0` | 2023-10-18 | [31562](https://github.com/airbytehq/airbyte/pull/31562) | Add `archived` field to `Flows` stream | -| `0.3.3` | 2023-10-13 | [31379](https://github.com/airbytehq/airbyte/pull/31379) | Skip streams that the connector no longer has access to | -| `0.3.2` | 2023-06-20 | [27498](https://github.com/airbytehq/airbyte/pull/27498) | Do not store state in the future | -| `0.3.1` | 2023-06-08 | [27162](https://github.com/airbytehq/airbyte/pull/27162) | Anonymize check connection error message | -| `0.3.0` | 2023-02-18 | [23236](https://github.com/airbytehq/airbyte/pull/23236) | Add ` Email Templates` stream | -| `0.2.0` | 2023-03-13 | [22942](https://github.com/airbytehq/airbyte/pull/23968) | Add `Profiles` stream | -| `0.1.13` | 2023-02-13 | [22942](https://github.com/airbytehq/airbyte/pull/22942) | Specified date formatting in specification | -| `0.1.12` | 2023-01-30 | [22071](https://github.com/airbytehq/airbyte/pull/22071) | Fix `Events` stream schema | -| `0.1.11` | 2023-01-27 | [22012](https://github.com/airbytehq/airbyte/pull/22012) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| `0.1.10` | 2022-09-29 | [17422](https://github.com/airbytehq/airbyte/issues/17422) | Update CDK dependency | -| `0.1.9` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/issues/17304) | Migrate to per-stream state. | -| `0.1.6` | 2022-07-20 | [14872](https://github.com/airbytehq/airbyte/issues/14872) | Increase test coverage | -| `0.1.5` | 2022-07-12 | [14617](https://github.com/airbytehq/airbyte/issues/14617) | Set max_retries = 10 for `lists` stream. | -| `0.1.4` | 2022-04-15 | [11723](https://github.com/airbytehq/airbyte/issues/11723) | Enhance klaviyo source for flows stream and update to events stream. | -| `0.1.3` | 2021-12-09 | [8592](https://github.com/airbytehq/airbyte/pull/8592) | Improve performance, make Global Exclusions stream incremental and enable Metrics stream. | -| `0.1.2` | 2021-10-19 | [6952](https://github.com/airbytehq/airbyte/pull/6952) | Update schema validation in SAT | +| Version | Date | Pull Request | Subject | +|:---------|:-----------| :--------------------------------------------------------- |:-------------------------------------------------------------------------------------------| +| `1.0.0` | 2023-10-18 | [31565](https://github.com/airbytehq/airbyte/pull/31565) | added new known fields for 'events' stream | +| `0.5.0` | 2023-10-19 | [31611](https://github.com/airbytehq/airbyte/pull/31611) | Add `date-time` format for `datetime` field in `Events` stream | +| `0.4.0` | 2023-10-18 | [31562](https://github.com/airbytehq/airbyte/pull/31562) | Add `archived` field to `Flows` stream | +| `0.3.3` | 2023-10-13 | [31379](https://github.com/airbytehq/airbyte/pull/31379) | Skip streams that the connector no longer has access to | +| `0.3.2` | 2023-06-20 | [27498](https://github.com/airbytehq/airbyte/pull/27498) | Do not store state in the future | +| `0.3.1` | 2023-06-08 | [27162](https://github.com/airbytehq/airbyte/pull/27162) | Anonymize check connection error message | +| `0.3.0` | 2023-02-18 | [23236](https://github.com/airbytehq/airbyte/pull/23236) | Add ` Email Templates` stream | +| `0.2.0` | 2023-03-13 | [22942](https://github.com/airbytehq/airbyte/pull/23968) | Add `Profiles` stream | +| `0.1.13` | 2023-02-13 | [22942](https://github.com/airbytehq/airbyte/pull/22942) | Specified date formatting in specification | +| `0.1.12` | 2023-01-30 | [22071](https://github.com/airbytehq/airbyte/pull/22071) | Fix `Events` stream schema | +| `0.1.11` | 2023-01-27 | [22012](https://github.com/airbytehq/airbyte/pull/22012) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| `0.1.10` | 2022-09-29 | [17422](https://github.com/airbytehq/airbyte/issues/17422) | Update CDK dependency | +| `0.1.9` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/issues/17304) | Migrate to per-stream state. | +| `0.1.6` | 2022-07-20 | [14872](https://github.com/airbytehq/airbyte/issues/14872) | Increase test coverage | +| `0.1.5` | 2022-07-12 | [14617](https://github.com/airbytehq/airbyte/issues/14617) | Set max_retries = 10 for `lists` stream. | +| `0.1.4` | 2022-04-15 | [11723](https://github.com/airbytehq/airbyte/issues/11723) | Enhance klaviyo source for flows stream and update to events stream. | +| `0.1.3` | 2021-12-09 | [8592](https://github.com/airbytehq/airbyte/pull/8592) | Improve performance, make Global Exclusions stream incremental and enable Metrics stream. | +| `0.1.2` | 2021-10-19 | [6952](https://github.com/airbytehq/airbyte/pull/6952) | Update schema validation in SAT |