fix(source-jira): update pk for workflows stream (#39467)

airbytehq · Jun 18, 2024 · 555a607 · 555a607
1 parent 1090fd6
commit 555a607
Show file tree

Hide file tree

Showing 7 changed files with 51 additions and 7 deletions.
diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl
@@ -145,9 +145,9 @@
 {"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "emailAddress": "[email protected]", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 30, "items": [{"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"}, {"name": "confluence-users", "groupId": "38d808e9-113f-45c4-817b-099e953b687a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=38d808e9-113f-45c4-817b-099e953b687a"}, {"name": "integration-test-group", "groupId": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"}, {"name": "jira-administrators", "groupId": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"}, {"name": "jira-admins-airbyteio", "groupId": "2d55cbe0-4cab-46a4-853e-ec31162ab9a3", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"}, {"name": "jira-servicemanagement-customers-airbyteio", "groupId": "125680d3-7e85-41ad-a662-892b6590272e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=125680d3-7e85-41ad-a662-892b6590272e"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}, {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, {"name": "site-admins", "groupId": "76dad095-fc1a-467a-88b4-fde534220985", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"}, {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}, {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}, {"name": "Test group 10", "groupId": "e9f74708-e33c-4158-919d-6457f50c6e74", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}, {"name": "Test group 11", "groupId": "b0e6d76f-701a-4208-a88d-4478f242edde", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=b0e6d76f-701a-4208-a88d-4478f242edde"}, {"name": "Test group 12", "groupId": "dddc24a0-ef00-407e-abef-5a660b6f55cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dddc24a0-ef00-407e-abef-5a660b6f55cf"}, {"name": "Test group 13", "groupId": "dbe4af74-8387-4b08-843b-86af78dd738e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dbe4af74-8387-4b08-843b-86af78dd738e"}, {"name": "Test group 14", "groupId": "d4570a20-38d8-44cc-a63b-0924d0d0d0ff", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=d4570a20-38d8-44cc-a63b-0924d0d0d0ff"}, {"name": "Test group 15", "groupId": "87bde5c0-7231-44a7-88b5-421da2ab8052", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=87bde5c0-7231-44a7-88b5-421da2ab8052"}, {"name": "Test group 16", "groupId": "538b6aa2-bf57-402f-93c0-c2e2d68b7155", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=538b6aa2-bf57-402f-93c0-c2e2d68b7155"}, {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=022bc924-ac57-442d-80c9-df042b73ad87"}, {"name": "Test group 18", "groupId": "bbfc6fc9-96db-4e66-88f4-c55b08298272", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bbfc6fc9-96db-4e66-88f4-c55b08298272"}, {"name": "Test group 19", "groupId": "3c4fef5d-9721-4f20-9a68-346d222de3cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3c4fef5d-9721-4f20-9a68-346d222de3cf"}, {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}, {"name": "Test group 3", "groupId": "638aa1ad-8707-4d56-9361-f5959b6c4785", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=638aa1ad-8707-4d56-9361-f5959b6c4785"}, {"name": "Test group 4", "groupId": "532554e0-43be-4eca-9186-b417dcf38547", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=532554e0-43be-4eca-9186-b417dcf38547"}, {"name": "Test group 5", "groupId": "6b663734-85b6-4185-8fb2-9ac27709b3aa", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=6b663734-85b6-4185-8fb2-9ac27709b3aa"}, {"name": "Test group 6", "groupId": "2d4af5cf-cd34-4e78-9445-abc000cdd5cc", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d4af5cf-cd34-4e78-9445-abc000cdd5cc"}, {"name": "Test group 7", "groupId": "e8a97909-d807-4f79-8548-1f2c156ae6f0", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e8a97909-d807-4f79-8548-1f2c156ae6f0"}, {"name": "Test group 8", "groupId": "3ee851e7-6688-495a-a6f6-737e85a23878", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3ee851e7-6688-495a-a6f6-737e85a23878"}, {"name": "Test group 9", "groupId": "af27d0b1-4378-443f-9a6d-f878848b144a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=af27d0b1-4378-443f-9a6d-f878848b144a"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1697453478823}
 {"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountId": "557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "24x24": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "16x16": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "32x32": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png"}, "displayName": "Automation for Jira", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 3, "items": [{"name": "atlassian-addons-admin", "groupId": "90b9ffb1-ed26-4b5e-af59-8f684900ce83", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1697453479094}
 {"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5d53f3cbc6b9320d9ea5bdc2", "accountId": "5d53f3cbc6b9320d9ea5bdc2", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "24x24": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "16x16": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "32x32": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png"}, "displayName": "Jira Outlook", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 1, "items": [{"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1697453479412}
-{"stream": "workflows", "data": {"id": {"name": "Builds Workflow", "entityId": "Builds Workflow"}, "description": "Builds Workflow", "created": "1969-12-31T16:00:00-08:00", "updated": "1969-12-31T16:00:00-08:00"}, "emitted_at": 1697453494031}
-{"stream": "workflows", "data": {"id": {"name": "classic default workflow", "entityId": "385bb764-dfb6-89a7-2e43-a25bdd0cbaf4"}, "description": "The classic JIRA default workflow", "created": "2020-12-03T23:41:38.951000-08:00", "updated": "2023-06-30T02:33:48.808000-07:00"}, "emitted_at": 1697453494032}
-{"stream": "workflows", "data": {"id": {"name": "jira", "entityId": "jira"}, "description": "The default Jira workflow.", "created": "1969-12-31T16:00:00-08:00", "updated": "1969-12-31T16:00:00-08:00"}, "emitted_at": 1697453494033}
+{"stream": "workflows", "data": {"id": {"name": "Builds Workflow", "entityId": "Builds Workflow"}, "description": "Builds Workflow", "created": "1969-12-31T16:00:00.000-0800", "updated": "1969-12-31T16:00:00.000-0800", "entityId": "Builds Workflow", "name": "Builds Workflow"}, "emitted_at": 1718294831111}
+{"stream": "workflows", "data": {"id": {"name": "classic default workflow", "entityId": "385bb764-dfb6-89a7-2e43-a25bdd0cbaf4"}, "description": "The classic JIRA default workflow", "created": "2020-12-03T23:41:38.951-0800", "updated": "2023-06-30T02:33:48.808-0700", "entityId": "385bb764-dfb6-89a7-2e43-a25bdd0cbaf4", "name": "classic default workflow"}, "emitted_at": 1718294831117}
+{"stream": "workflows", "data": {"id": {"name": "jira", "entityId": "jira"}, "description": "The default Jira workflow.", "created": "1969-12-31T16:00:00.000-0800", "updated": "1969-12-31T16:00:00.000-0800", "entityId": "jira", "name": "jira"}, "emitted_at": 1718294831121}
 {"stream": "workflow_schemes", "data": {"id": 10000, "name": "classic", "description": "classic", "defaultWorkflow": "classic default workflow", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10000"}, "emitted_at": 1697453495072}
 {"stream": "workflow_schemes", "data": {"id": 10001, "name": "IT: Software Simplified Workflow Scheme", "description": "Generated by JIRA Software version 1001.0.0-SNAPSHOT. This workflow scheme is managed internally by Jira Software. Do not manually modify this workflow scheme.", "defaultWorkflow": "Software Simplified Workflow for Project IT", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10001"}, "emitted_at": 1697453495073}
 {"stream": "workflow_schemes", "data": {"id": 10002, "name": "P2: Software Simplified Workflow Scheme", "description": "Generated by JIRA Software version 1001.0.0-SNAPSHOT. This workflow scheme is managed internally by Jira Software. Do not manually modify this workflow scheme.", "defaultWorkflow": "Software Simplified Workflow for Project P2", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10002"}, "emitted_at": 1697453495074}

diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml
@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993
-  dockerImageTag: 2.0.3
+  dockerImageTag: 3.0.0
   dockerRepository: airbyte/source-jira
   documentationUrl: https://docs.airbyte.com/integrations/sources/jira
   githubIssueLabel: source-jira
@@ -30,6 +30,12 @@ data:
   releaseStage: generally_available
   releases:
     breakingChanges:
+      3.0.0:
+        message: "Primary key for Workflows stream has been changed from `Id` to [`entityId`, `name`]. This change of pk makes the stream compatible with more types of destinations."
+        upgradeDeadline: "2024-07-02"
+        scopedImpact:
+          - scopeType: stream
+            impactedScopes: ["workflows"]
       2.0.0:
         message: "The source Jira connector is being migrated from the Python CDK to our declarative low-code CDK. Due to changes to the incremental stream state, this migration constitutes a breaking change. Additionally, an issue in the `ProjectAvatars` stream has been fixed. After updating, please reset your source before resuming syncs. For more information, see our migration documentation for source Jira."
         upgradeDeadline: "2024-05-31"

diff --git a/airbyte-integrations/connectors/source-jira/pyproject.toml b/airbyte-integrations/connectors/source-jira/pyproject.toml
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.0.3"
+version = "3.0.0"
 name = "source-jira"
 description = "Source implementation for Jira."
 authors = [ "Airbyte <[email protected]>",]

diff --git a/airbyte-integrations/connectors/source-jira/source_jira/manifest.yaml b/airbyte-integrations/connectors/source-jira/source_jira/manifest.yaml
@@ -468,10 +468,19 @@ definitions:
   workflows_stream:
     $ref: "#/definitions/full_refresh_stream"
     name: workflows
-    primary_key: "id"
+    primary_key: [["entityId"], ["name"]]
     $parameters:
       path: "workflow/search"
       extract_field: "values"
+    transformations:
+      - type: AddFields
+        fields:
+          - path: ["entityId"]
+            value_type: string
+            value: "{{ record.id.entityId }}"
+          - path: ["name"]
+            value_type: string
+            value: "{{ record.id.name }}"
 
   # https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflow-schemes/#api-rest-api-3-workflowscheme-get
   workflow_schemes_stream:

diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflows.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflows.json
@@ -15,6 +15,14 @@
         }
       }
     },
+    "entityId": {
+      "description": "The unique identifier of the workflow.",
+      "type": ["null", "string"]
+    },
+    "name": {
+      "description": "The unique name the workflow.",
+      "type": ["null", "string"]
+    },
     "description": {
       "description": "The detailed description of the workflow.",
       "type": "string"

diff --git a/docs/integrations/sources/jira-migrations.md b/docs/integrations/sources/jira-migrations.md
@@ -1,5 +1,25 @@
 # Jira Migration Guide
 
+## Upgrading to 3.0.0
+
+This is a breaking change for **Workflows** stream, which used `Id` field as pk.
+This version introduces changing of pk from `Id`(type: object) to composite pk `[entityId, name]`(type: string, string), as it makes stream compatible with destinations that do not support complex primary keys(e.g. BigQuery).
+
+To gracefully handle these changes for your existing connections, we highly recommend resetting your data before resuming your data syncs with the new version. The **Workflows** stream can be manually reset in the following way:
+
+1. Select **Connections** in the main navbar.
+   1. Select the connection(s) affected by the update.
+2. Select the **Schema** tab.
+3. Click **Refresh source schema**, then **Ok**.
+4. Select **Save changes** at the bottom of the page.
+5. Select the **Status** tab and click three dots on the right side of **Workflows**.
+6. Press the **Clear data** button.
+7. Return to the **Schema** tab.
+8. Check all your streams.
+9. Select **Sync now** to sync your data
+
+For more information on resetting your data in Airbyte, see [this page](/operator-guides/clear).
+
 ## Upgrading to 2.0.0
 
 We're continuously striving to enhance the quality and reliability of our connectors at Airbyte. As part of our commitment to delivering exceptional service, we are transitioning source-jira from the Python Connector Development Kit (CDK) to our innovative low-code framework. This is part of a strategic move to streamline many processes across connectors, bolstering maintainability and freeing us to focus more of our efforts on improving the performance and features of our evolving platform and growing catalog. However, due to differences between the Python and low-code CDKs, this migration constitutes a breaking change.

diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md
@@ -127,7 +127,8 @@ The Jira connector should not run into Jira API limitations under normal usage.
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                          |
 |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 2.0.3   | 2024-06-10 | [39347](https://github.com/airbytehq/airbyte/pull/39347)   | Update state handling for incremental Python streams                        |
+| 3.0.0   | 2024-06-14 | [39467](https://github.com/airbytehq/airbyte/pull/39467)   | Update pk for Workflows stream from Id(object) to entityId, name(string, string)                                                                                 |
+| 2.0.3   | 2024-06-10 | [39347](https://github.com/airbytehq/airbyte/pull/39347)   | Update state handling for incremental Python streams                                                                                                             |
 | 2.0.2   | 2024-06-06 | [39310](https://github.com/airbytehq/airbyte/pull/39310)   | Fix projects substreams for deleted projects                                                                                                                     |
 | 2.0.1   | 2024-05-20 | [38341](https://github.com/airbytehq/airbyte/pull/38341)   | Update CDK authenticator package                                                                                                                                 |
 | 2.0.0   | 2024-04-20 | [37374](https://github.com/airbytehq/airbyte/pull/37374)   | Migrate to low-code and fix `Project Avatars` stream                                                                                                             |