From a63a87257d828091f6056748c78a0f97f8e8339d Mon Sep 17 00:00:00 2001 From: aazam Date: Sun, 25 Aug 2024 15:10:52 +0530 Subject: [PATCH] fixed vars --- connectors/source_github/README.md | 30 +++++-------- connectors/source_github/dbt_project.yml | 5 ++- .../integration_tests/dbt_project.yml | 25 ++++------- .../fivetran_converter/github_issue_labels | 21 ---------- .../fivetran_converter/issue_assignee.sql | 34 +++++++++++++++ .../fivetran_converter/issue_assignee.yml | 15 +++++++ .../issue_closed_history.sql | 37 ++++++++++++++++ .../issue_closed_history.yml | 17 ++++++++ .../fivetran_converter/issue_merged.sql | 34 +++++++++++++++ .../fivetran_converter/issue_merged.yml | 15 +++++++ .../pull_request_review.sql | 42 +++++++++++++++++++ .../pull_request_review.yml | 21 ++++++++++ .../models/fivetran_converter/repo_team.sql | 33 +++++++++++++++ .../models/fivetran_converter/repo_team.yml | 15 +++++++ .../requested_reviewer_history.sql | 41 +++++++----------- 15 files changed, 301 insertions(+), 84 deletions(-) delete mode 100644 connectors/source_github/models/fivetran_converter/github_issue_labels create mode 100644 connectors/source_github/models/fivetran_converter/issue_assignee.sql create mode 100644 connectors/source_github/models/fivetran_converter/issue_assignee.yml create mode 100644 connectors/source_github/models/fivetran_converter/issue_closed_history.sql create mode 100644 connectors/source_github/models/fivetran_converter/issue_closed_history.yml create mode 100644 connectors/source_github/models/fivetran_converter/issue_merged.sql create mode 100644 connectors/source_github/models/fivetran_converter/issue_merged.yml create mode 100644 connectors/source_github/models/fivetran_converter/pull_request_review.sql create mode 100644 connectors/source_github/models/fivetran_converter/pull_request_review.yml create mode 100644 connectors/source_github/models/fivetran_converter/repo_team.sql create mode 100644 connectors/source_github/models/fivetran_converter/repo_team.yml diff --git a/connectors/source_github/README.md b/connectors/source_github/README.md index ccf4be13..47aae39f 100644 --- a/connectors/source_github/README.md +++ b/connectors/source_github/README.md @@ -37,9 +37,9 @@ packages: After you can run `dbt tests` or `dbt docs generate` to have a preview of Airbyte output data. -### Fivetran github Modeling dbt package +### Fivetran Github Modeling dbt package -This package transforms Airbyte connector output data, making it compatible with Fivetran's github dbt package. You can check the analytical models Fivetran creates [here](https://github.com/fivetran/dbt_github/tree/main?tab=readme-ov-file#-what-does-this-dbt-package-do). The link also provides information about how the package works and what is configurable. +This package transforms Airbyte connector output data, making it compatible with Fivetran's Github dbt package. You can check the analytical models Fivetran creates [here](https://github.com/fivetran/dbt_github/tree/main?tab=readme-ov-file#-what-does-this-dbt-package-do). The link also provides information about how the package works and what is configurable. Create the require files to use Airbyte and Fivetran dbt packages: @@ -71,23 +71,15 @@ vars: github_database: "airbyte_db_default" github_schema: "airbyte_dbt_github_support" - using_schedules: False - using_domain_names: False - using_user_tags: False - using_ticket_form_history: False - using_organization_tags: False - - github_organization_identifier: "organizations" - github_ticket_identifier: "tickets" - github_ticket_comment_identifier: "ticket_comments" - github_ticket_tag_identifier: "tags" - github_ticket_field_history_identifier: "ticket_field_history" - github_ticket_form_history_identifier: "ticket_forms" - github_brand_identifier: "brands" - github_group_identifier: "groups" - github_organization_tag_identifier: "organization_fields" - github_user_identifier: "users" - github_user_tag_identifier: "user_field" + using_repo_team: False + + github_issue_assignee_identifier: "issue_assignee" + github_issue_closed_history_identifier: "issue_closed_history" + github_issue_merged_identifier: "issue_merged" + github_pull_request_review_identifier: "pull_request_review" + github_repo_team_identifier: "repo_team" + github_requested_reviewer_history_identifier: "requested_reviewer_history" + ``` After run `dbt run`, you can see the models being created. diff --git a/connectors/source_github/dbt_project.yml b/connectors/source_github/dbt_project.yml index b470c08b..1ff012dc 100644 --- a/connectors/source_github/dbt_project.yml +++ b/connectors/source_github/dbt_project.yml @@ -30,5 +30,6 @@ models: materialized: view vars: - database: snowflake - schema: source_github + database: airbyte_dbt_default + schema: airbyte_schema_default + using_fivetran: False diff --git a/connectors/source_github/integration_tests/dbt_project.yml b/connectors/source_github/integration_tests/dbt_project.yml index 00537c1d..789f7f89 100644 --- a/connectors/source_github/integration_tests/dbt_project.yml +++ b/connectors/source_github/integration_tests/dbt_project.yml @@ -42,20 +42,11 @@ vars: github_database: "airbyte_db_default" github_schema: "airbyte_dbt_github" - using_schedules: False - using_domain_names: False - using_user_tags: False - using_ticket_form_history: False - using_organization_tags: False - - zendesk_organization_identifier: "organizations" - zendesk_ticket_identifier: "tickets" - zendesk_ticket_comment_identifier: "ticket_comments" - zendesk_ticket_tag_identifier: "tags" - zendesk_ticket_field_history_identifier: "ticket_field_history" - zendesk_ticket_form_history_identifier: "ticket_forms" - zendesk_brand_identifier: "brands" - zendesk_group_identifier: "groups" - zendesk_organization_tag_identifier: "organization_fields" - zendesk_user_identifier: "users" - zendesk_user_tag_identifier: "user_field" \ No newline at end of file + using_repo_team: False + + github_issue_assignee_identifier: "issue_assignee" + github_issue_closed_history_identifier: "issue_closed_history" + github_issue_merged_identifier: "issue_merged" + github_pull_request_review_identifier: "pull_request_review" + github_repo_team_identifier: "repo_team" + github_requested_reviewer_history_identifier: "requested_reviewer_history" diff --git a/connectors/source_github/models/fivetran_converter/github_issue_labels b/connectors/source_github/models/fivetran_converter/github_issue_labels deleted file mode 100644 index bd0bbf5f..00000000 --- a/connectors/source_github/models/fivetran_converter/github_issue_labels +++ /dev/null @@ -1,21 +0,0 @@ -version: 2 - -models: - - name: github_issue_labels - schema: "{{ var('airbyte_schema', target.schema) }}" - database: "{{ var('airbyte_database', target.database) }}" - identifier: "{{ var('zendesk_ticket_field_history_identifier', 'ticket_field_history') }}" - description: All fields and field values associated with tickets. - config: - +enabled: "{{ var('using_fivetran_model', False) }}" - columns: - - name: ticket_id - description: The ID of the ticket associated with the field - - name: field_name - description: The name of the ticket field - - name: updated - description: The time the ticket field value was created - - name: value - description: The value of the field - - name: user_id - description: The id of the user who made the update \ No newline at end of file diff --git a/connectors/source_github/models/fivetran_converter/issue_assignee.sql b/connectors/source_github/models/fivetran_converter/issue_assignee.sql new file mode 100644 index 00000000..e96d5dca --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/issue_assignee.sql @@ -0,0 +1,34 @@ +{% if target.type == "snowflake" %} + + with tmp as ( + SELECT + issue_id, + user_id + FROM + {{ source('source_github', 'issue_assignee') }} + ) + select * from tmp + +{% elif target.type == "bigquery" %} + + WITH tmp AS ( + SELECT + issue_id, + user_id + FROM + {{ source('source_github', 'issue_assignee') }} + ) + SELECT * FROM tmp + +{% elif target.type == "postgres" %} + + WITH tmp AS ( + SELECT + issue_id, + user_id + FROM + {{ source('source_github', 'issue_assignee') }} + ) + SELECT * FROM tmp + +{%endif%} \ No newline at end of file diff --git a/connectors/source_github/models/fivetran_converter/issue_assignee.yml b/connectors/source_github/models/fivetran_converter/issue_assignee.yml new file mode 100644 index 00000000..62c3acec --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/issue_assignee.yml @@ -0,0 +1,15 @@ +version: 2 + +models: + - name: issue_assignee + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('github_issue_assignee_identifier', 'issue_assignee') }}" + description: All fields and field values associated with issue assignees. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: issue_id + description: The ID of the issue associated with the field. + - name: user_id + description: The ID of the user assigned to the issue. diff --git a/connectors/source_github/models/fivetran_converter/issue_closed_history.sql b/connectors/source_github/models/fivetran_converter/issue_closed_history.sql new file mode 100644 index 00000000..cbda14ed --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/issue_closed_history.sql @@ -0,0 +1,37 @@ +{% if target.type == "snowflake" %} + + with tmp as ( + SELECT + issue_id, + updated_at, + is_closed + FROM + {{ source('source_github', 'issue_closed_history') }} + ) + select * from tmp + +{% elif target.type == "bigquery" %} + + WITH tmp AS ( + SELECT + issue_id, + updated_at, + is_closed + FROM + {{ source('source_github', 'issue_closed_history') }} + ) + SELECT * FROM tmp + +{% elif target.type == "postgres" %} + + WITH tmp AS ( + SELECT + issue_id, + updated_at, + is_closed + FROM + {{ source('source_github', 'issue_closed_hsitory') }} + ) + SELECT * FROM tmp + +{%endif%} \ No newline at end of file diff --git a/connectors/source_github/models/fivetran_converter/issue_closed_history.yml b/connectors/source_github/models/fivetran_converter/issue_closed_history.yml new file mode 100644 index 00000000..7edc28a5 --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/issue_closed_history.yml @@ -0,0 +1,17 @@ +version: 2 + +models: + - name: issue_closed_history + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('github_issue_closed_history_identifier', 'issue_closed_history') }}" + description: All fields and field values associated with issues with closed history. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: issue_id + description: The ID of the issue associated with the field. + - name: updated_at + description: The timestamp for when the issue was last modified. + - name: is_closed + description: The boolean for whether the issue is closed or not. diff --git a/connectors/source_github/models/fivetran_converter/issue_merged.sql b/connectors/source_github/models/fivetran_converter/issue_merged.sql new file mode 100644 index 00000000..82ce87f8 --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/issue_merged.sql @@ -0,0 +1,34 @@ +{% if target.type == "snowflake" %} + + with tmp as ( + SELECT + issue_id, + merged_at + FROM + {{ source('source_github', 'issue_merged') }} + ) + select * from tmp + +{% elif target.type == "bigquery" %} + + WITH tmp AS ( + SELECT + issue_id, + merged_at + FROM + {{ source('source_github', 'issue_merged') }} + ) + SELECT * FROM tmp + +{% elif target.type == "postgres" %} + + WITH tmp AS ( + SELECT + issue_id, + merged_at + FROM + {{ source('source_github', 'issue_closed_hsitory') }} + ) + SELECT * FROM tmp + +{%endif%} \ No newline at end of file diff --git a/connectors/source_github/models/fivetran_converter/issue_merged.yml b/connectors/source_github/models/fivetran_converter/issue_merged.yml new file mode 100644 index 00000000..7258157b --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/issue_merged.yml @@ -0,0 +1,15 @@ +version: 2 + +models: + - name: issue_merged + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('github_issue_merged_identifier', 'issue_merged') }}" + description: All fields and field values associated with issues with issues merged. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: issue_id + description: The ID of the issue associated with the field. + - name: merged_at + description: The timestamp for when the issue was merged. diff --git a/connectors/source_github/models/fivetran_converter/pull_request_review.sql b/connectors/source_github/models/fivetran_converter/pull_request_review.sql new file mode 100644 index 00000000..5e0b7cdb --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/pull_request_review.sql @@ -0,0 +1,42 @@ +{% if target.type == "snowflake" %} + + with tmp as ( + SELECT + id as pull_request_review_id, + pull_request_id, + submitted_at, + state, + user_id + FROM + {{ source('source_github', 'pull_request_review') }} + ) + select * from tmp + +{% elif target.type == "bigquery" %} + + WITH tmp AS ( + SELECT + id as pull_request_review_id, + pull_request_id, + submitted_at, + state, + user_id + FROM + {{ source('source_github', 'pull_request_review') }} + ) + SELECT * FROM tmp + +{% elif target.type == "postgres" %} + + WITH tmp AS ( + SELECT + id as pull_request_review_id, + pull_request_id, + submitted_at, + state, + user_id + FROM + {{ source('source_github', 'pull_request_review') }} + ) + SELECT * FROM tmp +{%endif%} \ No newline at end of file diff --git a/connectors/source_github/models/fivetran_converter/pull_request_review.yml b/connectors/source_github/models/fivetran_converter/pull_request_review.yml new file mode 100644 index 00000000..b7f8555b --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/pull_request_review.yml @@ -0,0 +1,21 @@ +version: 2 + +models: + - name: pull_request_review + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('github_pull_request_review_identifier', 'pull_request_review') }}" + description: All fields and field values associated with pull request reviews. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: pull_request_review_id + description: The ID of the issue associated with the field. + - name: pull_request_id + description: The ID of the issue associated with the field. + - name: submitted_at + description: The timestamp for when the issue was last submitted. + - name: state + description: The boolean for whether the pull request is reviewed or not. + - name: user_id + description: The ID of the issue associated with the field. diff --git a/connectors/source_github/models/fivetran_converter/repo_team.sql b/connectors/source_github/models/fivetran_converter/repo_team.sql new file mode 100644 index 00000000..b570bd26 --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/repo_team.sql @@ -0,0 +1,33 @@ +{% if target.type == "snowflake" %} + + with tmp as ( + SELECT + repository_id, + team_id + FROM + {{ source('source_github', 'repo_team') }} + ) + select * from tmp + +{% elif target.type == "bigquery" %} + + WITH tmp AS ( + SELECT + repository_id, + team_id + FROM + {{ source('source_github', 'repo_team') }} + ) + SELECT * FROM tmp + +{% elif target.type == "postgres" %} + + WITH tmp AS ( + SELECT + repository_id, + team_id + FROM + {{ source('source_github', 'repo_team') }} + ) + SELECT * FROM tmp +{%endif%} \ No newline at end of file diff --git a/connectors/source_github/models/fivetran_converter/repo_team.yml b/connectors/source_github/models/fivetran_converter/repo_team.yml new file mode 100644 index 00000000..5b59604b --- /dev/null +++ b/connectors/source_github/models/fivetran_converter/repo_team.yml @@ -0,0 +1,15 @@ +version: 2 + +models: + - name: repo_team + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('github_repo_team_identifier', 'repo_team') }}" + description: All fields and field values associated with repo teams. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: repository_id + description: The ID of the repo associated with the field. + - name: team_id + description: The ID of the team associated with the field. diff --git a/connectors/source_github/models/fivetran_converter/requested_reviewer_history.sql b/connectors/source_github/models/fivetran_converter/requested_reviewer_history.sql index 8ff9b80e..ae921c1e 100644 --- a/connectors/source_github/models/fivetran_converter/requested_reviewer_history.sql +++ b/connectors/source_github/models/fivetran_converter/requested_reviewer_history.sql @@ -2,47 +2,38 @@ with tmp as ( SELECT - t.id as TICKET_ID, - f.value:field_name::STRING AS FIELD_NAME, - f.value:value::STRING AS VALUE, - t.author_id as AUTHOR_ID, - t.created_at as UPDATED + pull_request_id, + created_at, + requested_id, + removed FROM - {{ source('source_zendesk_support', 'ticket_audits') }} t, - LATERAL FLATTEN(input => t.events) f + {{ source('source_github', 'requested_review_history') }} ) - select * from tmp where field_name is not null + select * from tmp {% elif target.type == "bigquery" %} WITH tmp AS ( SELECT - t.id AS TICKET_ID, - JSON_EXTRACT_SCALAR(f, '$.field_name') AS FIELD_NAME, - JSON_EXTRACT_SCALAR(f, '$.value') AS VALUE, - t.author_id AS AUTHOR_ID, - t.created_at AS UPDATED + pull_request_id, + created_at, + requested_id, + removed FROM - {{ source('source_zendesk_support', 'ticket_audits') }} t, - UNNEST(JSON_EXTRACT_ARRAY(t.events)) AS f + {{ source('source_github', 'requested_review_history') }} ) SELECT * FROM tmp - WHERE FIELD_NAME IS NOT NULL {% elif target.type == "postgres" %} WITH tmp AS ( SELECT - t.id AS ticket_id, - f.value->>'field_name' AS field_name, - f.value->>'value' AS value, - t.author_id AS author_id, - t.created_at AS updated + pull_request_id, + created_at, + requested_id, + removed FROM - {{ source('source_zendesk_support', 'ticket_audits') }} t, - LATERAL jsonb_array_elements(t.events::jsonb) AS f(value) + {{ source('source_github', 'requested_review_history') }} ) SELECT * FROM tmp - WHERE FIELD_NAME IS NOT NULL - {%endif%} \ No newline at end of file