Skip to content

Commit

Permalink
fixed vars
Browse files Browse the repository at this point in the history
  • Loading branch information
aazam-gh committed Aug 25, 2024
1 parent f43b934 commit a63a872
Show file tree
Hide file tree
Showing 15 changed files with 301 additions and 84 deletions.
30 changes: 11 additions & 19 deletions connectors/source_github/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ packages:

After you can run `dbt tests` or `dbt docs generate` to have a preview of Airbyte output data.

### Fivetran github Modeling dbt package
### Fivetran Github Modeling dbt package

This package transforms Airbyte connector output data, making it compatible with Fivetran's github dbt package. You can check the analytical models Fivetran creates [here](https://github.com/fivetran/dbt_github/tree/main?tab=readme-ov-file#-what-does-this-dbt-package-do). The link also provides information about how the package works and what is configurable.
This package transforms Airbyte connector output data, making it compatible with Fivetran's Github dbt package. You can check the analytical models Fivetran creates [here](https://github.com/fivetran/dbt_github/tree/main?tab=readme-ov-file#-what-does-this-dbt-package-do). The link also provides information about how the package works and what is configurable.

Create the require files to use Airbyte and Fivetran dbt packages:

Expand Down Expand Up @@ -71,23 +71,15 @@ vars:
github_database: "airbyte_db_default"
github_schema: "airbyte_dbt_github_support"
using_schedules: False
using_domain_names: False
using_user_tags: False
using_ticket_form_history: False
using_organization_tags: False
github_organization_identifier: "organizations"
github_ticket_identifier: "tickets"
github_ticket_comment_identifier: "ticket_comments"
github_ticket_tag_identifier: "tags"
github_ticket_field_history_identifier: "ticket_field_history"
github_ticket_form_history_identifier: "ticket_forms"
github_brand_identifier: "brands"
github_group_identifier: "groups"
github_organization_tag_identifier: "organization_fields"
github_user_identifier: "users"
github_user_tag_identifier: "user_field"
using_repo_team: False
github_issue_assignee_identifier: "issue_assignee"
github_issue_closed_history_identifier: "issue_closed_history"
github_issue_merged_identifier: "issue_merged"
github_pull_request_review_identifier: "pull_request_review"
github_repo_team_identifier: "repo_team"
github_requested_reviewer_history_identifier: "requested_reviewer_history"
```

After run `dbt run`, you can see the models being created.
Expand Down
5 changes: 3 additions & 2 deletions connectors/source_github/dbt_project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,6 @@ models:
materialized: view

vars:
database: snowflake
schema: source_github
database: airbyte_dbt_default
schema: airbyte_schema_default
using_fivetran: False
25 changes: 8 additions & 17 deletions connectors/source_github/integration_tests/dbt_project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,20 +42,11 @@ vars:
github_database: "airbyte_db_default"
github_schema: "airbyte_dbt_github"

using_schedules: False
using_domain_names: False
using_user_tags: False
using_ticket_form_history: False
using_organization_tags: False

zendesk_organization_identifier: "organizations"
zendesk_ticket_identifier: "tickets"
zendesk_ticket_comment_identifier: "ticket_comments"
zendesk_ticket_tag_identifier: "tags"
zendesk_ticket_field_history_identifier: "ticket_field_history"
zendesk_ticket_form_history_identifier: "ticket_forms"
zendesk_brand_identifier: "brands"
zendesk_group_identifier: "groups"
zendesk_organization_tag_identifier: "organization_fields"
zendesk_user_identifier: "users"
zendesk_user_tag_identifier: "user_field"
using_repo_team: False

github_issue_assignee_identifier: "issue_assignee"
github_issue_closed_history_identifier: "issue_closed_history"
github_issue_merged_identifier: "issue_merged"
github_pull_request_review_identifier: "pull_request_review"
github_repo_team_identifier: "repo_team"
github_requested_reviewer_history_identifier: "requested_reviewer_history"

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{% if target.type == "snowflake" %}

with tmp as (
SELECT
issue_id,
user_id
FROM
{{ source('source_github', 'issue_assignee') }}
)
select * from tmp

{% elif target.type == "bigquery" %}

WITH tmp AS (
SELECT
issue_id,
user_id
FROM
{{ source('source_github', 'issue_assignee') }}
)
SELECT * FROM tmp

{% elif target.type == "postgres" %}

WITH tmp AS (
SELECT
issue_id,
user_id
FROM
{{ source('source_github', 'issue_assignee') }}
)
SELECT * FROM tmp

{%endif%}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
version: 2

models:
- name: issue_assignee
schema: "{{ var('airbyte_schema', target.schema) }}"
database: "{{ var('airbyte_database', target.database) }}"
identifier: "{{ var('github_issue_assignee_identifier', 'issue_assignee') }}"
description: All fields and field values associated with issue assignees.
config:
+enabled: "{{ var('using_fivetran_model', False) }}"
columns:
- name: issue_id
description: The ID of the issue associated with the field.
- name: user_id
description: The ID of the user assigned to the issue.
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{% if target.type == "snowflake" %}

with tmp as (
SELECT
issue_id,
updated_at,
is_closed
FROM
{{ source('source_github', 'issue_closed_history') }}
)
select * from tmp

{% elif target.type == "bigquery" %}

WITH tmp AS (
SELECT
issue_id,
updated_at,
is_closed
FROM
{{ source('source_github', 'issue_closed_history') }}
)
SELECT * FROM tmp

{% elif target.type == "postgres" %}

WITH tmp AS (
SELECT
issue_id,
updated_at,
is_closed
FROM
{{ source('source_github', 'issue_closed_hsitory') }}
)
SELECT * FROM tmp

{%endif%}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
version: 2

models:
- name: issue_closed_history
schema: "{{ var('airbyte_schema', target.schema) }}"
database: "{{ var('airbyte_database', target.database) }}"
identifier: "{{ var('github_issue_closed_history_identifier', 'issue_closed_history') }}"
description: All fields and field values associated with issues with closed history.
config:
+enabled: "{{ var('using_fivetran_model', False) }}"
columns:
- name: issue_id
description: The ID of the issue associated with the field.
- name: updated_at
description: The timestamp for when the issue was last modified.
- name: is_closed
description: The boolean for whether the issue is closed or not.
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{% if target.type == "snowflake" %}

with tmp as (
SELECT
issue_id,
merged_at
FROM
{{ source('source_github', 'issue_merged') }}
)
select * from tmp

{% elif target.type == "bigquery" %}

WITH tmp AS (
SELECT
issue_id,
merged_at
FROM
{{ source('source_github', 'issue_merged') }}
)
SELECT * FROM tmp

{% elif target.type == "postgres" %}

WITH tmp AS (
SELECT
issue_id,
merged_at
FROM
{{ source('source_github', 'issue_closed_hsitory') }}
)
SELECT * FROM tmp

{%endif%}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
version: 2

models:
- name: issue_merged
schema: "{{ var('airbyte_schema', target.schema) }}"
database: "{{ var('airbyte_database', target.database) }}"
identifier: "{{ var('github_issue_merged_identifier', 'issue_merged') }}"
description: All fields and field values associated with issues with issues merged.
config:
+enabled: "{{ var('using_fivetran_model', False) }}"
columns:
- name: issue_id
description: The ID of the issue associated with the field.
- name: merged_at
description: The timestamp for when the issue was merged.
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
{% if target.type == "snowflake" %}

with tmp as (
SELECT
id as pull_request_review_id,
pull_request_id,
submitted_at,
state,
user_id
FROM
{{ source('source_github', 'pull_request_review') }}
)
select * from tmp

{% elif target.type == "bigquery" %}

WITH tmp AS (
SELECT
id as pull_request_review_id,
pull_request_id,
submitted_at,
state,
user_id
FROM
{{ source('source_github', 'pull_request_review') }}
)
SELECT * FROM tmp

{% elif target.type == "postgres" %}

WITH tmp AS (
SELECT
id as pull_request_review_id,
pull_request_id,
submitted_at,
state,
user_id
FROM
{{ source('source_github', 'pull_request_review') }}
)
SELECT * FROM tmp
{%endif%}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
version: 2

models:
- name: pull_request_review
schema: "{{ var('airbyte_schema', target.schema) }}"
database: "{{ var('airbyte_database', target.database) }}"
identifier: "{{ var('github_pull_request_review_identifier', 'pull_request_review') }}"
description: All fields and field values associated with pull request reviews.
config:
+enabled: "{{ var('using_fivetran_model', False) }}"
columns:
- name: pull_request_review_id
description: The ID of the issue associated with the field.
- name: pull_request_id
description: The ID of the issue associated with the field.
- name: submitted_at
description: The timestamp for when the issue was last submitted.
- name: state
description: The boolean for whether the pull request is reviewed or not.
- name: user_id
description: The ID of the issue associated with the field.
33 changes: 33 additions & 0 deletions connectors/source_github/models/fivetran_converter/repo_team.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{% if target.type == "snowflake" %}

with tmp as (
SELECT
repository_id,
team_id
FROM
{{ source('source_github', 'repo_team') }}
)
select * from tmp

{% elif target.type == "bigquery" %}

WITH tmp AS (
SELECT
repository_id,
team_id
FROM
{{ source('source_github', 'repo_team') }}
)
SELECT * FROM tmp

{% elif target.type == "postgres" %}

WITH tmp AS (
SELECT
repository_id,
team_id
FROM
{{ source('source_github', 'repo_team') }}
)
SELECT * FROM tmp
{%endif%}
15 changes: 15 additions & 0 deletions connectors/source_github/models/fivetran_converter/repo_team.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
version: 2

models:
- name: repo_team
schema: "{{ var('airbyte_schema', target.schema) }}"
database: "{{ var('airbyte_database', target.database) }}"
identifier: "{{ var('github_repo_team_identifier', 'repo_team') }}"
description: All fields and field values associated with repo teams.
config:
+enabled: "{{ var('using_fivetran_model', False) }}"
columns:
- name: repository_id
description: The ID of the repo associated with the field.
- name: team_id
description: The ID of the team associated with the field.
Loading

0 comments on commit a63a872

Please sign in to comment.