From 5dbd8fdd73200238c4457dc1d8cc2eb1112b9e24 Mon Sep 17 00:00:00 2001 From: Aazam Thakur <59562284+aazam-gh@users.noreply.github.com> Date: Wed, 16 Oct 2024 20:09:50 +0530 Subject: [PATCH] source-sigma-computing contribution from aazam-gh (#46869) Co-authored-by: Marcos Marx --- .../source-sigma-computing/README.md | 40 + .../acceptance-test-config.yml | 17 + .../source-sigma-computing/icon.svg | 19 + .../source-sigma-computing/manifest.yaml | 911 ++++++++++++++++++ .../source-sigma-computing/metadata.yaml | 35 + docs/integrations/sources/sigma-computing.md | 43 + 6 files changed, 1065 insertions(+) create mode 100644 airbyte-integrations/connectors/source-sigma-computing/README.md create mode 100644 airbyte-integrations/connectors/source-sigma-computing/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-sigma-computing/icon.svg create mode 100644 airbyte-integrations/connectors/source-sigma-computing/manifest.yaml create mode 100644 airbyte-integrations/connectors/source-sigma-computing/metadata.yaml create mode 100644 docs/integrations/sources/sigma-computing.md diff --git a/airbyte-integrations/connectors/source-sigma-computing/README.md b/airbyte-integrations/connectors/source-sigma-computing/README.md new file mode 100644 index 000000000000..e2df1515e0e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-sigma-computing/README.md @@ -0,0 +1,40 @@ +# Sigma Computing +This directory contains the manifest-only connector for `source-sigma-computing`. + +This is the setup for the Sigma Computing source that ingests data from the sigma API. + +Sigma is next-generation analytics and business intelligence that scales billions of records using spreadsheets, SQL, Python, or AI—without compromising speed and security https://www.sigmacomputing.com/ + +In order to use this source, you must first create an account on Sigma Computing. Go to Account > General Settings and review the Site section for the Cloud provider, this will be used to find the base url of your API. Compare it at https://help.sigmacomputing.com/reference/get-started-sigma-api + +Next, head over to Developer Access and click on create. This will generate your Client ID and Client Secret required by the API. You can learn more about the API here https://help.sigmacomputing.com/reference + + +## Usage +There are multiple ways to use this connector: +- You can use this connector as any other connector in Airbyte Marketplace. +- You can load this connector in `pyairbyte` using `get_source`! +- You can open this connector in Connector Builder, edit it, and publish to your workspaces. + +Please refer to the manifest-only connector documentation for more details. + +## Local Development +We recommend you use the Connector Builder to edit this connector. + +But, if you want to develop this connector locally, you can use the following steps. + +### Environment Setup +You will need `airbyte-ci` installed. You can find the documentation [here](airbyte-ci). + +### Build +This will create a dev image (`source-sigma-computing:dev`) that you can use to test the connector locally. +```bash +airbyte-ci connectors --name=source-sigma-computing build +``` + +### Test +This will run the acceptance tests for the connector. +```bash +airbyte-ci connectors --name=source-sigma-computing test +``` + diff --git a/airbyte-integrations/connectors/source-sigma-computing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sigma-computing/acceptance-test-config.yml new file mode 100644 index 000000000000..62b5560d3d20 --- /dev/null +++ b/airbyte-integrations/connectors/source-sigma-computing/acceptance-test-config.yml @@ -0,0 +1,17 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-sigma-computing:dev +acceptance_tests: + spec: + tests: + - spec_path: "manifest.yaml" + connection: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + discovery: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + basic_read: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + incremental: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + full_refresh: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" diff --git a/airbyte-integrations/connectors/source-sigma-computing/icon.svg b/airbyte-integrations/connectors/source-sigma-computing/icon.svg new file mode 100644 index 000000000000..b8398f83a7fc --- /dev/null +++ b/airbyte-integrations/connectors/source-sigma-computing/icon.svg @@ -0,0 +1,19 @@ + + + + diff --git a/airbyte-integrations/connectors/source-sigma-computing/manifest.yaml b/airbyte-integrations/connectors/source-sigma-computing/manifest.yaml new file mode 100644 index 000000000000..b7587fb4892b --- /dev/null +++ b/airbyte-integrations/connectors/source-sigma-computing/manifest.yaml @@ -0,0 +1,911 @@ +version: 5.12.0 + +type: DeclarativeSource + +description: > + This is the setup for the Sigma Computing source that ingests data from the + sigma API. + + + Sigma is next-generation analytics and business intelligence that scales + billions of records using spreadsheets, SQL, Python, or AI—without + compromising speed and security https://www.sigmacomputing.com/ + + + In order to use this source, you must first create an account on Sigma + Computing. Go to Account > General Settings and review the Site section for + the Cloud provider, this will be used to find the base url of your API. + Compare it at https://help.sigmacomputing.com/reference/get-started-sigma-api + + + Next, head over to Developer Access and click on create. This will generate + your Client ID and Client Secret required by the API. You can learn more about + the API here https://help.sigmacomputing.com/reference + +check: + type: CheckStream + stream_names: + - files + +definitions: + streams: + files: + type: DeclarativeStream + name: files + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /files + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - entries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/files" + connections: + type: DeclarativeStream + name: connections + primary_key: + - connectionId + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /connections + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - entries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/connections" + datasets: + type: DeclarativeStream + name: datasets + primary_key: + - datasetId + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /datasets + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - entries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/datasets" + members: + type: DeclarativeStream + name: members + primary_key: + - memberId + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /members + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - entries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/members" + teams: + type: DeclarativeStream + name: teams + primary_key: + - teamId + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /teams + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - entries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/teams" + templates: + type: DeclarativeStream + name: templates + primary_key: + - templateId + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /templates + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - entries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/templates" + workspaces: + type: DeclarativeStream + name: workspaces + primary_key: + - workspaceId + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /workspaces + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - entries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/workspaces" + workbooks: + type: DeclarativeStream + name: workbooks + primary_key: + - workbookId + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /workbooks + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - entries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/workbooks" + base_requester: + type: HttpRequester + url_base: "{{ config[\"base_url\"] }}/v2" + authenticator: + type: OAuthAuthenticator + client_id: "{{ config[\"client_id\"] }}" + grant_type: refresh_token + client_secret: "{{ config[\"client_secret\"] }}" + refresh_token: "{{ config[\"client_refresh_token\"] }}" + access_token_name: access_token + refresh_request_body: {} + refresh_token_updater: + refresh_token_name: refresh_token + access_token_config_path: + - oauth_access_token + token_expiry_date_config_path: + - oauth_token_expiry_date + refresh_token_config_path: + - client_refresh_token + token_refresh_endpoint: "{{ config[\"base_url\"] }}/v2/auth/token" + +streams: + - $ref: "#/definitions/streams/files" + - $ref: "#/definitions/streams/connections" + - $ref: "#/definitions/streams/datasets" + - $ref: "#/definitions/streams/members" + - $ref: "#/definitions/streams/teams" + - $ref: "#/definitions/streams/templates" + - $ref: "#/definitions/streams/workspaces" + - $ref: "#/definitions/streams/workbooks" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - client_id + - client_secret + - client_refresh_token + - base_url + properties: + client_id: + type: string + order: 0 + title: Client ID + airbyte_secret: true + client_secret: + type: string + order: 1 + title: Client secret + airbyte_secret: true + client_refresh_token: + type: string + order: 2 + title: Refresh token + airbyte_secret: true + oauth_access_token: + type: string + description: >- + The current access token. This field might be overridden by the + connector based on the token refresh endpoint response. + order: 3 + title: Access token + airbyte_secret: true + oauth_token_expiry_date: + type: string + description: >- + The date the current access token expires in. This field might be + overridden by the connector based on the token refresh endpoint + response. + order: 4 + title: Token expiry date + format: date-time + base_url: + type: string + description: The base url of your sigma organization + order: 5 + title: Base URL + additionalProperties: true + +metadata: + autoImportSchema: + files: true + connections: true + datasets: true + members: true + teams: true + templates: true + workspaces: true + workbooks: true + testedStreams: + files: + streamHash: 1d8bd67f61aa7a09da12502adbd85e4424e08fa6 + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + connections: + streamHash: 82a16ebc6d4d67ea0b0e493dbe0d52fe629fa4c6 + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + datasets: + streamHash: d006e2b678deaa9e3f7c85c56dce867047036bdd + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + members: + streamHash: 78fc1e2e11be3328b8185ac70739bdc7674800d0 + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + teams: + streamHash: f7b8788e9bf9a67f498b67caf94ef47a51f06bbf + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + templates: + streamHash: 4927cf0816f57f5e95170455469eed4100002786 + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + workspaces: + streamHash: d063c7b5740c9a12cfc0aca87b0cd0ea1e9318bd + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + workbooks: + hasRecords: true + streamHash: 7ce07f245f2f77e2fd0b1b737d909fa51d5eea6a + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + assist: + docsUrl: https://help.sigmacomputing.com/reference + openapiSpecUrl: https://help.sigmacomputing.com/openapi/65d4ee76379a330060654669 + +schemas: + files: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + createdBy: + type: + - string + - "null" + id: + type: string + isArchived: + type: + - boolean + - "null" + name: + type: + - string + - "null" + ownerId: + type: + - string + - "null" + parentId: + type: + - string + - "null" + parentUrlId: + type: + - string + - "null" + path: + type: + - string + - "null" + permission: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + updatedBy: + type: + - string + - "null" + urlId: + type: + - string + - "null" + required: + - id + connections: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + account: + type: + - string + - "null" + connectionId: + type: string + createdAt: + type: + - string + - "null" + createdBy: + type: + - string + - "null" + friendlyName: + type: + - boolean + - "null" + host: + type: + - string + - "null" + isArchived: + type: + - boolean + - "null" + isAuditLog: + type: + - boolean + - "null" + isSample: + type: + - boolean + - "null" + lastActiveAt: + type: + - string + - "null" + name: + type: + - string + - "null" + organizationId: + type: + - string + - "null" + poolSizes: + type: + - object + - "null" + properties: + adhocPoolSize: + type: + - number + - "null" + catalogPoolSize: + type: + - number + - "null" + isPoolConfigured: + type: + - boolean + - "null" + resultPoolSize: + type: + - number + - "null" + schedulePoolSize: + type: + - number + - "null" + role: + type: + - string + - "null" + timeout: + type: + - object + - "null" + properties: + default: + type: + - number + - "null" + updatedAt: + type: + - string + - "null" + updatedBy: + type: + - string + - "null" + useOauth: + type: + - boolean + - "null" + user: + type: + - string + - "null" + warehouse: + type: + - string + - "null" + writeAccess: + type: + - boolean + - "null" + writebacks: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + database: + type: + - string + - "null" + schema: + type: + - string + - "null" + required: + - connectionId + datasets: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + createdBy: + type: + - string + - "null" + datasetId: + type: string + name: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + updatedBy: + type: + - string + - "null" + url: + type: + - string + - "null" + required: + - datasetId + members: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + createdBy: + type: + - string + - "null" + email: + type: + - string + - "null" + firstName: + type: + - string + - "null" + homeFolderId: + type: + - string + - "null" + lastName: + type: + - string + - "null" + memberId: + type: string + memberType: + type: + - string + - "null" + organizationId: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + updatedBy: + type: + - string + - "null" + userKind: + type: + - string + - "null" + required: + - memberId + teams: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - string + - "null" + createdAt: + type: + - string + - "null" + createdBy: + type: + - string + - "null" + name: + type: + - string + - "null" + teamId: + type: string + updatedAt: + type: + - string + - "null" + updatedBy: + type: + - string + - "null" + visibility: + type: + - string + - "null" + required: + - teamId + templates: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + createdBy: + type: + - string + - "null" + latestVersion: + type: + - number + - "null" + name: + type: + - string + - "null" + path: + type: + - string + - "null" + templateId: + type: string + templateUrlId: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + updatedBy: + type: + - string + - "null" + url: + type: + - string + - "null" + required: + - templateId + workspaces: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + createdBy: + type: + - string + - "null" + name: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + updatedBy: + type: + - string + - "null" + workspaceId: + type: + - string + - "null" + workbooks: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + createdAt: + type: + - string + - "null" + createdBy: + type: + - string + - "null" + latestVersion: + type: + - number + - "null" + name: + type: + - string + - "null" + ownerId: + type: + - string + - "null" + path: + type: + - string + - "null" + updatedAt: + type: + - string + - "null" + updatedBy: + type: + - string + - "null" + url: + type: + - string + - "null" + workbookId: + type: string + workbookUrlId: + type: + - string + - "null" + required: + - workbookId diff --git a/airbyte-integrations/connectors/source-sigma-computing/metadata.yaml b/airbyte-integrations/connectors/source-sigma-computing/metadata.yaml new file mode 100644 index 000000000000..95ba91a2e69f --- /dev/null +++ b/airbyte-integrations/connectors/source-sigma-computing/metadata.yaml @@ -0,0 +1,35 @@ +metadataSpecVersion: "1.0" +data: + allowedHosts: + hosts: + - "*" + registryOverrides: + oss: + enabled: true + cloud: + enabled: true + remoteRegistries: + pypi: + enabled: false + packageName: airbyte-source-sigma-computing + connectorBuildOptions: + baseImage: docker.io/airbyte/source-declarative-manifest:5.13.0@sha256:ffc5977f59e1f38bf3f5dd70b6fa0520c2450ebf85153c5a8df315b8c918d5c3 + connectorSubtype: api + connectorType: source + definitionId: 40fed53b-3a55-4ce3-a25f-93af5b5379b0 + dockerImageTag: 0.0.1 + dockerRepository: airbyte/source-sigma-computing + githubIssueLabel: source-sigma-computing + icon: icon.svg + license: MIT + name: Sigma Computing + releaseDate: 2024-10-13 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/sigma-computing + tags: + - language:manifest-only + - cdk:low-code + ab_internal: + ql: 100 + sl: 100 diff --git a/docs/integrations/sources/sigma-computing.md b/docs/integrations/sources/sigma-computing.md new file mode 100644 index 000000000000..2496d231c6a5 --- /dev/null +++ b/docs/integrations/sources/sigma-computing.md @@ -0,0 +1,43 @@ +# Sigma Computing +This is the setup for the Sigma Computing source that ingests data from the sigma API. + +Sigma is next-generation analytics and business intelligence that scales billions of records using spreadsheets, SQL, Python, or AI—without compromising speed and security https://www.sigmacomputing.com/ + +In order to use this source, you must first create an account on Sigma Computing. Go to Account General Settings and review the Site section for the Cloud provider, this will be used to find the base url of your API. Compare it at https://help.sigmacomputing.com/reference/get-started-sigma-api + +Next, head over to Developer Access and click on create. This will generate your Client ID and Client Secret required by the API. You can learn more about the API here https://help.sigmacomputing.com/reference + + +## Configuration + +| Input | Type | Description | Default Value | +|-------|------|-------------|---------------| +| `client_id` | `string` | Client ID. | | +| `client_secret` | `string` | Client secret. | | +| `client_refresh_token` | `string` | Refresh token. | | +| `oauth_access_token` | `string` | Access token. The current access token. This field might be overridden by the connector based on the token refresh endpoint response. | | +| `oauth_token_expiry_date` | `string` | Token expiry date. The date the current access token expires in. This field might be overridden by the connector based on the token refresh endpoint response. | | +| `base_url` | `string` | Base URL. The base url of your sigma organization | | + +## Streams +| Stream Name | Primary Key | Pagination | Supports Full Sync | Supports Incremental | +|-------------|-------------|------------|---------------------|----------------------| +| files | id | DefaultPaginator | ✅ | ❌ | +| connections | connectionId | DefaultPaginator | ✅ | ❌ | +| datasets | datasetId | DefaultPaginator | ✅ | ❌ | +| members | memberId | DefaultPaginator | ✅ | ❌ | +| teams | teamId | DefaultPaginator | ✅ | ❌ | +| templates | templateId | DefaultPaginator | ✅ | ❌ | +| workspaces | workspaceId | DefaultPaginator | ✅ | ❌ | +| workbooks | workbookId | DefaultPaginator | ✅ | ❌ | + +## Changelog + +
+ Expand to review + +| Version | Date | Pull Request | Subject | +|------------------|-------------------|--------------|----------------| +| 0.0.1 | 2024-10-13 | | Initial release by [@aazam-gh](https://github.com/aazam-gh) via Connector Builder | + +