diff --git a/airbyte.yaml b/airbyte.yaml
index cb96de072..8ab0923e6 100644
--- a/airbyte.yaml
+++ b/airbyte.yaml
@@ -10,6 +10,8 @@ paths:
/jobs:
get:
tags:
+ - "public_jobs"
+ - "public"
- "Jobs"
parameters:
- name: "connectionId"
@@ -66,7 +68,7 @@ paths:
format: "date-time"
in: "query"
required: false
- example: "2023-06-22T16:15:00Z"
+ example: 1687450500000
- name: "createdAtEnd"
description: "The end date to filter by"
schema:
@@ -74,13 +76,13 @@ paths:
format: "date-time"
in: "query"
required: false
- example: "2023-06-22T16:15:00Z"
+ example: 1687450500000
- name: "updatedAtStart"
description: "The start date to filter by"
schema:
type: "string"
format: "date-time"
- example: "2023-06-22T16:15:00Z"
+ example: 1687450500000
in: "query"
required: false
- name: "updatedAtEnd"
@@ -90,10 +92,9 @@ paths:
format: "date-time"
in: "query"
required: false
- example: "2023-06-22T16:15:00Z"
+ example: 1687450500000
- name: "orderBy"
- description: "The field and method to use for ordering. Currently allowed\
- \ are createdAt and updatedAt."
+ description: "The field and method to use for ordering"
schema:
type: "string"
pattern: "\\w+|(ASC|DESC)"
@@ -120,7 +121,7 @@ paths:
description: "Not allowed"
operationId: "listJobs"
summary: "List Jobs by sync type"
- x-use-speakeasy-middleware: true
+ x-speakeasy-alias: "listJobs"
post:
requestBody:
content:
@@ -134,6 +135,8 @@ paths:
jobType: "sync"
required: true
tags:
+ - "public_jobs"
+ - "public"
- "Jobs"
responses:
"200":
@@ -155,10 +158,12 @@ paths:
description: "Not allowed"
operationId: "createJob"
summary: "Trigger a sync or reset job of a connection"
- x-use-speakeasy-middleware: true
+ x-speakeasy-alias: "createJob"
/jobs/{jobId}:
get:
tags:
+ - "public_jobs"
+ - "public"
- "Jobs"
responses:
"200":
@@ -178,10 +183,12 @@ paths:
"404":
description: "Not found"
operationId: "getJob"
+ x-speakeasy-alias: "getJob"
summary: "Get Job status and details"
- x-use-speakeasy-middleware: true
delete:
tags:
+ - "public_jobs"
+ - "public"
- "Jobs"
responses:
"200":
@@ -195,8 +202,8 @@ paths:
"404":
description: "Not found"
operationId: "cancelJob"
+ x-speakeasy-alias: "cancelJob"
summary: "Cancel a running Job"
- x-use-speakeasy-middleware: true
parameters:
- name: "jobId"
schema:
@@ -207,6 +214,8 @@ paths:
/sources:
get:
tags:
+ - "public_sources"
+ - "public"
- "Sources"
responses:
"200":
@@ -220,8 +229,8 @@ paths:
"404":
description: "Not found"
operationId: "listSources"
+ x-speakeasy-alias: "listSources"
summary: "List sources"
- x-use-speakeasy-middleware: true
parameters:
- name: "workspaceIds"
description: "The UUIDs of the workspaces you wish to list sources for. Empty\
@@ -273,11 +282,13 @@ paths:
airbyte_source_name: "google-ads"
conversion_window_days: 14
customer_id: "1234567890"
- start_date: "2023-01-01"
- end_date: "2024-01-01"
+ start_date: 1672531200000
+ end_date: 1704067200000
name: "My Source"
workspaceId: "744cc0ed-7f05-4949-9e60-2a814f90c035"
tags:
+ - "public_sources"
+ - "public"
- "Sources"
responses:
"200":
@@ -295,13 +306,15 @@ paths:
"403":
description: "Not allowed"
operationId: "createSource"
+ x-speakeasy-alias: "createSource"
summary: "Create a source"
description: "Creates a source given a name, workspace id, and a json blob containing\
\ the configuration for the source."
- x-use-speakeasy-middleware: true
/sources/{sourceId}:
get:
tags:
+ - "public_sources"
+ - "public"
- "Sources"
responses:
"200":
@@ -322,10 +335,12 @@ paths:
"404":
description: "Not found"
operationId: "getSource"
+ x-speakeasy-alias: "getSource"
summary: "Get Source details"
- x-use-speakeasy-middleware: true
patch:
tags:
+ - "public_sources"
+ - "public"
- "Sources"
requestBody:
content:
@@ -339,8 +354,8 @@ paths:
airbyte_source_name: "google-ads"
conversion_window_days: 14
customer_id: "1234567890"
- start_date: "2023-01-01"
- end_date: "2024-01-01"
+ start_date: 1672531200000
+ end_date: 1704067200000
name: "My Source"
workspaceId: "744cc0ed-7f05-4949-9e60-2a814f90c035"
responses:
@@ -362,10 +377,12 @@ paths:
"404":
description: "Not found"
operationId: "patchSource"
+ x-speakeasy-alias: "patchSource"
summary: "Update a Source"
- x-use-speakeasy-middleware: true
put:
tags:
+ - "public_sources"
+ - "public"
- "Sources"
requestBody:
content:
@@ -379,8 +396,8 @@ paths:
airbyte_source_name: "google-ads"
conversion_window_days: 14
customer_id: "1234567890"
- start_date: "2023-01-01"
- end_date: "2024-01-01"
+ start_date: 1672531200000
+ end_date: 1704067200000
name: "My Source"
workspaceId: "744cc0ed-7f05-4949-9e60-2a814f90c035"
responses:
@@ -402,11 +419,13 @@ paths:
"404":
description: "Not found"
operationId: "putSource"
+ x-speakeasy-alias: "putSource"
summary: "Update a Source and fully overwrite it"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Source#update
delete:
tags:
+ - "public_sources"
+ - "public"
- "Sources"
responses:
"2XX":
@@ -416,8 +435,8 @@ paths:
"404":
description: "Not found"
operationId: "deleteSource"
+ x-speakeasy-alias: "deleteSource"
summary: "Delete a Source"
- x-use-speakeasy-middleware: true
parameters:
- name: "sourceId"
schema:
@@ -428,6 +447,8 @@ paths:
/destinations:
get:
tags:
+ - "public_destinations"
+ - "public"
- "Destinations"
responses:
"200":
@@ -441,8 +462,8 @@ paths:
"404":
description: "Not found"
operationId: "listDestinations"
+ x-speakeasy-alias: "listDestinations"
summary: "List destinations"
- x-use-speakeasy-middleware: true
parameters:
- name: "workspaceIds"
description: "The UUIDs of the workspaces you wish to list destinations for.\
@@ -504,6 +525,8 @@ paths:
username: "postgres"
password: "test"
tags:
+ - "public_destinations"
+ - "public"
- "Destinations"
responses:
"200":
@@ -523,13 +546,15 @@ paths:
"404":
description: "Not found"
operationId: "createDestination"
+ x-speakeasy-alias: "createDestination"
summary: "Create a destination"
description: "Creates a destination given a name, workspace id, and a json blob\
\ containing the configuration for the source."
- x-use-speakeasy-middleware: true
/destinations/{destinationId}:
get:
tags:
+ - "public_destinations"
+ - "public"
- "Destinations"
responses:
"200":
@@ -547,18 +572,20 @@ paths:
configuration:
conversion_window_days: 14
customer_id: "1234567890"
- start_date: "2023-01-01"
- end_date: "2024-01-01"
+ start_date: 1672531200000
+ end_date: 1704067200000
description: "Get a Destination by the id in the path."
"403":
description: "Not allowed"
"404":
description: "Not found"
operationId: "getDestination"
+ x-speakeasy-alias: "getDestination"
summary: "Get Destination details"
- x-use-speakeasy-middleware: true
delete:
tags:
+ - "public_destinations"
+ - "public"
- "Destinations"
responses:
"2XX":
@@ -568,10 +595,12 @@ paths:
"404":
description: "Not found"
operationId: "deleteDestination"
+ x-speakeasy-alias: "deleteDestination"
summary: "Delete a Destination"
- x-use-speakeasy-middleware: true
patch:
tags:
+ - "public_destinations"
+ - "public"
- "Destinations"
requestBody:
content:
@@ -584,8 +613,8 @@ paths:
configuration:
conversion_window_days: 14
customer_id: "1234567890"
- start_date: "2023-01-01"
- end_date: "2024-01-01"
+ start_date: 1672531200000
+ end_date: 1704067200000
name: "My Destination"
responses:
"200":
@@ -603,18 +632,20 @@ paths:
configuration:
conversion_window_days: 14
customer_id: "1234567890"
- start_date: "2023-01-01"
- end_date: "2024-01-01"
+ start_date: 1672531200000
+ end_date: 1704067200000
description: "Update a Destination"
"403":
description: "Not allowed"
"404":
description: "Not found"
operationId: "patchDestination"
+ x-speakeasy-alias: "patchDestination"
summary: "Update a Destination"
- x-use-speakeasy-middleware: true
put:
tags:
+ - "public_destinations"
+ - "public"
- "Destinations"
requestBody:
content:
@@ -627,8 +658,8 @@ paths:
configuration:
conversion_window_days: 14
customer_id: "1234567890"
- start_date: "2023-01-01"
- end_date: "2024-01-01"
+ start_date: 1672531200000
+ end_date: 1704067200000
name: "My Destination"
responses:
"200":
@@ -646,16 +677,16 @@ paths:
configuration:
conversion_window_days: 14
customer_id: "1234567890"
- start_date: "2023-01-01"
- end_date: "2024-01-01"
+ start_date: 1672531200000
+ end_date: 1704067200000
description: "Update a Destination and fully overwrite it"
"403":
description: "Not allowed"
"404":
description: "Not found"
operationId: "putDestination"
+ x-speakeasy-alias: "putDestination"
summary: "Update a Destination and fully overwrite it"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Destination#update
parameters:
- name: "destinationId"
@@ -673,6 +704,8 @@ paths:
$ref: "#/components/schemas/InitiateOauthRequest"
required: true
tags:
+ - "public_sources"
+ - "public"
- "Sources"
responses:
"200":
@@ -682,6 +715,7 @@ paths:
"403":
description: "API key is invalid."
operationId: "initiateOAuth"
+ x-speakeasy-alias: "initiateOAuth"
summary: "Initiate OAuth for a source"
description: "Given a source ID, workspace ID, and redirect URL, initiates OAuth\
\ for the source.\n\nThis returns a fully formed URL for performing user authentication\
@@ -691,7 +725,6 @@ paths:
\ ID to the redirect URL specified in the `secret_id` query string parameter.\n\
\nThat secret ID can be used to create a source with credentials in place\
\ of actual tokens."
- x-use-speakeasy-middleware: true
/connections:
post:
requestBody:
@@ -707,6 +740,8 @@ paths:
name: "Postgres-to-Bigquery"
required: true
tags:
+ - "public_connections"
+ - "public"
- "Connections"
responses:
"200":
@@ -724,11 +759,13 @@ paths:
"403":
description: "Not allowed"
operationId: "createConnection"
+ x-speakeasy-alias: "createConnection"
summary: "Create a connection"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Connection#create
get:
tags:
+ - "public_connections"
+ - "public"
- "Connections"
responses:
"200":
@@ -742,8 +779,8 @@ paths:
"404":
description: "Not found"
operationId: "listConnections"
+ x-speakeasy-alias: "listConnections"
summary: "List connections"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Connection#list
parameters:
- name: "workspaceIds"
@@ -785,6 +822,8 @@ paths:
/connections/{connectionId}:
get:
tags:
+ - "public_connections"
+ - "public"
- "Connections"
responses:
"200":
@@ -805,11 +844,13 @@ paths:
"404":
description: "Not found"
operationId: "getConnection"
+ x-speakeasy-alias: "getConnection"
summary: "Get Connection details"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Connection#read
patch:
tags:
+ - "public_connections"
+ - "public"
- "Connections"
requestBody:
content:
@@ -842,11 +883,13 @@ paths:
"404":
description: "Not found"
operationId: "patchConnection"
+ x-speakeasy-alias: "patchConnection"
summary: "Update Connection details"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Connection#update
delete:
tags:
+ - "public_connections"
+ - "public"
- "Connections"
responses:
"2XX":
@@ -856,8 +899,8 @@ paths:
"404":
description: "Not found"
operationId: "deleteConnection"
+ x-speakeasy-alias: "deleteConnection"
summary: "Delete a Connection"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Connection#delete
parameters:
- name: "connectionId"
@@ -869,6 +912,8 @@ paths:
/streams:
get:
tags:
+ - "public_streams"
+ - "public"
- "Streams"
responses:
"200":
@@ -885,8 +930,8 @@ paths:
"404":
description: "Not found"
operationId: "getStreamProperties"
+ x-speakeasy-alias: "getStreamProperties"
summary: "Get stream properties"
- x-use-speakeasy-middleware: true
parameters:
- name: "sourceId"
description: "ID of the source"
@@ -901,7 +946,7 @@ paths:
format: "UUID"
type: "string"
in: "query"
- required: true
+ required: false
- name: "ignoreCache"
description: "If true pull the latest schema from the source, else pull from\
\ cache (default false)"
@@ -913,6 +958,8 @@ paths:
/workspaces:
get:
tags:
+ - "public_workspaces"
+ - "public"
- "Workspaces"
responses:
"200":
@@ -926,8 +973,9 @@ paths:
"404":
description: "Not found"
operationId: "listWorkspaces"
+ x-speakeasy-alias: "listWorkspaces"
summary: "List workspaces"
- x-use-speakeasy-middleware: true
+ x-speakeasy-entity-operation: Workspace#list
parameters:
- name: "workspaceIds"
description: "The UUIDs of the workspaces you wish to fetch. Empty list will\
@@ -965,9 +1013,10 @@ paths:
minimum: 0
default: 0
in: "query"
- x-speakeasy-entity-operation: Workspace#list
post:
tags:
+ - "public_workspaces"
+ - "public"
- "Workspaces"
requestBody:
content:
@@ -979,7 +1028,6 @@ paths:
value:
name: "Company Workspace Name"
required: true
- x-speakeasy-entity-operation: Workspace#create
responses:
"200":
content:
@@ -996,8 +1044,8 @@ paths:
"403":
description: "Not allowed"
operationId: "createWorkspace"
+ x-speakeasy-alias: "createWorkspace"
summary: "Create a workspace"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Workspace#create
/workspaces/{workspaceId}:
parameters:
@@ -1009,6 +1057,8 @@ paths:
required: true
get:
tags:
+ - "public_workspaces"
+ - "public"
- "Workspaces"
responses:
"200":
@@ -1028,11 +1078,13 @@ paths:
"404":
description: "Not found"
operationId: "getWorkspace"
+ x-speakeasy-alias: "getWorkspace"
summary: "Get Workspace details"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Workspace#read
patch:
tags:
+ - "public_workspaces"
+ - "public"
- "Workspaces"
requestBody:
content:
@@ -1060,11 +1112,13 @@ paths:
"403":
description: "Not allowed"
operationId: "updateWorkspace"
+ x-speakeasy-alias: "updateWorkspace"
summary: "Update a workspace"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Workspace#update
delete:
tags:
+ - "public_workspaces"
+ - "public"
- "Workspaces"
responses:
"2XX":
@@ -1074,8 +1128,8 @@ paths:
"404":
description: "Not found"
operationId: "deleteWorkspace"
+ x-speakeasy-alias: "deleteWorkspace"
summary: "Delete a Workspace"
- x-use-speakeasy-middleware: true
x-speakeasy-entity-operation: Workspace#delete
/workspaces/{workspaceId}/oauthCredentials:
put:
@@ -1086,6 +1140,8 @@ paths:
$ref: "#/components/schemas/WorkspaceOAuthCredentialsRequest"
required: true
tags:
+ - "public_workspaces"
+ - "public"
- "Workspaces"
responses:
"200":
@@ -1095,12 +1151,12 @@ paths:
"403":
description: "API key is invalid."
operationId: "createOrUpdateWorkspaceOAuthCredentials"
+ x-speakeasy-alias: "createOrUpdateWorkspaceOAuthCredentials"
summary: "Create OAuth override credentials for a workspace and source type."
description: "Create/update a set of OAuth credentials to override the Airbyte-provided\
\ OAuth credentials used for source/destination OAuth.\nIn order to determine\
\ what the credential configuration needs to be, please see the connector\
\ specification of the relevant source/destination."
- x-use-speakeasy-middleware: true
parameters:
- name: "workspaceId"
schema:
@@ -12288,92 +12344,6 @@ paths:
type: "string"
in: "path"
required: true
- /sources#Quickbooks:
- post:
- requestBody:
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/SourceQuickbooksCreateRequest"
- tags:
- - "Sources"
- responses:
- "200":
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/SourceResponse"
- description: "Successful operation"
- "400":
- description: "Invalid data"
- "403":
- description: "Not allowed"
- operationId: "createSourceQuickbooks"
- summary: "Create a source"
- description:
- "Creates a source given a name, workspace id, and a json blob containing\
- \ the configuration for the source."
- x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Source_Quickbooks#create
- /sources/{sourceId}#Quickbooks:
- get:
- tags:
- - "Sources"
- responses:
- "200":
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/SourceResponse"
- description: "Get a Source by the id in the path."
- "403":
- description: "Not allowed"
- "404":
- description: "Not found"
- operationId: "getSourceQuickbooks"
- summary: "Get Source details"
- x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Source_Quickbooks#read
- put:
- tags:
- - "Sources"
- requestBody:
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/SourceQuickbooksPutRequest"
- responses:
- "2XX":
- description: "The resource was updated successfully"
- "403":
- description: "Not allowed"
- "404":
- description: "Not found"
- operationId: "putSourceQuickbooks"
- summary: "Update a Source fully"
- x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Source_Quickbooks#update
- delete:
- tags:
- - "Sources"
- responses:
- "2XX":
- description: "The resource was deleted successfully"
- "403":
- description: "Not allowed"
- "404":
- description: "Not found"
- operationId: "deleteSourceQuickbooks"
- summary: "Delete a Source"
- x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Source_Quickbooks#delete
- parameters:
- - name: "sourceId"
- schema:
- format: "UUID"
- type: "string"
- in: "path"
- required: true
/sources#Railz:
post:
requestBody:
@@ -17792,92 +17762,6 @@ paths:
type: "string"
in: "path"
required: true
- /sources#Custom:
- post:
- requestBody:
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/SourceCustomCreateRequest"
- tags:
- - "Sources"
- responses:
- "200":
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/SourceResponse"
- description: "Successful operation"
- "400":
- description: "Invalid data"
- "403":
- description: "Not allowed"
- operationId: "createSourceCustom"
- summary: "Create a source"
- description:
- "Creates a source given a name, workspace id, and a json blob containing\
- \ the configuration for the source."
- x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Source_Custom#create
- /sources/{sourceId}#Custom:
- get:
- tags:
- - "Sources"
- responses:
- "200":
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/SourceResponse"
- description: "Get a Source by the id in the path."
- "403":
- description: "Not allowed"
- "404":
- description: "Not found"
- operationId: "getSourceCustom"
- summary: "Get Source details"
- x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Source_Custom#read
- put:
- tags:
- - "Sources"
- requestBody:
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/SourceCustomPutRequest"
- responses:
- "2XX":
- description: "The resource was updated successfully"
- "403":
- description: "Not allowed"
- "404":
- description: "Not found"
- operationId: "putSourceCustom"
- summary: "Update a Source fully"
- x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Source_Custom#update
- delete:
- tags:
- - "Sources"
- responses:
- "2XX":
- description: "The resource was deleted successfully"
- "403":
- description: "Not allowed"
- "404":
- description: "Not found"
- operationId: "deleteSourceCustom"
- summary: "Delete a Source"
- x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Source_Custom#delete
- parameters:
- - name: "sourceId"
- schema:
- format: "UUID"
- type: "string"
- in: "path"
- required: true
/destinations#Astra:
post:
requestBody:
@@ -20802,13 +20686,13 @@ paths:
type: "string"
in: "path"
required: true
- /destinations#Custom:
+ /destinations#Yellowbrick:
post:
requestBody:
content:
application/json:
schema:
- $ref: "#/components/schemas/DestinationCustomCreateRequest"
+ $ref: "#/components/schemas/DestinationYellowbrickCreateRequest"
tags:
- "Destinations"
responses:
@@ -20822,14 +20706,14 @@ paths:
description: "Invalid data"
"403":
description: "Not allowed"
- operationId: "createDestinationCustom"
+ operationId: "createDestinationYellowbrick"
summary: "Create a destination"
description:
"Creates a destination given a name, workspace id, and a json blob containing\
\ the configuration for the destination."
x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Destination_Custom#create
- /destinations/{destinationId}#Custom:
+ x-speakeasy-entity-operation: Destination_Yellowbrick#create
+ /destinations/{destinationId}#Yellowbrick:
get:
tags:
- "Destinations"
@@ -20844,10 +20728,10 @@ paths:
description: "Not allowed"
"404":
description: "Not found"
- operationId: "getDestinationCustom"
+ operationId: "getDestinationYellowbrick"
summary: "Get Destination details"
x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Destination_Custom#read
+ x-speakeasy-entity-operation: Destination_Yellowbrick#read
put:
tags:
- "Destinations"
@@ -20855,7 +20739,7 @@ paths:
content:
application/json:
schema:
- $ref: "#/components/schemas/DestinationCustomPutRequest"
+ $ref: "#/components/schemas/DestinationYellowbrickPutRequest"
responses:
"2XX":
description: "The resource was updated successfully"
@@ -20863,10 +20747,10 @@ paths:
description: "Not allowed"
"404":
description: "Not found"
- operationId: "putDestinationCustom"
+ operationId: "putDestinationYellowbrick"
summary: "Update a Destination fully"
x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Destination_Custom#update
+ x-speakeasy-entity-operation: Destination_Yellowbrick#update
delete:
tags:
- "Destinations"
@@ -20877,10 +20761,10 @@ paths:
description: "Not allowed"
"404":
description: "Not found"
- operationId: "deleteDestinationCustom"
+ operationId: "deleteDestinationYellowbrick"
summary: "Delete a Destination"
x-use-speakeasy-middleware: true
- x-speakeasy-entity-operation: Destination_Custom#delete
+ x-speakeasy-entity-operation: Destination_Yellowbrick#delete
parameters:
- name: "destinationId"
schema:
@@ -20896,7 +20780,43 @@ components:
description: "Response from the initiate OAuth call should be an object with\
\ a single property which will be the `redirect_url`. If a user is redirected\
\ to this URL, they'll be prompted by the identity provider to authenticate."
+ x-speakeasy-component: true
schemas:
+ WorkspaceId:
+ type: "string"
+ format: "uuid"
+ x-speakeasy-component: true
+ OrganizationId:
+ type: "string"
+ format: "uuid"
+ x-speakeasy-component: true
+ PermissionType:
+ type: "string"
+ description: "Describes what actions/endpoints the permission entitles to"
+ enum:
+ - "instance_admin"
+ - "organization_admin"
+ - "organization_editor"
+ - "organization_reader"
+ - "organization_member"
+ - "workspace_owner"
+ - "workspace_admin"
+ - "workspace_editor"
+ - "workspace_reader"
+ x-speakeasy-component: true
+ UserId:
+ type: "string"
+ description: "Internal Airbyte user ID"
+ format: "uuid"
+ x-speakeasy-component: true
+ OAuthConfiguration:
+ description: "The values required to configure OAuth flows. The schema for this\
+ \ must match the `OAuthConfigSpecification.oauthUserInputFromConnectorConfigSpecification`\
+ \ schema."
+ x-speakeasy-component: true
+ OAuthInputConfiguration:
+ $ref: "#/components/schemas/OAuthConfiguration"
+ x-speakeasy-component: true
RedirectUrlResponse:
title: "Root Type for RedirectUrlResponse"
description: ""
@@ -20906,7 +20826,8 @@ components:
format: "url"
type: "string"
example:
- redirectUrl: "https://airbyte.portal.speakeasyapi.dev?speakeasyAccessToken=eydas.ad45.1234"
+ redirectUrl: "https://example.com"
+ x-speakeasy-component: true
JobResponse:
title: "Root Type for JobResponse"
description: "Provides details of a single job."
@@ -20947,6 +20868,7 @@ components:
jobType: "sync"
startTime: "2023-03-25T01:30:50Z"
duration: "PT8H6M12S"
+ x-speakeasy-component: true
JobsResponse:
title: "Root Type for JobsResponse"
description: ""
@@ -20970,6 +20892,7 @@ components:
status: "running"
jobType: "sync"
startTime: "2023-03-25T01:30:50Z"
+ x-speakeasy-component: true
ConnectionCreateRequest:
required:
- "sourceId"
@@ -20988,7 +20911,7 @@ components:
configurations:
$ref: "#/components/schemas/StreamConfigurations"
schedule:
- $ref: "#/components/schemas/ConnectionSchedule"
+ $ref: "#/components/schemas/AirbyteApiConnectionSchedule"
dataResidency:
$ref: "#/components/schemas/GeographyEnum"
namespaceDefinition:
@@ -21011,6 +20934,7 @@ components:
$ref: "#/components/schemas/ConnectionStatusEnum"
x-speakeasy-entity: Connection
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
ConnectionPatchRequest:
type: "object"
properties:
@@ -21020,7 +20944,7 @@ components:
configurations:
$ref: "#/components/schemas/StreamConfigurations"
schedule:
- $ref: "#/components/schemas/ConnectionSchedule"
+ $ref: "#/components/schemas/AirbyteApiConnectionSchedule"
dataResidency:
$ref: "#/components/schemas/GeographyEnumNoDefault"
namespaceDefinition:
@@ -21043,6 +20967,7 @@ components:
$ref: "#/components/schemas/ConnectionStatusEnum"
x-speakeasy-entity: Connection
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
JobCreateRequest:
title: "Root Type for JobCreate"
description: "Creates a new Job from the configuration provided in the request\
@@ -21060,6 +20985,7 @@ components:
example:
connectionId: "18dccc91-0ab1-4f72-9ed7-0b8fc27c5826"
jobType: "sync"
+ x-speakeasy-component: true
JobStatusEnum:
enum:
- "pending"
@@ -21069,6 +20995,7 @@ components:
- "succeeded"
- "cancelled"
type: "string"
+ x-speakeasy-component: true
JobTypeEnum:
description: "Enum that describes the different types of jobs that the platform\
\ runs."
@@ -21076,6 +21003,7 @@ components:
- "sync"
- "reset"
type: "string"
+ x-speakeasy-component: true
SourceCreateRequest:
required:
- "name"
@@ -21097,12 +21025,12 @@ components:
configuration:
$ref: "#/components/schemas/SourceConfiguration"
secretId:
- description: "Optional secretID obtained through the public API OAuth redirect\
- \ flow."
+ description: "Optional secretID obtained through the OAuth redirect flow."
type: "string"
- x-implements: "io.airbyte.public_api.server.helpers.ConfigurableActor"
+ x-implements: "io.airbyte.api.common.ConfigurableActor"
x-speakeasy-entity: Source
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
SourcePutRequest:
required:
- "name"
@@ -21113,9 +21041,10 @@ components:
type: "string"
configuration:
$ref: "#/components/schemas/SourceConfiguration"
- x-implements: "io.airbyte.public_api.server.helpers.ConfigurableActor"
+ x-implements: "io.airbyte.api.common.ConfigurableActor"
x-speakeasy-entity: Source
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
SourcePatchRequest:
type: "object"
properties:
@@ -21128,15 +21057,14 @@ components:
configuration:
$ref: "#/components/schemas/SourceConfiguration"
secretId:
- description: "Optional secretID obtained through the public API OAuth redirect\
- \ flow."
+ description: "Optional secretID obtained through the OAuth redirect flow."
type: "string"
- x-implements: "io.airbyte.public_api.server.helpers.ConfigurableActor"
+ x-implements: "io.airbyte.api.common.ConfigurableActor"
x-speakeasy-entity: Source
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
InitiateOauthRequest:
title: "Root Type for initiate-oauth-post-body"
- description: "POST body for initiating OAuth via the public API"
required:
- "redirectUrl"
- "workspaceId"
@@ -21165,6 +21093,7 @@ components:
redirectUrl: "https://cloud.airbyte.io/v1/api/oauth/callback"
workspaceId: "871d9b60-11d1-44cb-8c92-c246d53bf87e"
destinationId: "3d93b16c-ff5f-421c-8908-5a3c82088f14"
+ x-speakeasy-component: true
WorkspaceOAuthCredentialsRequest:
title: "Root Type for WorkspaceOAuthCredentials"
description: "POST body for creating/updating workspace level OAuth credentials"
@@ -21181,6 +21110,7 @@ components:
description: "The name of the source i.e. google-ads"
configuration:
$ref: "#/components/schemas/OAuthCredentialsConfiguration"
+ x-speakeasy-component: true
OAuthCredentialsConfiguration:
description: "The configuration for this source/destination based on the OAuth\
\ section of the relevant specification."
@@ -21189,12 +21119,7 @@ components:
credentials:
client_id: "871d9b60-11d1-44cb-8c92-c246d53bf87e"
client_secret: "shhhhhh"
- OAuthInputConfiguration:
- description: "Arbitrary vars to pass for OAuth depending on what the source/destination\
- \ spec requires."
- type: "object"
- example:
- host: "test.snowflake.com"
+ x-speakeasy-component: true
ConnectionResponse:
title: "Root Type for ConnectionResponse"
description: "Provides details of a single connection."
@@ -21242,7 +21167,8 @@ components:
$ref: "#/components/schemas/StreamConfigurations"
x-speakeasy-entity: Connection
x-speakeasy-param-suppress-computed-diff: true
- ConnectionSchedule:
+ x-speakeasy-component: true
+ AirbyteApiConnectionSchedule:
description: "schedule for when the the connection should run, per the schedule\
\ type"
type: "object"
@@ -21253,11 +21179,13 @@ components:
$ref: "#/components/schemas/ScheduleTypeEnum"
cronExpression:
type: "string"
+ x-speakeasy-component: true
ScheduleTypeEnum:
type: "string"
enum:
- "manual"
- "cron"
+ x-speakeasy-component: true
ConnectionScheduleResponse:
description: "schedule for when the the connection should run, per the schedule\
\ type"
@@ -21271,12 +21199,14 @@ components:
type: "string"
basicTiming:
type: "string"
+ x-speakeasy-component: true
ScheduleTypeWithBasicEnum:
type: "string"
enum:
- "manual"
- "cron"
- "basic"
+ x-speakeasy-component: true
GeographyEnum:
type: "string"
enum:
@@ -21284,18 +21214,21 @@ components:
- "us"
- "eu"
default: "auto"
+ x-speakeasy-component: true
GeographyEnumNoDefault:
type: "string"
enum:
- "auto"
- "us"
- "eu"
+ x-speakeasy-component: true
ConnectionStatusEnum:
type: "string"
enum:
- "active"
- "inactive"
- "deprecated"
+ x-speakeasy-component: true
NamespaceDefinitionEnum:
type: "string"
description: "Define the location where the data will be stored in the destination"
@@ -21304,6 +21237,7 @@ components:
- "destination"
- "custom_format"
default: "destination"
+ x-speakeasy-component: true
NonBreakingSchemaUpdatesBehaviorEnum:
type: "string"
description: "Set how Airbyte handles syncs when it detects a non-breaking schema\
@@ -21314,6 +21248,7 @@ components:
- "propagate_columns"
- "propagate_fully"
default: "ignore"
+ x-speakeasy-component: true
NamespaceDefinitionEnumNoDefault:
type: "string"
description: "Define the location where the data will be stored in the destination"
@@ -21321,6 +21256,7 @@ components:
- "source"
- "destination"
- "custom_format"
+ x-speakeasy-component: true
NonBreakingSchemaUpdatesBehaviorEnumNoDefault:
type: "string"
description: "Set how Airbyte handles syncs when it detects a non-breaking schema\
@@ -21330,6 +21266,7 @@ components:
- "disable_connection"
- "propagate_columns"
- "propagate_fully"
+ x-speakeasy-component: true
DestinationResponse:
title: "Root Type for DestinationResponse"
description: "Provides details of a single destination."
@@ -21358,6 +21295,7 @@ components:
name: "Analytics Team Postgres"
destinationType: "postgres"
workspaceId: "871d9b60-11d1-44cb-8c92-c246d53bf87e"
+ x-speakeasy-component: true
SourceResponse:
title: "Root Type for SourceResponse"
description: "Provides details of a single source."
@@ -21386,6 +21324,7 @@ components:
name: "Analytics Team Postgres"
sourceType: "postgres"
workspaceId: "871d9b60-11d1-44cb-8c92-c246d53bf87e"
+ x-speakeasy-component: true
DestinationCreateRequest:
required:
- "name"
@@ -21406,9 +21345,10 @@ components:
type: "string"
configuration:
$ref: "#/components/schemas/DestinationConfiguration"
- x-implements: "io.airbyte.public_api.server.helpers.ConfigurableActor"
+ x-implements: "io.airbyte.api.common.ConfigurableActor"
x-speakeasy-entity: Destination
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
DestinationPatchRequest:
type: "object"
properties:
@@ -21416,9 +21356,10 @@ components:
type: "string"
configuration:
$ref: "#/components/schemas/DestinationConfiguration"
- x-implements: "io.airbyte.public_api.server.helpers.ConfigurableActor"
+ x-implements: "io.airbyte.api.common.ConfigurableActor"
x-speakeasy-entity: Destination
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
DestinationPutRequest:
required:
- "name"
@@ -21429,9 +21370,10 @@ components:
type: "string"
configuration:
$ref: "#/components/schemas/DestinationConfiguration"
- x-implements: "io.airbyte.public_api.server.helpers.ConfigurableActor"
+ x-implements: "io.airbyte.api.common.ConfigurableActor"
x-speakeasy-entity: Destination
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
WorkspaceCreateRequest:
required:
- "name"
@@ -21442,6 +21384,7 @@ components:
type: "string"
x-speakeasy-entity: Workspace
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
WorkspaceUpdateRequest:
required:
- "name"
@@ -21452,6 +21395,7 @@ components:
type: "string"
x-speakeasy-entity: Workspace
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
WorkspaceResponse:
title: "Root Type for WorkspaceResponse"
description: "Provides details of a single workspace."
@@ -21470,6 +21414,47 @@ components:
$ref: "#/components/schemas/GeographyEnum"
x-speakeasy-entity: Workspace
x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
+ PermissionCreateRequest:
+ required:
+ - "permissionType"
+ - "userId"
+ type: "object"
+ properties:
+ permissionType:
+ $ref: "#/components/schemas/PermissionType"
+ userId:
+ $ref: "#/components/schemas/UserId"
+ workspaceId:
+ $ref: "#/components/schemas/WorkspaceId"
+ organizationId:
+ $ref: "#/components/schemas/OrganizationId"
+ x-speakeasy-entity: Permission
+ x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
+ PermissionResponse:
+ title: "Root Type for PermissionResponse"
+ description: "Provides details of a single permission."
+ type: "object"
+ required:
+ - "permissionId"
+ - "permissionType"
+ - "userId"
+ properties:
+ permissionId:
+ type: "string"
+ format: "uuid"
+ permissionType:
+ $ref: "#/components/schemas/PermissionType"
+ userId:
+ $ref: "#/components/schemas/UserId"
+ workspaceId:
+ $ref: "#/components/schemas/WorkspaceId"
+ organizationId:
+ $ref: "#/components/schemas/OrganizationId"
+ x-speakeasy-entity: Permission
+ x-speakeasy-param-suppress-computed-diff: true
+ x-speakeasy-component: true
ConnectionsResponse:
title: "Root Type for ConnectionsResponse"
description: ""
@@ -21498,6 +21483,7 @@ components:
scheduleType: "manual"
- status: "active"
- dataResidency: "auto"
+ x-speakeasy-component: true
SourcesResponse:
title: "Root Type for SourcesResponse"
description: ""
@@ -21521,6 +21507,7 @@ components:
name: "Analytics Team Postgres"
sourceType: "postgres"
workspaceId: "871d9b60-11d1-44cb-8c92-c246d53bf87e"
+ x-speakeasy-component: true
DestinationsResponse:
title: "Root Type for DestinationsResponse"
description: ""
@@ -21544,6 +21531,7 @@ components:
name: "Analytics Team Postgres"
destinationType: "postgres"
workspaceId: "871d9b60-11d1-44cb-8c92-c246d53bf87e"
+ x-speakeasy-component: true
WorkspacesResponse:
title: "Root Type for WorkspacesResponse"
description: ""
@@ -21566,6 +21554,7 @@ components:
workspaceId: "18dccc91-0ab1-4f72-9ed7-0b8fc27c5826"
name: "Acme Company"
dataResidency: "auto"
+ x-speakeasy-component: true
StreamConfiguration:
description: "Configurations for a single stream."
type: "object"
@@ -21592,6 +21581,7 @@ components:
type: "array"
items:
type: "string"
+ x-speakeasy-component: true
StreamConfigurations:
description: "A list of configured stream options for a connection."
type: "object"
@@ -21600,14 +21590,13 @@ components:
type: "array"
items:
$ref: "#/components/schemas/StreamConfiguration"
+ x-speakeasy-component: true
StreamPropertiesResponse:
description: "A list of stream properties."
- type: "object"
- properties:
- streams:
- type: "array"
- items:
- $ref: "#/components/schemas/StreamProperties"
+ type: "array"
+ items:
+ $ref: "#/components/schemas/StreamProperties"
+ x-speakeasy-component: true
StreamProperties:
description: "The stream properties associated with a connection."
type: "object"
@@ -21636,17 +21625,20 @@ components:
type: "array"
items:
type: "string"
+ x-speakeasy-component: true
ConnectionSyncModeEnum:
enum:
- "full_refresh_overwrite"
- "full_refresh_append"
- "incremental_append"
- "incremental_deduped_history"
+ x-speakeasy-component: true
ActorTypeEnum:
description: "Whether you're setting this override for a source or destination"
enum:
- "source"
- "destination"
+ x-speakeasy-component: true
source-trello:
type: "object"
required:
@@ -22806,177 +22798,6 @@ components:
order: 1
required:
- "api_key"
- source-quickbooks:
- title: "Source QuickBooks Spec"
- type: "object"
- required:
- - "credentials"
- - "start_date"
- - "sandbox"
- - "sourceType"
- properties:
- credentials:
- title: "Authorization Method"
- type: "object"
- order: 0
- oneOf:
- - type: "object"
- title: "OAuth2.0"
- required:
- - "client_id"
- - "client_secret"
- - "refresh_token"
- - "access_token"
- - "token_expiry_date"
- - "realm_id"
- properties:
- auth_type:
- type: "string"
- const: "oauth2.0"
- enum:
- - "oauth2.0"
- client_id:
- type: "string"
- title: "Client ID"
- description: "Identifies which app is making the request. Obtain this\
- \ value from the Keys tab on the app profile via My Apps on the\
- \ developer site. There are two versions of this key: development\
- \ and production."
- client_secret:
- description: " Obtain this value from the Keys tab on the app profile\
- \ via My Apps on the developer site. There are two versions of this\
- \ key: development and production."
- title: "Client Secret"
- type: "string"
- airbyte_secret: true
- refresh_token:
- description: "A token used when refreshing the access token."
- title: "Refresh Token"
- type: "string"
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- access_token:
- description: "Access token for making authenticated requests."
- title: "Access Token"
- type: "string"
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- token_expiry_date:
- type: "string"
- title: "Token Expiry Date"
- description: "The date-time when the access token should be refreshed."
- format: "date-time"
- x-speakeasy-param-sensitive: true
- realm_id:
- description: "Labeled Company ID. The Make API Calls panel is populated\
- \ with the realm id and the current access token."
- title: "Realm ID"
- type: "string"
- airbyte_secret: true
- start_date:
- order: 1
- description: "The default value to use if no bookmark exists for an endpoint\
- \ (rfc3339 date string). E.g, 2021-03-20T00:00:00Z. Any data before this\
- \ date will not be replicated."
- title: "Start Date"
- type: "string"
- format: "date-time"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- examples:
- - "2021-03-20T00:00:00Z"
- sandbox:
- order: 2
- description: "Determines whether to use the sandbox or production environment."
- title: "Sandbox"
- type: "boolean"
- default: false
- sourceType:
- title: "quickbooks"
- const: "quickbooks"
- enum:
- - "quickbooks"
- order: 0
- type: "string"
- source-quickbooks-update:
- title: "Source QuickBooks Spec"
- type: "object"
- required:
- - "credentials"
- - "start_date"
- - "sandbox"
- properties:
- credentials:
- title: "Authorization Method"
- type: "object"
- order: 0
- oneOf:
- - type: "object"
- title: "OAuth2.0"
- required:
- - "client_id"
- - "client_secret"
- - "refresh_token"
- - "access_token"
- - "token_expiry_date"
- - "realm_id"
- properties:
- auth_type:
- type: "string"
- const: "oauth2.0"
- enum:
- - "oauth2.0"
- client_id:
- type: "string"
- title: "Client ID"
- description: "Identifies which app is making the request. Obtain this\
- \ value from the Keys tab on the app profile via My Apps on the\
- \ developer site. There are two versions of this key: development\
- \ and production."
- client_secret:
- description: " Obtain this value from the Keys tab on the app profile\
- \ via My Apps on the developer site. There are two versions of this\
- \ key: development and production."
- title: "Client Secret"
- type: "string"
- airbyte_secret: true
- refresh_token:
- description: "A token used when refreshing the access token."
- title: "Refresh Token"
- type: "string"
- airbyte_secret: true
- access_token:
- description: "Access token for making authenticated requests."
- title: "Access Token"
- type: "string"
- airbyte_secret: true
- token_expiry_date:
- type: "string"
- title: "Token Expiry Date"
- description: "The date-time when the access token should be refreshed."
- format: "date-time"
- realm_id:
- description: "Labeled Company ID. The Make API Calls panel is populated\
- \ with the realm id and the current access token."
- title: "Realm ID"
- type: "string"
- airbyte_secret: true
- start_date:
- order: 1
- description: "The default value to use if no bookmark exists for an endpoint\
- \ (rfc3339 date string). E.g, 2021-03-20T00:00:00Z. Any data before this\
- \ date will not be replicated."
- title: "Start Date"
- type: "string"
- format: "date-time"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- examples:
- - "2021-03-20T00:00:00Z"
- sandbox:
- order: 2
- description: "Determines whether to use the sandbox or production environment."
- title: "Sandbox"
- type: "boolean"
- default: false
source-dockerhub:
type: "object"
required:
@@ -24850,31 +24671,28 @@ components:
type: "integer"
order: 4
source-sendgrid:
- title: "Sendgrid Spec"
type: "object"
required:
- - "apikey"
+ - "start_date"
+ - "api_key"
- "sourceType"
properties:
- apikey:
- title: "Sendgrid API key"
- airbyte_secret: true
+ start_date:
type: "string"
- description: "API Key, use admin to generate this key."
+ title: "Start date"
+ format: "date-time"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
order: 0
- x-speakeasy-param-sensitive: true
- start_time:
- title: "Start time"
+ description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
+ \ data before this date will not be replicated."
+ api_key:
type: "string"
- format: "date-time"
- pattern: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.\\d+)?Z$"
- description: "Start time in ISO8601 format. Any data before this time point\
- \ will not be replicated."
- examples:
- - "2020-01-01T01:01:01Z"
- - "2020-01-01T01:01:01.000001Z"
+ title: "API Key"
+ airbyte_secret: true
order: 1
+ description: "Sendgrid API Key, use admin to generate this key."
+ x-speakeasy-param-sensitive: true
sourceType:
title: "sendgrid"
const: "sendgrid"
@@ -24883,29 +24701,26 @@ components:
order: 0
type: "string"
source-sendgrid-update:
- title: "Sendgrid Spec"
type: "object"
required:
- - "apikey"
+ - "start_date"
+ - "api_key"
properties:
- apikey:
- title: "Sendgrid API key"
- airbyte_secret: true
+ start_date:
type: "string"
- description: "API Key, use admin to generate this key."
+ title: "Start date"
+ format: "date-time"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
order: 0
- start_time:
- title: "Start time"
+ description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
+ \ data before this date will not be replicated."
+ api_key:
type: "string"
- format: "date-time"
- pattern: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.\\d+)?Z$"
- description: "Start time in ISO8601 format. Any data before this time point\
- \ will not be replicated."
- examples:
- - "2020-01-01T01:01:01Z"
- - "2020-01-01T01:01:01.000001Z"
+ title: "API Key"
+ airbyte_secret: true
order: 1
+ description: "Sendgrid API Key, use admin to generate this key."
source-gnews:
title: "Gnews Spec"
type: "object"
@@ -26902,6 +26717,21 @@ components:
default: "Fail sync"
order: 11
group: "advanced"
+ update_capture_mode:
+ type: "string"
+ title: "Capture mode (Advanced)"
+ description: "Determines how Airbyte looks up the value of an updated document.\
+ \ If 'Lookup' is chosen, the current value of the document will be read.\
+ \ If 'Post Image' is chosen, then the version of the document immediately\
+ \ after an update will be read. WARNING : Severe data loss will occur\
+ \ if this option is chosen and the appropriate settings are not set on\
+ \ your Mongo instance : https://www.mongodb.com/docs/manual/changeStreams/#change-streams-with-document-pre-and-post-images."
+ enum:
+ - "Lookup"
+ - "Post Image"
+ default: "Lookup"
+ order: 12
+ group: "advanced"
sourceType:
title: "mongodb-v2"
const: "mongodb-v2"
@@ -27091,6 +26921,21 @@ components:
default: "Fail sync"
order: 11
group: "advanced"
+ update_capture_mode:
+ type: "string"
+ title: "Capture mode (Advanced)"
+ description: "Determines how Airbyte looks up the value of an updated document.\
+ \ If 'Lookup' is chosen, the current value of the document will be read.\
+ \ If 'Post Image' is chosen, then the version of the document immediately\
+ \ after an update will be read. WARNING : Severe data loss will occur\
+ \ if this option is chosen and the appropriate settings are not set on\
+ \ your Mongo instance : https://www.mongodb.com/docs/manual/changeStreams/#change-streams-with-document-pre-and-post-images."
+ enum:
+ - "Lookup"
+ - "Post Image"
+ default: "Lookup"
+ order: 12
+ group: "advanced"
groups:
- id: "connection"
- id: "advanced"
@@ -27638,6 +27483,13 @@ components:
enum:
- "None"
- "Primitive Types Only"
+ ignore_errors_on_fields_mismatch:
+ title: "Ignore errors on field mismatch"
+ description: "Whether to ignore errors that occur when the number\
+ \ of fields in the CSV does not match the number of columns\
+ \ in the schema."
+ default: false
+ type: "boolean"
required:
- "filetype"
- title: "Jsonl Format"
@@ -28402,6 +28254,13 @@ components:
enum:
- "None"
- "Primitive Types Only"
+ ignore_errors_on_fields_mismatch:
+ title: "Ignore errors on field mismatch"
+ description: "Whether to ignore errors that occur when the number\
+ \ of fields in the CSV does not match the number of columns\
+ \ in the schema."
+ default: false
+ type: "boolean"
required:
- "filetype"
- title: "Jsonl Format"
@@ -28877,7 +28736,7 @@ components:
- "streams"
- "bucket"
source-azure-blob-storage:
- title: "Config"
+ title: "SourceAzureBlobStorageSpec"
description: "NOTE: When this Spec is changed, legacy_config_transformer.py\
\ must also be modified to uptake the changes\nbecause it is responsible for\
\ converting legacy Azure Blob Storage v0 configs into v1 configs using the\
@@ -29266,22 +29125,78 @@ components:
required:
- "name"
- "format"
+ credentials:
+ title: "Authentication"
+ description: "Credentials for connecting to the Azure Blob Storage"
+ type: "object"
+ order: 2
+ oneOf:
+ - title: "Authenticate via Oauth2"
+ type: "object"
+ properties:
+ auth_type:
+ title: "Auth Type"
+ default: "oauth2"
+ const: "oauth2"
+ enum:
+ - "oauth2"
+ type: "string"
+ tenant_id:
+ title: "Tenant ID"
+ description: "Tenant ID of the Microsoft Azure Application user"
+ airbyte_secret: true
+ type: "string"
+ client_id:
+ title: "Client ID"
+ description: "Client ID of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ client_secret:
+ title: "Client Secret"
+ description: "Client Secret of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ refresh_token:
+ title: "Refresh Token"
+ description: "Refresh Token of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ required:
+ - "tenant_id"
+ - "client_id"
+ - "client_secret"
+ - "refresh_token"
+ - "auth_type"
+ - title: "Authenticate via Storage Account Key"
+ type: "object"
+ properties:
+ auth_type:
+ title: "Auth Type"
+ default: "storage_account_key"
+ const: "storage_account_key"
+ enum:
+ - "storage_account_key"
+ type: "string"
+ azure_blob_storage_account_key:
+ title: "Azure Blob Storage account key"
+ description: "The Azure blob storage account key."
+ airbyte_secret: true
+ examples:
+ - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
+ order: 3
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ required:
+ - "azure_blob_storage_account_key"
+ - "auth_type"
azure_blob_storage_account_name:
title: "Azure Blob Storage account name"
description: "The account's name of the Azure Blob Storage."
examples:
- "airbyte5storage"
- order: 2
- type: "string"
- azure_blob_storage_account_key:
- title: "Azure Blob Storage account key"
- description: "The Azure blob storage account key."
- airbyte_secret: true
- examples:
- - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
order: 3
type: "string"
- x-speakeasy-param-sensitive: true
azure_blob_storage_container_name:
title: "Azure blob storage container (Bucket) Name"
description: "The name of the Azure blob storage container."
@@ -29307,12 +29222,12 @@ components:
type: "string"
required:
- "streams"
+ - "credentials"
- "azure_blob_storage_account_name"
- - "azure_blob_storage_account_key"
- "azure_blob_storage_container_name"
- "sourceType"
source-azure-blob-storage-update:
- title: "Config"
+ title: "SourceAzureBlobStorageSpec"
description: "NOTE: When this Spec is changed, legacy_config_transformer.py\
\ must also be modified to uptake the changes\nbecause it is responsible for\
\ converting legacy Azure Blob Storage v0 configs into v1 configs using the\
@@ -29700,19 +29615,74 @@ components:
required:
- "name"
- "format"
+ credentials:
+ title: "Authentication"
+ description: "Credentials for connecting to the Azure Blob Storage"
+ type: "object"
+ order: 2
+ oneOf:
+ - title: "Authenticate via Oauth2"
+ type: "object"
+ properties:
+ auth_type:
+ title: "Auth Type"
+ default: "oauth2"
+ const: "oauth2"
+ enum:
+ - "oauth2"
+ type: "string"
+ tenant_id:
+ title: "Tenant ID"
+ description: "Tenant ID of the Microsoft Azure Application user"
+ airbyte_secret: true
+ type: "string"
+ client_id:
+ title: "Client ID"
+ description: "Client ID of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ client_secret:
+ title: "Client Secret"
+ description: "Client Secret of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ refresh_token:
+ title: "Refresh Token"
+ description: "Refresh Token of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ required:
+ - "tenant_id"
+ - "client_id"
+ - "client_secret"
+ - "refresh_token"
+ - "auth_type"
+ - title: "Authenticate via Storage Account Key"
+ type: "object"
+ properties:
+ auth_type:
+ title: "Auth Type"
+ default: "storage_account_key"
+ const: "storage_account_key"
+ enum:
+ - "storage_account_key"
+ type: "string"
+ azure_blob_storage_account_key:
+ title: "Azure Blob Storage account key"
+ description: "The Azure blob storage account key."
+ airbyte_secret: true
+ examples:
+ - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
+ order: 3
+ type: "string"
+ required:
+ - "azure_blob_storage_account_key"
+ - "auth_type"
azure_blob_storage_account_name:
title: "Azure Blob Storage account name"
description: "The account's name of the Azure Blob Storage."
examples:
- "airbyte5storage"
- order: 2
- type: "string"
- azure_blob_storage_account_key:
- title: "Azure Blob Storage account key"
- description: "The Azure blob storage account key."
- airbyte_secret: true
- examples:
- - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
order: 3
type: "string"
azure_blob_storage_container_name:
@@ -29733,8 +29703,8 @@ components:
type: "string"
required:
- "streams"
+ - "credentials"
- "azure_blob_storage_account_name"
- - "azure_blob_storage_account_key"
- "azure_blob_storage_container_name"
source-close-com:
title: "Close.com Spec"
@@ -32208,7 +32178,6 @@ components:
- "tenant_id"
- "client_id"
- "client_secret"
- - "refresh_token"
- title: "Service Key Authentication"
description: "ServiceCredentials class for service key authentication.\n\
This class is structured similarly to OAuthCredentials but for a different\
@@ -32697,7 +32666,6 @@ components:
- "tenant_id"
- "client_id"
- "client_secret"
- - "refresh_token"
- title: "Service Key Authentication"
description: "ServiceCredentials class for service key authentication.\n\
This class is structured similarly to OAuthCredentials but for a different\
@@ -33662,8 +33630,8 @@ components:
api_key:
type: "string"
title: "API Key"
- description: "Iterable API Key. See the docs for more information on how to obtain this key."
+ description: "Iterable API Key. See the docs for more information on how to obtain this key."
airbyte_secret: true
order: 0
x-speakeasy-param-sensitive: true
@@ -33671,7 +33639,7 @@ components:
type: "string"
title: "Start Date"
description: "The date from which you'd like to replicate data for Iterable,\
- \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\
+ \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\
\ will be replicated."
examples:
- "2021-04-01T00:00:00Z"
@@ -33695,15 +33663,15 @@ components:
api_key:
type: "string"
title: "API Key"
- description: "Iterable API Key. See the docs for more information on how to obtain this key."
+ description: "Iterable API Key. See the docs for more information on how to obtain this key."
airbyte_secret: true
order: 0
start_date:
type: "string"
title: "Start Date"
description: "The date from which you'd like to replicate data for Iterable,\
- \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\
+ \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\
\ will be replicated."
examples:
- "2021-04-01T00:00:00Z"
@@ -35065,9 +35033,6 @@ components:
source-notion:
title: "Notion Source Spec"
type: "object"
- required:
- - "credentials"
- - "sourceType"
properties:
start_date:
title: "Start Date"
@@ -35153,8 +35118,6 @@ components:
source-notion-update:
title: "Notion Source Spec"
type: "object"
- required:
- - "credentials"
properties:
start_date:
title: "Start Date"
@@ -35554,116 +35517,130 @@ components:
description: "Whether to join all channels or to sync data only from channels\
\ the bot is already in. If false, you'll need to manually add the bot\
\ to all the channels from which you'd like to sync messages. "
- channel_filter:
- type: "array"
- default: []
- items:
- type: "string"
- minLength: 0
- title: "Channel name filter"
- description: "A channel name list (without leading '#' char) which limit\
- \ the channels from which you'd like to sync. Empty list means no filter."
- examples:
- - "channel_one"
- - "channel_two"
- credentials:
- title: "Authentication mechanism"
- description: "Choose how to authenticate into Slack"
- type: "object"
- oneOf:
- - type: "object"
- title: "Sign in via Slack (OAuth)"
- required:
- - "option_title"
- - "client_id"
- - "client_secret"
- - "access_token"
- properties:
- option_title:
- type: "string"
- const: "Default OAuth2.0 authorization"
- enum:
- - "Default OAuth2.0 authorization"
- client_id:
- type: "string"
- title: "Client ID"
- description: "Slack client_id. See our docs if you need help finding this id."
- client_secret:
- type: "string"
- title: "Client Secret"
- description: "Slack client_secret. See our docs if you need help finding this secret."
- airbyte_secret: true
- access_token:
- type: "string"
- title: "Access token"
- description: "Slack access_token. See our docs if you need help generating the token."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- order: 0
- - type: "object"
- title: "API Token"
- required:
- - "option_title"
- - "api_token"
- properties:
- option_title:
- type: "string"
- const: "API Token Credentials"
- enum:
- - "API Token Credentials"
- api_token:
- type: "string"
- title: "API Token"
- description: "A Slack bot token. See the docs for instructions on how to generate it."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- order: 1
- sourceType:
- title: "slack"
- const: "slack"
- enum:
- - "slack"
- order: 0
- type: "string"
- source-slack-update:
- title: "Slack Spec"
- type: "object"
- required:
- - "start_date"
- - "lookback_window"
- - "join_channels"
- properties:
- start_date:
- type: "string"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
- \ data before this date will not be replicated."
- examples:
- - "2017-01-25T00:00:00Z"
- title: "Start Date"
- format: "date-time"
- lookback_window:
- type: "integer"
- title: "Threads Lookback window (Days)"
- description: "How far into the past to look for messages in threads, default\
- \ is 0 days"
- examples:
- - 7
- - 14
- minimum: 0
- default: 0
- maximum: 365
- join_channels:
+ include_private_channels:
type: "boolean"
- default: true
- title: "Join all channels"
- description: "Whether to join all channels or to sync data only from channels\
- \ the bot is already in. If false, you'll need to manually add the bot\
- \ to all the channels from which you'd like to sync messages. "
+ default: false
+ title: "Include private channels"
+ description: "Whether to read information from private channels that the\
+ \ bot is already in. If false, only public channels will be read. If\
+ \ true, the bot must be manually added to private channels. "
+ channel_filter:
+ type: "array"
+ default: []
+ items:
+ type: "string"
+ minLength: 0
+ title: "Channel name filter"
+ description: "A channel name list (without leading '#' char) which limit\
+ \ the channels from which you'd like to sync. Empty list means no filter."
+ examples:
+ - "channel_one"
+ - "channel_two"
+ credentials:
+ title: "Authentication mechanism"
+ description: "Choose how to authenticate into Slack"
+ type: "object"
+ oneOf:
+ - type: "object"
+ title: "Sign in via Slack (OAuth)"
+ required:
+ - "option_title"
+ - "client_id"
+ - "client_secret"
+ - "access_token"
+ properties:
+ option_title:
+ type: "string"
+ const: "Default OAuth2.0 authorization"
+ enum:
+ - "Default OAuth2.0 authorization"
+ client_id:
+ type: "string"
+ title: "Client ID"
+ description: "Slack client_id. See our docs if you need help finding this id."
+ client_secret:
+ type: "string"
+ title: "Client Secret"
+ description: "Slack client_secret. See our docs if you need help finding this secret."
+ airbyte_secret: true
+ access_token:
+ type: "string"
+ title: "Access token"
+ description: "Slack access_token. See our docs if you need help generating the token."
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ order: 0
+ - type: "object"
+ title: "API Token"
+ required:
+ - "option_title"
+ - "api_token"
+ properties:
+ option_title:
+ type: "string"
+ const: "API Token Credentials"
+ enum:
+ - "API Token Credentials"
+ api_token:
+ type: "string"
+ title: "API Token"
+ description: "A Slack bot token. See the docs for instructions on how to generate it."
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ order: 1
+ sourceType:
+ title: "slack"
+ const: "slack"
+ enum:
+ - "slack"
+ order: 0
+ type: "string"
+ source-slack-update:
+ title: "Slack Spec"
+ type: "object"
+ required:
+ - "start_date"
+ - "lookback_window"
+ - "join_channels"
+ properties:
+ start_date:
+ type: "string"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
+ \ data before this date will not be replicated."
+ examples:
+ - "2017-01-25T00:00:00Z"
+ title: "Start Date"
+ format: "date-time"
+ lookback_window:
+ type: "integer"
+ title: "Threads Lookback window (Days)"
+ description: "How far into the past to look for messages in threads, default\
+ \ is 0 days"
+ examples:
+ - 7
+ - 14
+ minimum: 0
+ default: 0
+ maximum: 365
+ join_channels:
+ type: "boolean"
+ default: true
+ title: "Join all channels"
+ description: "Whether to join all channels or to sync data only from channels\
+ \ the bot is already in. If false, you'll need to manually add the bot\
+ \ to all the channels from which you'd like to sync messages. "
+ include_private_channels:
+ type: "boolean"
+ default: false
+ title: "Include private channels"
+ description: "Whether to read information from private channels that the\
+ \ bot is already in. If false, only public channels will be read. If\
+ \ true, the bot must be manually added to private channels. "
channel_filter:
type: "array"
default: []
@@ -36705,6 +36682,21 @@ components:
- "credentials"
- "sourceType"
properties:
+ batch_size:
+ type: "integer"
+ title: "Row Batch Size"
+ description: "Default value is 200. An integer representing row batch size\
+ \ for each sent request to Google Sheets API. Row batch size means how\
+ \ many rows are processed from the google sheet, for example default value\
+ \ 200 would process rows 1-201, then 201-401 and so on. Based on Google\
+ \ Sheets API limits documentation, it is possible to send up to 300\
+ \ requests per minute, but each individual request has to be processed\
+ \ under 180 seconds, otherwise the request returns a timeout error. In\
+ \ regards to this information, consider network speed and number of columns\
+ \ of the google sheet when deciding a batch_size value. Default value\
+ \ should cover most of the cases, but if a google sheet has over 100,000\
+ \ records or more, consider increasing batch_size value."
+ default: 200
spreadsheet_id:
type: "string"
title: "Spreadsheet Link"
@@ -36794,6 +36786,21 @@ components:
- "spreadsheet_id"
- "credentials"
properties:
+ batch_size:
+ type: "integer"
+ title: "Row Batch Size"
+ description: "Default value is 200. An integer representing row batch size\
+ \ for each sent request to Google Sheets API. Row batch size means how\
+ \ many rows are processed from the google sheet, for example default value\
+ \ 200 would process rows 1-201, then 201-401 and so on. Based on Google\
+ \ Sheets API limits documentation, it is possible to send up to 300\
+ \ requests per minute, but each individual request has to be processed\
+ \ under 180 seconds, otherwise the request returns a timeout error. In\
+ \ regards to this information, consider network speed and number of columns\
+ \ of the google sheet when deciding a batch_size value. Default value\
+ \ should cover most of the cases, but if a google sheet has over 100,000\
+ \ records or more, consider increasing batch_size value."
+ default: 200
spreadsheet_id:
type: "string"
title: "Spreadsheet Link"
@@ -39912,434 +39919,10 @@ components:
format: "date-time"
order: 3
source-sftp-bulk:
- title: "FTP Source Spec"
- type: "object"
- required:
- - "username"
- - "host"
- - "port"
- - "stream_name"
- - "start_date"
- - "folder_path"
- - "sourceType"
- properties:
- username:
- title: "User Name"
- description: "The server user"
- type: "string"
- order: 0
- password:
- title: "Password"
- description: "OS-level password for logging into the jump server host"
- type: "string"
- airbyte_secret: true
- order: 1
- x-speakeasy-param-sensitive: true
- private_key:
- title: "Private key"
- description: "The private key"
- type: "string"
- multiline: true
- order: 2
- x-speakeasy-param-sensitive: true
- host:
- title: "Host Address"
- description: "The server host address"
- type: "string"
- examples:
- - "www.host.com"
- - "192.0.2.1"
- order: 3
- port:
- title: "Port"
- description: "The server port"
- type: "integer"
- default: 22
- examples:
- - "22"
- order: 4
- stream_name:
- title: "Stream name"
- description: "The name of the stream or table you want to create"
- type: "string"
- examples:
- - "ftp_contacts"
- order: 5
- file_type:
- title: "File type"
- description: "The file type you want to sync. Currently only 'csv' and 'json'\
- \ files are supported."
- type: "string"
- default: "csv"
- enum:
- - "csv"
- - "json"
- order: 6
- examples:
- - "csv"
- - "json"
- separator:
- title: "CSV Separator (Optional)"
- description: "The separator used in the CSV files. Define None if you want\
- \ to use the Sniffer functionality"
- type: "string"
- default: ","
- examples:
- - ","
- order: 7
- folder_path:
- title: "Folder Path (Optional)"
- description: "The directory to search files for sync"
- type: "string"
- default: ""
- examples:
- - "/logs/2022"
- order: 8
- file_pattern:
- title: "File Pattern (Optional)"
- description: "The regular expression to specify files for sync in a chosen\
- \ Folder Path"
- type: "string"
- default: ""
- examples:
- - "log-([0-9]{4})([0-9]{2})([0-9]{2}) - This will filter files which `log-yearmmdd`"
- order: 9
- file_most_recent:
- title: "Most recent file (Optional)"
- description: "Sync only the most recent file for the configured folder path\
- \ and file pattern"
- type: "boolean"
- default: false
- order: 10
- start_date:
- type: "string"
- title: "Start Date"
- format: "date-time"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- examples:
- - "2017-01-25T00:00:00Z"
- description: "The date from which you'd like to replicate data for all incremental\
- \ streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after\
- \ this date will be replicated."
- order: 11
- sourceType:
- title: "sftp-bulk"
- const: "sftp-bulk"
- enum:
- - "sftp-bulk"
- order: 0
- type: "string"
- source-sftp-bulk-update:
- title: "FTP Source Spec"
- type: "object"
- required:
- - "username"
- - "host"
- - "port"
- - "stream_name"
- - "start_date"
- - "folder_path"
- properties:
- username:
- title: "User Name"
- description: "The server user"
- type: "string"
- order: 0
- password:
- title: "Password"
- description: "OS-level password for logging into the jump server host"
- type: "string"
- airbyte_secret: true
- order: 1
- private_key:
- title: "Private key"
- description: "The private key"
- type: "string"
- multiline: true
- order: 2
- host:
- title: "Host Address"
- description: "The server host address"
- type: "string"
- examples:
- - "www.host.com"
- - "192.0.2.1"
- order: 3
- port:
- title: "Port"
- description: "The server port"
- type: "integer"
- default: 22
- examples:
- - "22"
- order: 4
- stream_name:
- title: "Stream name"
- description: "The name of the stream or table you want to create"
- type: "string"
- examples:
- - "ftp_contacts"
- order: 5
- file_type:
- title: "File type"
- description: "The file type you want to sync. Currently only 'csv' and 'json'\
- \ files are supported."
- type: "string"
- default: "csv"
- enum:
- - "csv"
- - "json"
- order: 6
- examples:
- - "csv"
- - "json"
- separator:
- title: "CSV Separator (Optional)"
- description: "The separator used in the CSV files. Define None if you want\
- \ to use the Sniffer functionality"
- type: "string"
- default: ","
- examples:
- - ","
- order: 7
- folder_path:
- title: "Folder Path (Optional)"
- description: "The directory to search files for sync"
- type: "string"
- default: ""
- examples:
- - "/logs/2022"
- order: 8
- file_pattern:
- title: "File Pattern (Optional)"
- description: "The regular expression to specify files for sync in a chosen\
- \ Folder Path"
- type: "string"
- default: ""
- examples:
- - "log-([0-9]{4})([0-9]{2})([0-9]{2}) - This will filter files which `log-yearmmdd`"
- order: 9
- file_most_recent:
- title: "Most recent file (Optional)"
- description: "Sync only the most recent file for the configured folder path\
- \ and file pattern"
- type: "boolean"
- default: false
- order: 10
- start_date:
- type: "string"
- title: "Start Date"
- format: "date-time"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- examples:
- - "2017-01-25T00:00:00Z"
- description: "The date from which you'd like to replicate data for all incremental\
- \ streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after\
- \ this date will be replicated."
- order: 11
- source-zendesk-support:
- title: "Source Zendesk Support Spec"
- type: "object"
- required:
- - "subdomain"
- - "sourceType"
- properties:
- start_date:
- type: "string"
- title: "Start Date"
- description: "The UTC date and time from which you'd like to replicate data,\
- \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\
- \ will be replicated."
- examples:
- - "2020-10-15T00:00:00Z"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- pattern_descriptor: "YYYY-MM-DDTHH:mm:ssZ"
- format: "date-time"
- order: 2
- subdomain:
- type: "string"
- title: "Subdomain"
- description: "This is your unique Zendesk subdomain that can be found in\
- \ your account URL. For example, in https://MY_SUBDOMAIN.zendesk.com/,\
- \ MY_SUBDOMAIN is the value of your subdomain."
- order: 0
- credentials:
- title: "Authentication"
- type: "object"
- description: "Zendesk allows two authentication methods. We recommend using\
- \ `OAuth2.0` for Airbyte Cloud users and `API token` for Airbyte Open\
- \ Source users."
- order: 1
- oneOf:
- - title: "OAuth2.0"
- type: "object"
- required:
- - "access_token"
- additionalProperties: true
- properties:
- credentials:
- type: "string"
- const: "oauth2.0"
- order: 0
- enum:
- - "oauth2.0"
- access_token:
- type: "string"
- title: "Access Token"
- description: "The OAuth access token. See the Zendesk docs for more information on generating this token."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- client_id:
- type: "string"
- title: "Client ID"
- description: "The OAuth client's ID. See this guide for more information."
- airbyte_secret: true
- client_secret:
- type: "string"
- title: "Client Secret"
- description: "The OAuth client secret. See this guide for more information."
- airbyte_secret: true
- - title: "API Token"
- type: "object"
- required:
- - "email"
- - "api_token"
- additionalProperties: true
- properties:
- credentials:
- type: "string"
- const: "api_token"
- order: 0
- enum:
- - "api_token"
- email:
- title: "Email"
- type: "string"
- description: "The user email for your Zendesk account."
- api_token:
- title: "API Token"
- type: "string"
- description: "The value of the API token generated. See our full documentation for more information on generating this\
- \ token."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- ignore_pagination:
- type: "boolean"
- default: false
- description: "Makes each stream read a single page of data."
- title: "Should the connector read the second and further pages of data."
- airbyte_hidden: true
- sourceType:
- title: "zendesk-support"
- const: "zendesk-support"
- enum:
- - "zendesk-support"
- order: 0
- type: "string"
- source-zendesk-support-update:
- title: "Source Zendesk Support Spec"
- type: "object"
- required:
- - "subdomain"
- properties:
- start_date:
- type: "string"
- title: "Start Date"
- description: "The UTC date and time from which you'd like to replicate data,\
- \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\
- \ will be replicated."
- examples:
- - "2020-10-15T00:00:00Z"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- pattern_descriptor: "YYYY-MM-DDTHH:mm:ssZ"
- format: "date-time"
- order: 2
- subdomain:
- type: "string"
- title: "Subdomain"
- description: "This is your unique Zendesk subdomain that can be found in\
- \ your account URL. For example, in https://MY_SUBDOMAIN.zendesk.com/,\
- \ MY_SUBDOMAIN is the value of your subdomain."
- order: 0
- credentials:
- title: "Authentication"
- type: "object"
- description: "Zendesk allows two authentication methods. We recommend using\
- \ `OAuth2.0` for Airbyte Cloud users and `API token` for Airbyte Open\
- \ Source users."
- order: 1
- oneOf:
- - title: "OAuth2.0"
- type: "object"
- required:
- - "access_token"
- additionalProperties: true
- properties:
- credentials:
- type: "string"
- const: "oauth2.0"
- order: 0
- enum:
- - "oauth2.0"
- access_token:
- type: "string"
- title: "Access Token"
- description: "The OAuth access token. See the Zendesk docs for more information on generating this token."
- airbyte_secret: true
- client_id:
- type: "string"
- title: "Client ID"
- description: "The OAuth client's ID. See this guide for more information."
- airbyte_secret: true
- client_secret:
- type: "string"
- title: "Client Secret"
- description: "The OAuth client secret. See this guide for more information."
- airbyte_secret: true
- - title: "API Token"
- type: "object"
- required:
- - "email"
- - "api_token"
- additionalProperties: true
- properties:
- credentials:
- type: "string"
- const: "api_token"
- order: 0
- enum:
- - "api_token"
- email:
- title: "Email"
- type: "string"
- description: "The user email for your Zendesk account."
- api_token:
- title: "API Token"
- type: "string"
- description: "The value of the API token generated. See our full documentation for more information on generating this\
- \ token."
- airbyte_secret: true
- ignore_pagination:
- type: "boolean"
- default: false
- description: "Makes each stream read a single page of data."
- title: "Should the connector read the second and further pages of data."
- airbyte_hidden: true
- source-microsoft-onedrive:
- title: "Microsoft OneDrive Source Spec"
- description: "SourceMicrosoftOneDriveSpec class for Microsoft OneDrive Source\
- \ Specification.\nThis class combines the authentication details with additional\
- \ configuration for the OneDrive API."
+ title: "SFTP Bulk Source Spec"
+ description: "Used during spec; allows the developer to configure the cloud\
+ \ provider specific options\nthat are needed when users configure a file-based\
+ \ source."
type: "object"
properties:
start_date:
@@ -40382,6 +39965,12 @@ components:
type: "array"
items:
type: "string"
+ legacy_prefix:
+ title: "Legacy Prefix"
+ description: "The path prefix configured in v3 versions of the S3\
+ \ connector. This option is deprecated in favor of a single glob."
+ airbyte_hidden: true
+ type: "string"
validation_policy:
title: "Validation Policy"
description: "The name of the validation policy that dictates sync\
@@ -40599,6 +40188,22 @@ components:
items:
type: "string"
uniqueItems: true
+ inference_type:
+ title: "Inference Type"
+ description: "How to infer the types of the columns. If none,\
+ \ inference default to strings."
+ default: "None"
+ airbyte_hidden: true
+ enum:
+ - "None"
+ - "Primitive Types Only"
+ ignore_errors_on_fields_mismatch:
+ title: "Ignore errors on field mismatch"
+ description: "Whether to ignore errors that occur when the number\
+ \ of fields in the CSV does not match the number of columns\
+ \ in the schema."
+ default: false
+ type: "boolean"
required:
- "filetype"
- title: "Jsonl Format"
@@ -40689,6 +40294,66 @@ components:
\ `ocr` modes. This is the default option."
required:
- "mode"
+ - title: "via API"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "api"
+ const: "api"
+ enum:
+ - "api"
+ type: "string"
+ api_key:
+ title: "API Key"
+ description: "The API key to use matching the environment"
+ default: ""
+ always_show: true
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ api_url:
+ title: "API URL"
+ description: "The URL of the unstructured API to use"
+ default: "https://api.unstructured.io"
+ always_show: true
+ examples:
+ - "https://api.unstructured.com"
+ type: "string"
+ parameters:
+ title: "Additional URL Parameters"
+ description: "List of parameters send to the API"
+ default: []
+ always_show: true
+ type: "array"
+ items:
+ title: "APIParameterConfigModel"
+ type: "object"
+ properties:
+ name:
+ title: "Parameter name"
+ description: "The name of the unstructured API parameter\
+ \ to use"
+ examples:
+ - "combine_under_n_chars"
+ - "languages"
+ type: "string"
+ value:
+ title: "Value"
+ description: "The value of the parameter"
+ examples:
+ - "true"
+ - "hi_res"
+ type: "string"
+ required:
+ - "name"
+ - "value"
+ description: "Process files via an API, using the `hi_res`\
+ \ mode. This option is useful for increased performance\
+ \ and accuracy, but requires an API key and a hosted instance\
+ \ of unstructured."
+ required:
+ - "mode"
description: "Extract text from document formats (.pdf, .docx, .md,\
\ .pptx) and emit as one record per file."
required:
@@ -40702,133 +40367,100 @@ components:
required:
- "name"
- "format"
+ host:
+ title: "Host Address"
+ description: "The server host address"
+ examples:
+ - "www.host.com"
+ - "192.0.2.1"
+ order: 2
+ type: "string"
+ username:
+ title: "User Name"
+ description: "The server user"
+ order: 3
+ type: "string"
credentials:
title: "Authentication"
- description: "Credentials for connecting to the One Drive API"
+ description: "Credentials for connecting to the SFTP Server"
type: "object"
- order: 0
+ order: 4
oneOf:
- - title: "Authenticate via Microsoft (OAuth)"
- description: "OAuthCredentials class to hold authentication details for\
- \ Microsoft OAuth authentication.\nThis class uses pydantic for data\
- \ validation and settings management."
+ - title: "Authenticate via Password"
type: "object"
properties:
auth_type:
title: "Auth Type"
- default: "Client"
- const: "Client"
+ default: "password"
+ const: "password"
enum:
- - "Client"
- type: "string"
- tenant_id:
- title: "Tenant ID"
- description: "Tenant ID of the Microsoft OneDrive user"
- airbyte_secret: true
- type: "string"
- client_id:
- title: "Client ID"
- description: "Client ID of your Microsoft developer application"
- airbyte_secret: true
- type: "string"
- client_secret:
- title: "Client Secret"
- description: "Client Secret of your Microsoft developer application"
- airbyte_secret: true
+ - "password"
type: "string"
- refresh_token:
- title: "Refresh Token"
- description: "Refresh Token of your Microsoft developer application"
+ password:
+ title: "Password"
+ description: "Password"
airbyte_secret: true
+ order: 3
type: "string"
x-speakeasy-param-sensitive: true
required:
- - "tenant_id"
- - "client_id"
- - "client_secret"
- - "refresh_token"
- - title: "Service Key Authentication"
- description: "ServiceCredentials class for service key authentication.\n\
- This class is structured similarly to OAuthCredentials but for a different\
- \ authentication method."
+ - "password"
+ - "auth_type"
+ - title: "Authenticate via Private Key"
type: "object"
properties:
auth_type:
title: "Auth Type"
- default: "Service"
- const: "Service"
+ default: "private_key"
+ const: "private_key"
enum:
- - "Service"
- type: "string"
- tenant_id:
- title: "Tenant ID"
- description: "Tenant ID of the Microsoft OneDrive user"
- airbyte_secret: true
- type: "string"
- user_principal_name:
- title: "User Principal Name"
- description: "Special characters such as a period, comma, space, and\
- \ the at sign (@) are converted to underscores (_). More details:\
- \ https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls"
- airbyte_secret: true
- type: "string"
- client_id:
- title: "Client ID"
- description: "Client ID of your Microsoft developer application"
- airbyte_secret: true
+ - "private_key"
type: "string"
- client_secret:
- title: "Client Secret"
- description: "Client Secret of your Microsoft developer application"
- airbyte_secret: true
+ private_key:
+ title: "Private key"
+ description: "The Private key"
+ multiline: true
+ order: 4
type: "string"
+ x-speakeasy-param-sensitive: true
required:
- - "tenant_id"
- - "user_principal_name"
- - "client_id"
- - "client_secret"
- drive_name:
- title: "Drive Name"
- description: "Name of the Microsoft OneDrive drive where the file(s) exist."
- default: "OneDrive"
- order: 2
- type: "string"
- search_scope:
- title: "Search Scope"
- description: "Specifies the location(s) to search for files. Valid options\
- \ are 'ACCESSIBLE_DRIVES' to search in the selected OneDrive drive, 'SHARED_ITEMS'\
- \ for shared items the user has access to, and 'ALL' to search both."
- default: "ALL"
- enum:
- - "ACCESSIBLE_DRIVES"
- - "SHARED_ITEMS"
- - "ALL"
- order: 3
- type: "string"
+ - "private_key"
+ - "auth_type"
+ port:
+ title: "Host Address"
+ description: "The server port"
+ default: 22
+ examples:
+ - "22"
+ order: 5
+ type: "integer"
folder_path:
title: "Folder Path"
- description: "Path to a specific folder within the drives to search for\
- \ files. Leave empty to search all folders of the drives. This does not\
- \ apply to shared items."
- default: "."
- order: 4
+ description: "The directory to search files for sync"
+ default: "/"
+ examples:
+ - "/logs/2022"
+ order: 6
+ pattern_descriptor: "/folder_to_sync"
type: "string"
sourceType:
- title: "microsoft-onedrive"
- const: "microsoft-onedrive"
+ title: "sftp-bulk"
+ const: "sftp-bulk"
enum:
- - "microsoft-onedrive"
+ - "sftp-bulk"
order: 0
type: "string"
required:
- "streams"
+ - "host"
+ - "username"
- "credentials"
- "sourceType"
- source-microsoft-onedrive-update:
- title: "Microsoft OneDrive Source Spec"
- description: "SourceMicrosoftOneDriveSpec class for Microsoft OneDrive Source\
- \ Specification.\nThis class combines the authentication details with additional\
- \ configuration for the OneDrive API."
+ source-sftp-bulk-update:
+ title: "SFTP Bulk Source Spec"
+ description: "Used during spec; allows the developer to configure the cloud\
+ \ provider specific options\nthat are needed when users configure a file-based\
+ \ source."
type: "object"
properties:
start_date:
@@ -40871,6 +40503,12 @@ components:
type: "array"
items:
type: "string"
+ legacy_prefix:
+ title: "Legacy Prefix"
+ description: "The path prefix configured in v3 versions of the S3\
+ \ connector. This option is deprecated in favor of a single glob."
+ airbyte_hidden: true
+ type: "string"
validation_policy:
title: "Validation Policy"
description: "The name of the validation policy that dictates sync\
@@ -41087,6 +40725,22 @@ components:
items:
type: "string"
uniqueItems: true
+ inference_type:
+ title: "Inference Type"
+ description: "How to infer the types of the columns. If none,\
+ \ inference default to strings."
+ default: "None"
+ airbyte_hidden: true
+ enum:
+ - "None"
+ - "Primitive Types Only"
+ ignore_errors_on_fields_mismatch:
+ title: "Ignore errors on field mismatch"
+ description: "Whether to ignore errors that occur when the number\
+ \ of fields in the CSV does not match the number of columns\
+ \ in the schema."
+ default: false
+ type: "boolean"
required:
- "filetype"
- title: "Jsonl Format"
@@ -41177,6 +40831,65 @@ components:
\ `ocr` modes. This is the default option."
required:
- "mode"
+ - title: "via API"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "api"
+ const: "api"
+ enum:
+ - "api"
+ type: "string"
+ api_key:
+ title: "API Key"
+ description: "The API key to use matching the environment"
+ default: ""
+ always_show: true
+ airbyte_secret: true
+ type: "string"
+ api_url:
+ title: "API URL"
+ description: "The URL of the unstructured API to use"
+ default: "https://api.unstructured.io"
+ always_show: true
+ examples:
+ - "https://api.unstructured.com"
+ type: "string"
+ parameters:
+ title: "Additional URL Parameters"
+ description: "List of parameters send to the API"
+ default: []
+ always_show: true
+ type: "array"
+ items:
+ title: "APIParameterConfigModel"
+ type: "object"
+ properties:
+ name:
+ title: "Parameter name"
+ description: "The name of the unstructured API parameter\
+ \ to use"
+ examples:
+ - "combine_under_n_chars"
+ - "languages"
+ type: "string"
+ value:
+ title: "Value"
+ description: "The value of the parameter"
+ examples:
+ - "true"
+ - "hi_res"
+ type: "string"
+ required:
+ - "name"
+ - "value"
+ description: "Process files via an API, using the `hi_res`\
+ \ mode. This option is useful for increased performance\
+ \ and accuracy, but requires an API key and a hosted instance\
+ \ of unstructured."
+ required:
+ - "mode"
description: "Extract text from document formats (.pdf, .docx, .md,\
\ .pptx) and emit as one record per file."
required:
@@ -41190,1452 +40903,2584 @@ components:
required:
- "name"
- "format"
+ host:
+ title: "Host Address"
+ description: "The server host address"
+ examples:
+ - "www.host.com"
+ - "192.0.2.1"
+ order: 2
+ type: "string"
+ username:
+ title: "User Name"
+ description: "The server user"
+ order: 3
+ type: "string"
credentials:
title: "Authentication"
- description: "Credentials for connecting to the One Drive API"
+ description: "Credentials for connecting to the SFTP Server"
type: "object"
- order: 0
+ order: 4
oneOf:
- - title: "Authenticate via Microsoft (OAuth)"
- description: "OAuthCredentials class to hold authentication details for\
- \ Microsoft OAuth authentication.\nThis class uses pydantic for data\
- \ validation and settings management."
+ - title: "Authenticate via Password"
type: "object"
properties:
auth_type:
title: "Auth Type"
- default: "Client"
- const: "Client"
+ default: "password"
+ const: "password"
enum:
- - "Client"
- type: "string"
- tenant_id:
- title: "Tenant ID"
- description: "Tenant ID of the Microsoft OneDrive user"
- airbyte_secret: true
- type: "string"
- client_id:
- title: "Client ID"
- description: "Client ID of your Microsoft developer application"
- airbyte_secret: true
- type: "string"
- client_secret:
- title: "Client Secret"
- description: "Client Secret of your Microsoft developer application"
- airbyte_secret: true
+ - "password"
type: "string"
- refresh_token:
- title: "Refresh Token"
- description: "Refresh Token of your Microsoft developer application"
+ password:
+ title: "Password"
+ description: "Password"
airbyte_secret: true
+ order: 3
type: "string"
required:
- - "tenant_id"
- - "client_id"
- - "client_secret"
- - "refresh_token"
- - title: "Service Key Authentication"
- description: "ServiceCredentials class for service key authentication.\n\
- This class is structured similarly to OAuthCredentials but for a different\
- \ authentication method."
+ - "password"
+ - "auth_type"
+ - title: "Authenticate via Private Key"
type: "object"
properties:
auth_type:
title: "Auth Type"
- default: "Service"
- const: "Service"
+ default: "private_key"
+ const: "private_key"
enum:
- - "Service"
- type: "string"
- tenant_id:
- title: "Tenant ID"
- description: "Tenant ID of the Microsoft OneDrive user"
- airbyte_secret: true
- type: "string"
- user_principal_name:
- title: "User Principal Name"
- description: "Special characters such as a period, comma, space, and\
- \ the at sign (@) are converted to underscores (_). More details:\
- \ https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls"
- airbyte_secret: true
- type: "string"
- client_id:
- title: "Client ID"
- description: "Client ID of your Microsoft developer application"
- airbyte_secret: true
+ - "private_key"
type: "string"
- client_secret:
- title: "Client Secret"
- description: "Client Secret of your Microsoft developer application"
- airbyte_secret: true
+ private_key:
+ title: "Private key"
+ description: "The Private key"
+ multiline: true
+ order: 4
type: "string"
required:
- - "tenant_id"
- - "user_principal_name"
- - "client_id"
- - "client_secret"
- drive_name:
- title: "Drive Name"
- description: "Name of the Microsoft OneDrive drive where the file(s) exist."
- default: "OneDrive"
- order: 2
- type: "string"
- search_scope:
- title: "Search Scope"
- description: "Specifies the location(s) to search for files. Valid options\
- \ are 'ACCESSIBLE_DRIVES' to search in the selected OneDrive drive, 'SHARED_ITEMS'\
- \ for shared items the user has access to, and 'ALL' to search both."
- default: "ALL"
- enum:
- - "ACCESSIBLE_DRIVES"
- - "SHARED_ITEMS"
- - "ALL"
- order: 3
- type: "string"
+ - "private_key"
+ - "auth_type"
+ port:
+ title: "Host Address"
+ description: "The server port"
+ default: 22
+ examples:
+ - "22"
+ order: 5
+ type: "integer"
folder_path:
title: "Folder Path"
- description: "Path to a specific folder within the drives to search for\
- \ files. Leave empty to search all folders of the drives. This does not\
- \ apply to shared items."
- default: "."
- order: 4
+ description: "The directory to search files for sync"
+ default: "/"
+ examples:
+ - "/logs/2022"
+ order: 6
+ pattern_descriptor: "/folder_to_sync"
type: "string"
required:
- "streams"
+ - "host"
+ - "username"
- "credentials"
- source-punk-api:
- title: "Punk Api Spec"
+ source-zendesk-support:
+ title: "Source Zendesk Support Spec"
type: "object"
required:
- - "brewed_before"
- - "brewed_after"
+ - "subdomain"
- "sourceType"
properties:
- id:
- title: "Beers with specific ID"
- type: "string"
- description: "To extract specific data with Unique ID"
- examples:
- - 1
- - 22
- brewed_before:
- title: "Brewed before data to get incremental reads"
+ start_date:
type: "string"
- description: "To extract specific data with Unique ID"
- pattern: "^[0-9]{2}-[0-9]{4}$"
+ title: "Start Date"
+ description: "The UTC date and time from which you'd like to replicate data,\
+ \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\
+ \ will be replicated."
examples:
- - "MM-YYYY"
- brewed_after:
- title: "Brewed after data to get incremental reads"
+ - "2020-10-15T00:00:00Z"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ pattern_descriptor: "YYYY-MM-DDTHH:mm:ssZ"
+ format: "date-time"
+ order: 2
+ subdomain:
type: "string"
- description: "To extract specific data with Unique ID"
- pattern: "^[0-9]{2}-[0-9]{4}$"
- examples:
- - "MM-YYYY"
- sourceType:
- title: "punk-api"
- const: "punk-api"
- enum:
- - "punk-api"
+ title: "Subdomain"
+ description: "This is your unique Zendesk subdomain that can be found in\
+ \ your account URL. For example, in https://MY_SUBDOMAIN.zendesk.com/,\
+ \ MY_SUBDOMAIN is the value of your subdomain."
order: 0
- type: "string"
- source-punk-api-update:
- title: "Punk Api Spec"
- type: "object"
- required:
- - "brewed_before"
- - "brewed_after"
- properties:
- id:
- title: "Beers with specific ID"
- type: "string"
- description: "To extract specific data with Unique ID"
- examples:
- - 1
- - 22
- brewed_before:
- title: "Brewed before data to get incremental reads"
- type: "string"
- description: "To extract specific data with Unique ID"
- pattern: "^[0-9]{2}-[0-9]{4}$"
- examples:
- - "MM-YYYY"
- brewed_after:
- title: "Brewed after data to get incremental reads"
- type: "string"
- description: "To extract specific data with Unique ID"
- pattern: "^[0-9]{2}-[0-9]{4}$"
- examples:
- - "MM-YYYY"
- source-tiktok-marketing:
- title: "TikTok Marketing Source Spec"
- type: "object"
- properties:
credentials:
- title: "Authentication Method"
- description: "Authentication method"
- default: {}
- order: 0
+ title: "Authentication"
type: "object"
+ description: "Zendesk allows two authentication methods. We recommend using\
+ \ `OAuth2.0` for Airbyte Cloud users and `API token` for Airbyte Open\
+ \ Source users."
+ order: 1
oneOf:
- title: "OAuth2.0"
type: "object"
+ required:
+ - "access_token"
+ additionalProperties: true
properties:
- auth_type:
- title: "Auth Type"
+ credentials:
+ type: "string"
const: "oauth2.0"
order: 0
- type: "string"
enum:
- "oauth2.0"
- app_id:
- title: "App ID"
- description: "The Developer Application App ID."
- airbyte_secret: true
- type: "string"
- secret:
- title: "Secret"
- description: "The Developer Application Secret."
- airbyte_secret: true
- type: "string"
access_token:
+ type: "string"
title: "Access Token"
- description: "Long-term Authorized Access Token."
+ description: "The OAuth access token. See the Zendesk docs for more information on generating this token."
airbyte_secret: true
- type: "string"
x-speakeasy-param-sensitive: true
- advertiser_id:
- title: "Advertiser ID"
- description: "The Advertiser ID to filter reports and streams. Let\
- \ this empty to retrieve all."
+ client_id:
type: "string"
- required:
- - "app_id"
- - "secret"
- - "access_token"
- - title: "Sandbox Access Token"
+ title: "Client ID"
+ description: "The OAuth client's ID. See this guide for more information."
+ airbyte_secret: true
+ client_secret:
+ type: "string"
+ title: "Client Secret"
+ description: "The OAuth client secret. See this guide for more information."
+ airbyte_secret: true
+ - title: "API Token"
type: "object"
+ required:
+ - "email"
+ - "api_token"
+ additionalProperties: true
properties:
- auth_type:
- title: "Auth Type"
- const: "sandbox_access_token"
- order: 0
+ credentials:
type: "string"
+ const: "api_token"
+ order: 0
enum:
- - "sandbox_access_token"
- advertiser_id:
- title: "Advertiser ID"
- description: "The Advertiser ID which generated for the developer's\
- \ Sandbox application."
+ - "api_token"
+ email:
+ title: "Email"
type: "string"
- access_token:
- title: "Access Token"
- description: "The long-term authorized access token."
- airbyte_secret: true
+ description: "The user email for your Zendesk account."
+ api_token:
+ title: "API Token"
type: "string"
- x-speakeasy-param-sensitive: true
- required:
- - "advertiser_id"
- - "access_token"
- start_date:
- title: "Replication Start Date"
- description: "The Start Date in format: YYYY-MM-DD. Any data before this\
- \ date will not be replicated. If this parameter is not set, all data\
- \ will be replicated."
- default: "2016-09-01"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
- order: 1
- type: "string"
- format: "date"
- end_date:
- title: "End Date"
- description: "The date until which you'd like to replicate data for all\
- \ incremental streams, in the format YYYY-MM-DD. All data generated between\
- \ start_date and this date will be replicated. Not setting this option\
- \ will result in always syncing the data till the current date."
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
- order: 2
- type: "string"
- format: "date"
- attribution_window:
- title: "Attribution Window"
- description: "The attribution window in days."
- minimum: 0
- maximum: 364
- default: 3
- order: 3
- type: "integer"
- include_deleted:
- title: "Include Deleted Data in Reports"
- description: "Set to active if you want to include deleted data in reports."
- default: false
- order: 4
+ description: "The value of the API token generated. See our full documentation for more information on generating this\
+ \ token."
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ ignore_pagination:
type: "boolean"
+ default: false
+ description: "Makes each stream read a single page of data."
+ title: "Should the connector read the second and further pages of data."
+ airbyte_hidden: true
sourceType:
- title: "tiktok-marketing"
- const: "tiktok-marketing"
+ title: "zendesk-support"
+ const: "zendesk-support"
enum:
- - "tiktok-marketing"
+ - "zendesk-support"
order: 0
type: "string"
- source-tiktok-marketing-update:
- title: "TikTok Marketing Source Spec"
+ source-zendesk-support-update:
+ title: "Source Zendesk Support Spec"
type: "object"
+ required:
+ - "subdomain"
properties:
- credentials:
- title: "Authentication Method"
- description: "Authentication method"
- default: {}
+ start_date:
+ type: "string"
+ title: "Start Date"
+ description: "The UTC date and time from which you'd like to replicate data,\
+ \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\
+ \ will be replicated."
+ examples:
+ - "2020-10-15T00:00:00Z"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ pattern_descriptor: "YYYY-MM-DDTHH:mm:ssZ"
+ format: "date-time"
+ order: 2
+ subdomain:
+ type: "string"
+ title: "Subdomain"
+ description: "This is your unique Zendesk subdomain that can be found in\
+ \ your account URL. For example, in https://MY_SUBDOMAIN.zendesk.com/,\
+ \ MY_SUBDOMAIN is the value of your subdomain."
order: 0
+ credentials:
+ title: "Authentication"
type: "object"
+ description: "Zendesk allows two authentication methods. We recommend using\
+ \ `OAuth2.0` for Airbyte Cloud users and `API token` for Airbyte Open\
+ \ Source users."
+ order: 1
oneOf:
- title: "OAuth2.0"
type: "object"
+ required:
+ - "access_token"
+ additionalProperties: true
properties:
- auth_type:
- title: "Auth Type"
+ credentials:
+ type: "string"
const: "oauth2.0"
order: 0
- type: "string"
enum:
- "oauth2.0"
- app_id:
- title: "App ID"
- description: "The Developer Application App ID."
- airbyte_secret: true
- type: "string"
- secret:
- title: "Secret"
- description: "The Developer Application Secret."
- airbyte_secret: true
- type: "string"
access_token:
+ type: "string"
title: "Access Token"
- description: "Long-term Authorized Access Token."
+ description: "The OAuth access token. See the Zendesk docs for more information on generating this token."
airbyte_secret: true
+ client_id:
type: "string"
- advertiser_id:
- title: "Advertiser ID"
- description: "The Advertiser ID to filter reports and streams. Let\
- \ this empty to retrieve all."
+ title: "Client ID"
+ description: "The OAuth client's ID. See this guide for more information."
+ airbyte_secret: true
+ client_secret:
type: "string"
- required:
- - "app_id"
- - "secret"
- - "access_token"
- - title: "Sandbox Access Token"
+ title: "Client Secret"
+ description: "The OAuth client secret. See this guide for more information."
+ airbyte_secret: true
+ - title: "API Token"
type: "object"
+ required:
+ - "email"
+ - "api_token"
+ additionalProperties: true
properties:
- auth_type:
- title: "Auth Type"
- const: "sandbox_access_token"
- order: 0
+ credentials:
type: "string"
+ const: "api_token"
+ order: 0
enum:
- - "sandbox_access_token"
- advertiser_id:
- title: "Advertiser ID"
- description: "The Advertiser ID which generated for the developer's\
- \ Sandbox application."
+ - "api_token"
+ email:
+ title: "Email"
type: "string"
- access_token:
- title: "Access Token"
- description: "The long-term authorized access token."
- airbyte_secret: true
+ description: "The user email for your Zendesk account."
+ api_token:
+ title: "API Token"
type: "string"
- required:
- - "advertiser_id"
- - "access_token"
- start_date:
- title: "Replication Start Date"
- description: "The Start Date in format: YYYY-MM-DD. Any data before this\
- \ date will not be replicated. If this parameter is not set, all data\
- \ will be replicated."
- default: "2016-09-01"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
- order: 1
- type: "string"
- format: "date"
- end_date:
- title: "End Date"
- description: "The date until which you'd like to replicate data for all\
- \ incremental streams, in the format YYYY-MM-DD. All data generated between\
- \ start_date and this date will be replicated. Not setting this option\
- \ will result in always syncing the data till the current date."
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
- order: 2
- type: "string"
- format: "date"
- attribution_window:
- title: "Attribution Window"
- description: "The attribution window in days."
- minimum: 0
- maximum: 364
- default: 3
- order: 3
- type: "integer"
- include_deleted:
- title: "Include Deleted Data in Reports"
- description: "Set to active if you want to include deleted data in reports."
- default: false
- order: 4
- type: "boolean"
- source-aws-cloudtrail:
- title: "Aws CloudTrail Spec"
- type: "object"
- required:
- - "aws_key_id"
- - "aws_secret_key"
- - "aws_region_name"
- - "start_date"
- - "sourceType"
- properties:
- aws_key_id:
- type: "string"
- title: "Key ID"
- description: "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- aws_secret_key:
- type: "string"
- title: "Secret Key"
- description: "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- aws_region_name:
- type: "string"
- title: "Region Name"
- description: "The default AWS Region to use, for example, us-west-1 or us-west-2.\
- \ When specifying a Region inline during client initialization, this property\
- \ is named region_name."
- start_date:
- type: "string"
- title: "Start Date"
- description: "The date you would like to replicate data. Data in AWS CloudTrail\
- \ is available for last 90 days only. Format: YYYY-MM-DD."
- examples:
- - "2021-01-01"
- default: "1970-01-01"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
- format: "date"
- sourceType:
- title: "aws-cloudtrail"
- const: "aws-cloudtrail"
- enum:
- - "aws-cloudtrail"
- order: 0
- type: "string"
- source-aws-cloudtrail-update:
- title: "Aws CloudTrail Spec"
- type: "object"
- required:
- - "aws_key_id"
- - "aws_secret_key"
- - "aws_region_name"
- - "start_date"
- properties:
- aws_key_id:
- type: "string"
- title: "Key ID"
- description: "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key."
- airbyte_secret: true
- aws_secret_key:
- type: "string"
- title: "Secret Key"
- description: "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key."
- airbyte_secret: true
- aws_region_name:
- type: "string"
- title: "Region Name"
- description: "The default AWS Region to use, for example, us-west-1 or us-west-2.\
- \ When specifying a Region inline during client initialization, this property\
- \ is named region_name."
- start_date:
- type: "string"
- title: "Start Date"
- description: "The date you would like to replicate data. Data in AWS CloudTrail\
- \ is available for last 90 days only. Format: YYYY-MM-DD."
- examples:
- - "2021-01-01"
- default: "1970-01-01"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
- format: "date"
- source-jira:
- title: "Jira Spec"
- type: "object"
- required:
- - "api_token"
- - "domain"
- - "email"
- - "sourceType"
- properties:
- api_token:
- type: "string"
- title: "API Token"
- description: "Jira API Token. See the docs for more information on how to generate this key. API Token\
- \ is used for Authorization to your account by BasicAuth."
- airbyte_secret: true
- order: 0
- x-speakeasy-param-sensitive: true
- domain:
- type: "string"
- title: "Domain"
- examples:
- - ".atlassian.net"
- - ".jira.com"
- - "jira..com"
- description: "The Domain for your Jira account, e.g. airbyteio.atlassian.net,\
- \ airbyteio.jira.com, jira.your-domain.com"
- order: 1
- email:
- type: "string"
- title: "Email"
- description: "The user email for your Jira account which you used to generate\
- \ the API token. This field is used for Authorization to your account\
- \ by BasicAuth."
- order: 2
- projects:
- type: "array"
- title: "Projects"
- items:
- type: "string"
- examples:
- - "PROJ1"
- - "PROJ2"
- description: "List of Jira project keys to replicate data for, or leave\
- \ it empty if you want to replicate data for all projects."
- order: 3
- start_date:
- type: "string"
- title: "Start Date"
- description: "The date from which you want to replicate data from Jira,\
- \ use the format YYYY-MM-DDT00:00:00Z. Note that this field only applies\
- \ to certain streams, and only data generated on or after the start date\
- \ will be replicated. Or leave it empty if you want to replicate all data.\
- \ For more information, refer to the documentation."
- examples:
- - "2021-03-01T00:00:00Z"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- format: "date-time"
- order: 4
- expand_issue_changelog:
- type: "boolean"
- title: "Expand Issue Changelog"
- airbyte_hidden: true
- description: "(DEPRECATED) Expand the changelog when replicating issues."
- default: false
- render_fields:
- type: "boolean"
- title: "Render Issue Fields"
- airbyte_hidden: true
- description: "(DEPRECATED) Render issue fields in HTML format in addition\
- \ to Jira JSON-like format."
- default: false
- expand_issue_transition:
+ description: "The value of the API token generated. See our full documentation for more information on generating this\
+ \ token."
+ airbyte_secret: true
+ ignore_pagination:
type: "boolean"
- title: "Expand Issue Transitions"
- airbyte_hidden: true
- description: "(DEPRECATED) Expand the transitions when replicating issues."
default: false
- issues_stream_expand_with:
- type: "array"
- items:
- type: "string"
- enum:
- - "renderedFields"
- - "transitions"
- - "changelog"
- title: "Expand Issues stream"
+ description: "Makes each stream read a single page of data."
+ title: "Should the connector read the second and further pages of data."
airbyte_hidden: true
- description: "Select fields to Expand the `Issues` stream when replicating\
- \ with: "
- default: []
- lookback_window_minutes:
- title: "Lookback window"
- description: "When set to N, the connector will always refresh resources\
- \ created within the past N minutes. By default, updated objects that\
- \ are not newly created are not incrementally synced."
- examples:
- - 60
- default: 0
- minimum: 0
- maximum: 576000
- type: "integer"
- order: 5
- enable_experimental_streams:
- type: "boolean"
- title: "Enable Experimental Streams"
- description: "Allow the use of experimental streams which rely on undocumented\
- \ Jira API endpoints. See https://docs.airbyte.com/integrations/sources/jira#experimental-tables\
- \ for more info."
- default: false
- order: 6
- sourceType:
- title: "jira"
- const: "jira"
- enum:
- - "jira"
- order: 0
- type: "string"
- source-jira-update:
- title: "Jira Spec"
+ source-microsoft-onedrive:
+ title: "Microsoft OneDrive Source Spec"
+ description: "SourceMicrosoftOneDriveSpec class for Microsoft OneDrive Source\
+ \ Specification.\nThis class combines the authentication details with additional\
+ \ configuration for the OneDrive API."
type: "object"
- required:
- - "api_token"
- - "domain"
- - "email"
properties:
- api_token:
- type: "string"
- title: "API Token"
- description: "Jira API Token. See the docs for more information on how to generate this key. API Token\
- \ is used for Authorization to your account by BasicAuth."
- airbyte_secret: true
- order: 0
- domain:
- type: "string"
- title: "Domain"
- examples:
- - ".atlassian.net"
- - ".jira.com"
- - "jira..com"
- description: "The Domain for your Jira account, e.g. airbyteio.atlassian.net,\
- \ airbyteio.jira.com, jira.your-domain.com"
- order: 1
- email:
- type: "string"
- title: "Email"
- description: "The user email for your Jira account which you used to generate\
- \ the API token. This field is used for Authorization to your account\
- \ by BasicAuth."
- order: 2
- projects:
- type: "array"
- title: "Projects"
- items:
- type: "string"
- examples:
- - "PROJ1"
- - "PROJ2"
- description: "List of Jira project keys to replicate data for, or leave\
- \ it empty if you want to replicate data for all projects."
- order: 3
start_date:
- type: "string"
title: "Start Date"
- description: "The date from which you want to replicate data from Jira,\
- \ use the format YYYY-MM-DDT00:00:00Z. Note that this field only applies\
- \ to certain streams, and only data generated on or after the start date\
- \ will be replicated. Or leave it empty if you want to replicate all data.\
- \ For more information, refer to the documentation."
+ description: "UTC date and time in the format 2017-01-25T00:00:00.000000Z.\
+ \ Any file modified before this date will not be replicated."
examples:
- - "2021-03-01T00:00:00Z"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ - "2021-01-01T00:00:00.000000Z"
format: "date-time"
- order: 4
- expand_issue_changelog:
- type: "boolean"
- title: "Expand Issue Changelog"
- airbyte_hidden: true
- description: "(DEPRECATED) Expand the changelog when replicating issues."
- default: false
- render_fields:
- type: "boolean"
- title: "Render Issue Fields"
- airbyte_hidden: true
- description: "(DEPRECATED) Render issue fields in HTML format in addition\
- \ to Jira JSON-like format."
- default: false
- expand_issue_transition:
- type: "boolean"
- title: "Expand Issue Transitions"
- airbyte_hidden: true
- description: "(DEPRECATED) Expand the transitions when replicating issues."
- default: false
- issues_stream_expand_with:
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$"
+ pattern_descriptor: "YYYY-MM-DDTHH:mm:ss.SSSSSSZ"
+ order: 1
+ type: "string"
+ streams:
+ title: "The list of streams to sync"
+ description: "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their\
+ \ format, and how they should be parsed and validated. When sending data\
+ \ to warehouse destination such as Snowflake or BigQuery, each stream\
+ \ is a separate table."
+ order: 10
type: "array"
items:
- type: "string"
- enum:
- - "renderedFields"
- - "transitions"
- - "changelog"
- title: "Expand Issues stream"
- airbyte_hidden: true
- description: "Select fields to Expand the `Issues` stream when replicating\
- \ with: "
- default: []
- lookback_window_minutes:
- title: "Lookback window"
- description: "When set to N, the connector will always refresh resources\
- \ created within the past N minutes. By default, updated objects that\
- \ are not newly created are not incrementally synced."
- examples:
- - 60
- default: 0
- minimum: 0
- maximum: 576000
- type: "integer"
- order: 5
- enable_experimental_streams:
- type: "boolean"
- title: "Enable Experimental Streams"
- description: "Allow the use of experimental streams which rely on undocumented\
- \ Jira API endpoints. See https://docs.airbyte.com/integrations/sources/jira#experimental-tables\
- \ for more info."
- default: false
- order: 6
- source-hubspot:
- title: "HubSpot Source Spec"
- type: "object"
- required:
- - "credentials"
- - "sourceType"
- properties:
- start_date:
- type: "string"
- title: "Start date"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
- \ data before this date will not be replicated. If not set, \"2006-06-01T00:00:00Z\"\
- \ (Hubspot creation date) will be used as start date. It's recommended\
- \ to provide relevant to your data start date value to optimize synchronization."
- examples:
- - "2017-01-25T00:00:00Z"
- format: "date-time"
+ title: "FileBasedStreamConfig"
+ type: "object"
+ properties:
+ name:
+ title: "Name"
+ description: "The name of the stream."
+ type: "string"
+ globs:
+ title: "Globs"
+ description: "The pattern used to specify which files should be selected\
+ \ from the file system. For more information on glob pattern matching\
+ \ look here."
+ default:
+ - "**"
+ order: 1
+ type: "array"
+ items:
+ type: "string"
+ validation_policy:
+ title: "Validation Policy"
+ description: "The name of the validation policy that dictates sync\
+ \ behavior when a record does not adhere to the stream schema."
+ default: "Emit Record"
+ enum:
+ - "Emit Record"
+ - "Skip Record"
+ - "Wait for Discover"
+ input_schema:
+ title: "Input Schema"
+ description: "The schema that will be used to validate records extracted\
+ \ from the file. This will override the stream schema that is auto-detected\
+ \ from incoming files."
+ type: "string"
+ primary_key:
+ title: "Primary Key"
+ description: "The column or columns (for a composite key) that serves\
+ \ as the unique identifier of a record. If empty, the primary key\
+ \ will default to the parser's default primary key."
+ airbyte_hidden: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ days_to_sync_if_history_is_full:
+ title: "Days To Sync If History Is Full"
+ description: "When the state history of the file store is full, syncs\
+ \ will only read files that were last modified in the provided day\
+ \ range."
+ default: 3
+ type: "integer"
+ format:
+ title: "Format"
+ description: "The configuration options that are used to alter how\
+ \ to read incoming files that deviate from the standard formatting."
+ type: "object"
+ oneOf:
+ - title: "Avro Format"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "avro"
+ const: "avro"
+ type: "string"
+ enum:
+ - "avro"
+ double_as_string:
+ title: "Convert Double Fields to Strings"
+ description: "Whether to convert double fields to strings. This\
+ \ is recommended if you have decimal numbers with a high degree\
+ \ of precision because there can be a loss precision when\
+ \ handling floating point numbers."
+ default: false
+ type: "boolean"
+ required:
+ - "filetype"
+ - title: "CSV Format"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "csv"
+ const: "csv"
+ type: "string"
+ enum:
+ - "csv"
+ delimiter:
+ title: "Delimiter"
+ description: "The character delimiting individual cells in the\
+ \ CSV data. This may only be a 1-character string. For tab-delimited\
+ \ data enter '\\t'."
+ default: ","
+ type: "string"
+ quote_char:
+ title: "Quote Character"
+ description: "The character used for quoting CSV values. To\
+ \ disallow quoting, make this field blank."
+ default: "\""
+ type: "string"
+ escape_char:
+ title: "Escape Character"
+ description: "The character used for escaping special characters.\
+ \ To disallow escaping, leave this field blank."
+ type: "string"
+ encoding:
+ title: "Encoding"
+ description: "The character encoding of the CSV data. Leave\
+ \ blank to default to UTF8. See list of python encodings for allowable\
+ \ options."
+ default: "utf8"
+ type: "string"
+ double_quote:
+ title: "Double Quote"
+ description: "Whether two quotes in a quoted CSV value denote\
+ \ a single quote in the data."
+ default: true
+ type: "boolean"
+ null_values:
+ title: "Null Values"
+ description: "A set of case-sensitive strings that should be\
+ \ interpreted as null values. For example, if the value 'NA'\
+ \ should be interpreted as null, enter 'NA' in this field."
+ default: []
+ type: "array"
+ items:
+ type: "string"
+ uniqueItems: true
+ strings_can_be_null:
+ title: "Strings Can Be Null"
+ description: "Whether strings can be interpreted as null values.\
+ \ If true, strings that match the null_values set will be\
+ \ interpreted as null. If false, strings that match the null_values\
+ \ set will be interpreted as the string itself."
+ default: true
+ type: "boolean"
+ skip_rows_before_header:
+ title: "Skip Rows Before Header"
+ description: "The number of rows to skip before the header row.\
+ \ For example, if the header row is on the 3rd row, enter\
+ \ 2 in this field."
+ default: 0
+ type: "integer"
+ skip_rows_after_header:
+ title: "Skip Rows After Header"
+ description: "The number of rows to skip after the header row."
+ default: 0
+ type: "integer"
+ header_definition:
+ title: "CSV Header Definition"
+ description: "How headers will be defined. `User Provided` assumes\
+ \ the CSV does not have a header row and uses the headers\
+ \ provided and `Autogenerated` assumes the CSV does not have\
+ \ a header row and the CDK will generate headers using for\
+ \ `f{i}` where `i` is the index starting from 0. Else, the\
+ \ default behavior is to use the header from the CSV file.\
+ \ If a user wants to autogenerate or provide column names\
+ \ for a CSV having headers, they can skip rows."
+ default:
+ header_definition_type: "From CSV"
+ oneOf:
+ - title: "From CSV"
+ type: "object"
+ properties:
+ header_definition_type:
+ title: "Header Definition Type"
+ default: "From CSV"
+ const: "From CSV"
+ type: "string"
+ enum:
+ - "From CSV"
+ required:
+ - "header_definition_type"
+ - title: "Autogenerated"
+ type: "object"
+ properties:
+ header_definition_type:
+ title: "Header Definition Type"
+ default: "Autogenerated"
+ const: "Autogenerated"
+ type: "string"
+ enum:
+ - "Autogenerated"
+ required:
+ - "header_definition_type"
+ - title: "User Provided"
+ type: "object"
+ properties:
+ header_definition_type:
+ title: "Header Definition Type"
+ default: "User Provided"
+ const: "User Provided"
+ type: "string"
+ enum:
+ - "User Provided"
+ column_names:
+ title: "Column Names"
+ description: "The column names that will be used while\
+ \ emitting the CSV records"
+ type: "array"
+ items:
+ type: "string"
+ required:
+ - "column_names"
+ - "header_definition_type"
+ type: "object"
+ true_values:
+ title: "True Values"
+ description: "A set of case-sensitive strings that should be\
+ \ interpreted as true values."
+ default:
+ - "y"
+ - "yes"
+ - "t"
+ - "true"
+ - "on"
+ - "1"
+ type: "array"
+ items:
+ type: "string"
+ uniqueItems: true
+ false_values:
+ title: "False Values"
+ description: "A set of case-sensitive strings that should be\
+ \ interpreted as false values."
+ default:
+ - "n"
+ - "no"
+ - "f"
+ - "false"
+ - "off"
+ - "0"
+ type: "array"
+ items:
+ type: "string"
+ uniqueItems: true
+ required:
+ - "filetype"
+ - title: "Jsonl Format"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "jsonl"
+ const: "jsonl"
+ type: "string"
+ enum:
+ - "jsonl"
+ required:
+ - "filetype"
+ - title: "Parquet Format"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "parquet"
+ const: "parquet"
+ type: "string"
+ enum:
+ - "parquet"
+ decimal_as_float:
+ title: "Convert Decimal Fields to Floats"
+ description: "Whether to convert decimal fields to floats. There\
+ \ is a loss of precision when converting decimals to floats,\
+ \ so this is not recommended."
+ default: false
+ type: "boolean"
+ required:
+ - "filetype"
+ - title: "Document File Type Format (Experimental)"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "unstructured"
+ const: "unstructured"
+ type: "string"
+ enum:
+ - "unstructured"
+ skip_unprocessable_files:
+ title: "Skip Unprocessable Files"
+ description: "If true, skip files that cannot be parsed and\
+ \ pass the error message along as the _ab_source_file_parse_error\
+ \ field. If false, fail the sync."
+ default: true
+ always_show: true
+ type: "boolean"
+ strategy:
+ title: "Parsing Strategy"
+ description: "The strategy used to parse documents. `fast` extracts\
+ \ text directly from the document which doesn't work for all\
+ \ files. `ocr_only` is more reliable, but slower. `hi_res`\
+ \ is the most reliable, but requires an API key and a hosted\
+ \ instance of unstructured and can't be used with local mode.\
+ \ See the unstructured.io documentation for more details:\
+ \ https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf"
+ default: "auto"
+ always_show: true
+ order: 0
+ enum:
+ - "auto"
+ - "fast"
+ - "ocr_only"
+ - "hi_res"
+ type: "string"
+ processing:
+ title: "Processing"
+ description: "Processing configuration"
+ default:
+ mode: "local"
+ type: "object"
+ oneOf:
+ - title: "Local"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "local"
+ const: "local"
+ enum:
+ - "local"
+ type: "string"
+ description: "Process files locally, supporting `fast` and\
+ \ `ocr` modes. This is the default option."
+ required:
+ - "mode"
+ description: "Extract text from document formats (.pdf, .docx, .md,\
+ \ .pptx) and emit as one record per file."
+ required:
+ - "filetype"
+ schemaless:
+ title: "Schemaless"
+ description: "When enabled, syncs will not validate or structure records\
+ \ against the stream's schema."
+ default: false
+ type: "boolean"
+ required:
+ - "name"
+ - "format"
credentials:
title: "Authentication"
- description: "Choose how to authenticate to HubSpot."
+ description: "Credentials for connecting to the One Drive API"
type: "object"
+ order: 0
oneOf:
- - type: "object"
- title: "OAuth"
- required:
- - "client_id"
- - "client_secret"
- - "refresh_token"
- - "credentials_title"
+ - title: "Authenticate via Microsoft (OAuth)"
+ description: "OAuthCredentials class to hold authentication details for\
+ \ Microsoft OAuth authentication.\nThis class uses pydantic for data\
+ \ validation and settings management."
+ type: "object"
properties:
- credentials_title:
- type: "string"
+ auth_type:
title: "Auth Type"
- description: "Name of the credentials"
- const: "OAuth Credentials"
- order: 0
+ default: "Client"
+ const: "Client"
enum:
- - "OAuth Credentials"
+ - "Client"
+ type: "string"
+ tenant_id:
+ title: "Tenant ID"
+ description: "Tenant ID of the Microsoft OneDrive user"
+ airbyte_secret: true
+ type: "string"
client_id:
title: "Client ID"
- description: "The Client ID of your HubSpot developer application.\
- \ See the Hubspot docs if you need help finding this ID."
+ description: "Client ID of your Microsoft developer application"
+ airbyte_secret: true
type: "string"
- examples:
- - "123456789000"
client_secret:
title: "Client Secret"
- description: "The client secret for your HubSpot developer application.\
- \ See the Hubspot docs if you need help finding this secret."
- type: "string"
- examples:
- - "secret"
+ description: "Client Secret of your Microsoft developer application"
airbyte_secret: true
+ type: "string"
refresh_token:
title: "Refresh Token"
- description: "Refresh token to renew an expired access token. See\
- \ the Hubspot docs if you need help finding this token."
- type: "string"
- examples:
- - "refresh_token"
+ description: "Refresh Token of your Microsoft developer application"
airbyte_secret: true
- x-speakeasy-param-sensitive: true
- - type: "object"
- title: "Private App"
- required:
- - "access_token"
- - "credentials_title"
- properties:
- credentials_title:
- type: "string"
- title: "Auth Type"
- description: "Name of the credentials set"
- const: "Private App Credentials"
- order: 0
- enum:
- - "Private App Credentials"
- access_token:
- title: "Access token"
- description: "HubSpot Access token. See the Hubspot docs if you need help finding this token."
type: "string"
- airbyte_secret: true
x-speakeasy-param-sensitive: true
- enable_experimental_streams:
- title: "Enable experimental streams"
- description: "If enabled then experimental streams become available for\
- \ sync."
- type: "boolean"
- default: false
- sourceType:
- title: "hubspot"
- const: "hubspot"
- enum:
- - "hubspot"
- order: 0
- type: "string"
- source-hubspot-update:
- title: "HubSpot Source Spec"
- type: "object"
- required:
- - "credentials"
- properties:
- start_date:
- type: "string"
- title: "Start date"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
- \ data before this date will not be replicated. If not set, \"2006-06-01T00:00:00Z\"\
- \ (Hubspot creation date) will be used as start date. It's recommended\
- \ to provide relevant to your data start date value to optimize synchronization."
- examples:
- - "2017-01-25T00:00:00Z"
- format: "date-time"
- credentials:
- title: "Authentication"
- description: "Choose how to authenticate to HubSpot."
- type: "object"
- oneOf:
- - type: "object"
- title: "OAuth"
required:
+ - "tenant_id"
- "client_id"
- "client_secret"
- "refresh_token"
- - "credentials_title"
+ - title: "Service Key Authentication"
+ description: "ServiceCredentials class for service key authentication.\n\
+ This class is structured similarly to OAuthCredentials but for a different\
+ \ authentication method."
+ type: "object"
properties:
- credentials_title:
- type: "string"
+ auth_type:
title: "Auth Type"
- description: "Name of the credentials"
- const: "OAuth Credentials"
- order: 0
+ default: "Service"
+ const: "Service"
enum:
- - "OAuth Credentials"
+ - "Service"
+ type: "string"
+ tenant_id:
+ title: "Tenant ID"
+ description: "Tenant ID of the Microsoft OneDrive user"
+ airbyte_secret: true
+ type: "string"
+ user_principal_name:
+ title: "User Principal Name"
+ description: "Special characters such as a period, comma, space, and\
+ \ the at sign (@) are converted to underscores (_). More details:\
+ \ https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls"
+ airbyte_secret: true
+ type: "string"
client_id:
title: "Client ID"
- description: "The Client ID of your HubSpot developer application.\
- \ See the Hubspot docs if you need help finding this ID."
+ description: "Client ID of your Microsoft developer application"
+ airbyte_secret: true
type: "string"
- examples:
- - "123456789000"
client_secret:
title: "Client Secret"
- description: "The client secret for your HubSpot developer application.\
- \ See the Hubspot docs if you need help finding this secret."
- type: "string"
- examples:
- - "secret"
+ description: "Client Secret of your Microsoft developer application"
airbyte_secret: true
- refresh_token:
- title: "Refresh Token"
- description: "Refresh token to renew an expired access token. See\
- \ the Hubspot docs if you need help finding this token."
type: "string"
- examples:
- - "refresh_token"
- airbyte_secret: true
- - type: "object"
- title: "Private App"
required:
- - "access_token"
- - "credentials_title"
- properties:
- credentials_title:
- type: "string"
- title: "Auth Type"
- description: "Name of the credentials set"
- const: "Private App Credentials"
- order: 0
- enum:
- - "Private App Credentials"
- access_token:
- title: "Access token"
- description: "HubSpot Access token. See the Hubspot docs if you need help finding this token."
- type: "string"
- airbyte_secret: true
- enable_experimental_streams:
- title: "Enable experimental streams"
- description: "If enabled then experimental streams become available for\
- \ sync."
- type: "boolean"
- default: false
- source-rss:
- title: "RSS Spec"
- type: "object"
- required:
- - "url"
- - "sourceType"
- properties:
- url:
- type: "string"
- description: "RSS Feed URL"
- sourceType:
- title: "rss"
- const: "rss"
- enum:
- - "rss"
- order: 0
- type: "string"
- source-rss-update:
- title: "RSS Spec"
- type: "object"
- required:
- - "url"
- properties:
- url:
- type: "string"
- description: "RSS Feed URL"
- source-sap-fieldglass:
- title: "Sap Fieldglass Spec"
- type: "object"
- required:
- - "api_key"
- - "sourceType"
- properties:
- api_key:
- type: "string"
- description: "API Key"
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- sourceType:
- title: "sap-fieldglass"
- const: "sap-fieldglass"
- enum:
- - "sap-fieldglass"
- order: 0
- type: "string"
- source-sap-fieldglass-update:
- title: "Sap Fieldglass Spec"
- type: "object"
- required:
- - "api_key"
- properties:
- api_key:
- type: "string"
- description: "API Key"
- airbyte_secret: true
- source-twilio-taskrouter:
- title: "Twilio Taskrouter Spec"
- type: "object"
- required:
- - "account_sid"
- - "auth_token"
- - "sourceType"
- properties:
- account_sid:
- title: "Account SID"
- type: "string"
- description: "Twilio Account ID"
- airbyte_secret: true
- auth_token:
- type: "string"
- description: "Twilio Auth Token"
- airbyte_secret: true
- title: "Auth Token"
- x-speakeasy-param-sensitive: true
- sourceType:
- title: "twilio-taskrouter"
- const: "twilio-taskrouter"
- enum:
- - "twilio-taskrouter"
- order: 0
- type: "string"
- source-twilio-taskrouter-update:
- title: "Twilio Taskrouter Spec"
- type: "object"
- required:
- - "account_sid"
- - "auth_token"
- properties:
- account_sid:
- title: "Account SID"
- type: "string"
- description: "Twilio Account ID"
- airbyte_secret: true
- auth_token:
+ - "tenant_id"
+ - "user_principal_name"
+ - "client_id"
+ - "client_secret"
+ drive_name:
+ title: "Drive Name"
+ description: "Name of the Microsoft OneDrive drive where the file(s) exist."
+ default: "OneDrive"
+ order: 2
type: "string"
- description: "Twilio Auth Token"
- airbyte_secret: true
- title: "Auth Token"
- source-xkcd:
- title: "Xkcd Spec"
- type: "object"
- properties:
- sourceType:
- title: "xkcd"
- const: "xkcd"
+ search_scope:
+ title: "Search Scope"
+ description: "Specifies the location(s) to search for files. Valid options\
+ \ are 'ACCESSIBLE_DRIVES' to search in the selected OneDrive drive, 'SHARED_ITEMS'\
+ \ for shared items the user has access to, and 'ALL' to search both."
+ default: "ALL"
enum:
- - "xkcd"
- order: 0
- type: "string"
- source-xkcd-update:
- title: "Xkcd Spec"
- type: "object"
- properties: {}
- source-zenloop:
- title: "Zenloop Spec"
- type: "object"
- required:
- - "api_token"
- - "sourceType"
- properties:
- api_token:
- type: "string"
- description: "Zenloop API Token. You can get the API token in settings page\
- \ here "
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- date_from:
- type: "string"
- description: "Zenloop date_from. Format: 2021-10-24T03:30:30Z or 2021-10-24.\
- \ Leave empty if only data from current data should be synced"
- examples:
- - "2021-10-24T03:30:30Z"
- survey_id:
+ - "ACCESSIBLE_DRIVES"
+ - "SHARED_ITEMS"
+ - "ALL"
+ order: 3
type: "string"
- description: "Zenloop Survey ID. Can be found here. Leave empty to pull answers from all surveys"
- airbyte_secret: true
- survey_group_id:
+ folder_path:
+ title: "Folder Path"
+ description: "Path to a specific folder within the drives to search for\
+ \ files. Leave empty to search all folders of the drives. This does not\
+ \ apply to shared items."
+ default: "."
+ order: 4
type: "string"
- description: "Zenloop Survey Group ID. Can be found by pulling All Survey\
- \ Groups via SurveyGroups stream. Leave empty to pull answers from all\
- \ survey groups"
- airbyte_secret: true
sourceType:
- title: "zenloop"
- const: "zenloop"
+ title: "microsoft-onedrive"
+ const: "microsoft-onedrive"
enum:
- - "zenloop"
+ - "microsoft-onedrive"
order: 0
type: "string"
- source-zenloop-update:
- title: "Zenloop Spec"
- type: "object"
- required:
- - "api_token"
- properties:
- api_token:
- type: "string"
- description: "Zenloop API Token. You can get the API token in settings page\
- \ here "
- airbyte_secret: true
- date_from:
- type: "string"
- description: "Zenloop date_from. Format: 2021-10-24T03:30:30Z or 2021-10-24.\
- \ Leave empty if only data from current data should be synced"
- examples:
- - "2021-10-24T03:30:30Z"
- survey_id:
- type: "string"
- description: "Zenloop Survey ID. Can be found here. Leave empty to pull answers from all surveys"
- airbyte_secret: true
- survey_group_id:
- type: "string"
- description: "Zenloop Survey Group ID. Can be found by pulling All Survey\
- \ Groups via SurveyGroups stream. Leave empty to pull answers from all\
- \ survey groups"
- airbyte_secret: true
- source-tempo:
- title: "Tempo Spec"
- type: "object"
required:
- - "api_token"
+ - "streams"
+ - "credentials"
- "sourceType"
- properties:
- api_token:
- type: "string"
- title: "API token"
- description: "Tempo API Token. Go to Tempo>Settings, scroll down to Data\
- \ Access and select API integration."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- sourceType:
- title: "tempo"
- const: "tempo"
- enum:
- - "tempo"
- order: 0
- type: "string"
- source-tempo-update:
- title: "Tempo Spec"
- type: "object"
- required:
- - "api_token"
- properties:
- api_token:
- type: "string"
- title: "API token"
- description: "Tempo API Token. Go to Tempo>Settings, scroll down to Data\
- \ Access and select API integration."
- airbyte_secret: true
- source-chargebee:
- title: "Chargebee Spec"
+ source-microsoft-onedrive-update:
+ title: "Microsoft OneDrive Source Spec"
+ description: "SourceMicrosoftOneDriveSpec class for Microsoft OneDrive Source\
+ \ Specification.\nThis class combines the authentication details with additional\
+ \ configuration for the OneDrive API."
type: "object"
- required:
- - "site"
- - "site_api_key"
- - "start_date"
- - "sourceType"
properties:
- site_api_key:
- type: "string"
- title: "API Key"
- description: "Chargebee API Key. See the docs for more information on how to obtain this key."
- airbyte_secret: true
- order: 0
- x-speakeasy-param-sensitive: true
- site:
- type: "string"
- title: "Site"
- description: "The site prefix for your Chargebee instance."
- examples:
- - "airbyte-test"
- order: 1
start_date:
- type: "string"
- format: "date-time"
title: "Start Date"
- description: "UTC date and time in the format 2017-01-25T00:00:00.000Z.\
- \ Any data before this date will not be replicated."
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- examples:
- - "2021-01-25T00:00:00Z"
- order: 2
- product_catalog:
- type: "string"
- title: "Product Catalog"
- description: "Product Catalog version of your Chargebee site. Instructions\
- \ on how to find your version you may find here under `API Version` section. If left blank, the product catalog\
- \ version will be set to 2.0."
- enum:
- - "1.0"
- - "2.0"
- default: "2.0"
- order: 3
- sourceType:
- title: "chargebee"
- const: "chargebee"
- enum:
- - "chargebee"
- order: 0
- type: "string"
- source-chargebee-update:
- title: "Chargebee Spec"
- type: "object"
- required:
- - "site"
- - "site_api_key"
- - "start_date"
- properties:
- site_api_key:
- type: "string"
- title: "API Key"
- description: "Chargebee API Key. See the docs for more information on how to obtain this key."
- airbyte_secret: true
- order: 0
- site:
- type: "string"
- title: "Site"
- description: "The site prefix for your Chargebee instance."
+ description: "UTC date and time in the format 2017-01-25T00:00:00.000000Z.\
+ \ Any file modified before this date will not be replicated."
examples:
- - "airbyte-test"
- order: 1
- start_date:
- type: "string"
+ - "2021-01-01T00:00:00.000000Z"
format: "date-time"
- title: "Start Date"
- description: "UTC date and time in the format 2017-01-25T00:00:00.000Z.\
- \ Any data before this date will not be replicated."
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- examples:
- - "2021-01-25T00:00:00Z"
- order: 2
- product_catalog:
- type: "string"
- title: "Product Catalog"
- description: "Product Catalog version of your Chargebee site. Instructions\
- \ on how to find your version you may find here under `API Version` section. If left blank, the product catalog\
- \ version will be set to 2.0."
- enum:
- - "1.0"
- - "2.0"
- default: "2.0"
- order: 3
- source-onesignal:
- title: "OneSignal Source Spec"
- type: "object"
- required:
- - "user_auth_key"
- - "start_date"
- - "outcome_names"
- - "applications"
- - "sourceType"
- properties:
- user_auth_key:
- type: "string"
- title: "User Auth Key"
- description: "OneSignal User Auth Key, see the docs for more information on how to obtain this key."
- airbyte_secret: true
- order: 0
- x-speakeasy-param-sensitive: true
- applications:
- type: "array"
- title: "Applications"
- description: "Applications keys, see the docs for more information on how to obtain this data"
- items:
- type: "object"
- properties:
- app_name:
- type: "string"
- title: "OneSignal App Name"
- order: 0
- app_id:
- type: "string"
- title: "OneSignal App ID"
- order: 1
- airbyte_secret: true
- app_api_key:
- type: "string"
- title: "REST API Key"
- order: 2
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- required:
- - "app_id"
- - "app_api_key"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$"
+ pattern_descriptor: "YYYY-MM-DDTHH:mm:ss.SSSSSSZ"
order: 1
- start_date:
- type: "string"
- title: "Start Date"
- description: "The date from which you'd like to replicate data for OneSignal\
- \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\
- \ date will be replicated."
- examples:
- - "2020-11-16T00:00:00Z"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- format: "date-time"
- order: 2
- outcome_names:
- type: "string"
- title: "Outcome Names"
- description: "Comma-separated list of names and the value (sum/count) for\
- \ the returned outcome data. See the docs for more details"
- examples:
- - "os__session_duration.count,os__click.count,CustomOutcomeName.sum"
- order: 3
- sourceType:
- title: "onesignal"
- const: "onesignal"
- enum:
- - "onesignal"
- order: 0
- type: "string"
- source-onesignal-update:
- title: "OneSignal Source Spec"
- type: "object"
- required:
- - "user_auth_key"
- - "start_date"
- - "outcome_names"
- - "applications"
- properties:
- user_auth_key:
type: "string"
- title: "User Auth Key"
- description: "OneSignal User Auth Key, see the docs for more information on how to obtain this key."
- airbyte_secret: true
- order: 0
- applications:
+ streams:
+ title: "The list of streams to sync"
+ description: "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their\
+ \ format, and how they should be parsed and validated. When sending data\
+ \ to warehouse destination such as Snowflake or BigQuery, each stream\
+ \ is a separate table."
+ order: 10
type: "array"
- title: "Applications"
- description: "Applications keys, see the docs for more information on how to obtain this data"
items:
+ title: "FileBasedStreamConfig"
type: "object"
properties:
- app_name:
- type: "string"
- title: "OneSignal App Name"
- order: 0
- app_id:
+ name:
+ title: "Name"
+ description: "The name of the stream."
type: "string"
- title: "OneSignal App ID"
+ globs:
+ title: "Globs"
+ description: "The pattern used to specify which files should be selected\
+ \ from the file system. For more information on glob pattern matching\
+ \ look here."
+ default:
+ - "**"
order: 1
- airbyte_secret: true
- app_api_key:
+ type: "array"
+ items:
+ type: "string"
+ validation_policy:
+ title: "Validation Policy"
+ description: "The name of the validation policy that dictates sync\
+ \ behavior when a record does not adhere to the stream schema."
+ default: "Emit Record"
+ enum:
+ - "Emit Record"
+ - "Skip Record"
+ - "Wait for Discover"
+ input_schema:
+ title: "Input Schema"
+ description: "The schema that will be used to validate records extracted\
+ \ from the file. This will override the stream schema that is auto-detected\
+ \ from incoming files."
type: "string"
- title: "REST API Key"
- order: 2
- airbyte_secret: true
+ primary_key:
+ title: "Primary Key"
+ description: "The column or columns (for a composite key) that serves\
+ \ as the unique identifier of a record. If empty, the primary key\
+ \ will default to the parser's default primary key."
+ airbyte_hidden: true
+ type: "string"
+ days_to_sync_if_history_is_full:
+ title: "Days To Sync If History Is Full"
+ description: "When the state history of the file store is full, syncs\
+ \ will only read files that were last modified in the provided day\
+ \ range."
+ default: 3
+ type: "integer"
+ format:
+ title: "Format"
+ description: "The configuration options that are used to alter how\
+ \ to read incoming files that deviate from the standard formatting."
+ type: "object"
+ oneOf:
+ - title: "Avro Format"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "avro"
+ const: "avro"
+ type: "string"
+ enum:
+ - "avro"
+ double_as_string:
+ title: "Convert Double Fields to Strings"
+ description: "Whether to convert double fields to strings. This\
+ \ is recommended if you have decimal numbers with a high degree\
+ \ of precision because there can be a loss precision when\
+ \ handling floating point numbers."
+ default: false
+ type: "boolean"
+ required:
+ - "filetype"
+ - title: "CSV Format"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "csv"
+ const: "csv"
+ type: "string"
+ enum:
+ - "csv"
+ delimiter:
+ title: "Delimiter"
+ description: "The character delimiting individual cells in the\
+ \ CSV data. This may only be a 1-character string. For tab-delimited\
+ \ data enter '\\t'."
+ default: ","
+ type: "string"
+ quote_char:
+ title: "Quote Character"
+ description: "The character used for quoting CSV values. To\
+ \ disallow quoting, make this field blank."
+ default: "\""
+ type: "string"
+ escape_char:
+ title: "Escape Character"
+ description: "The character used for escaping special characters.\
+ \ To disallow escaping, leave this field blank."
+ type: "string"
+ encoding:
+ title: "Encoding"
+ description: "The character encoding of the CSV data. Leave\
+ \ blank to default to UTF8. See list of python encodings for allowable\
+ \ options."
+ default: "utf8"
+ type: "string"
+ double_quote:
+ title: "Double Quote"
+ description: "Whether two quotes in a quoted CSV value denote\
+ \ a single quote in the data."
+ default: true
+ type: "boolean"
+ null_values:
+ title: "Null Values"
+ description: "A set of case-sensitive strings that should be\
+ \ interpreted as null values. For example, if the value 'NA'\
+ \ should be interpreted as null, enter 'NA' in this field."
+ default: []
+ type: "array"
+ items:
+ type: "string"
+ uniqueItems: true
+ strings_can_be_null:
+ title: "Strings Can Be Null"
+ description: "Whether strings can be interpreted as null values.\
+ \ If true, strings that match the null_values set will be\
+ \ interpreted as null. If false, strings that match the null_values\
+ \ set will be interpreted as the string itself."
+ default: true
+ type: "boolean"
+ skip_rows_before_header:
+ title: "Skip Rows Before Header"
+ description: "The number of rows to skip before the header row.\
+ \ For example, if the header row is on the 3rd row, enter\
+ \ 2 in this field."
+ default: 0
+ type: "integer"
+ skip_rows_after_header:
+ title: "Skip Rows After Header"
+ description: "The number of rows to skip after the header row."
+ default: 0
+ type: "integer"
+ header_definition:
+ title: "CSV Header Definition"
+ description: "How headers will be defined. `User Provided` assumes\
+ \ the CSV does not have a header row and uses the headers\
+ \ provided and `Autogenerated` assumes the CSV does not have\
+ \ a header row and the CDK will generate headers using for\
+ \ `f{i}` where `i` is the index starting from 0. Else, the\
+ \ default behavior is to use the header from the CSV file.\
+ \ If a user wants to autogenerate or provide column names\
+ \ for a CSV having headers, they can skip rows."
+ default:
+ header_definition_type: "From CSV"
+ oneOf:
+ - title: "From CSV"
+ type: "object"
+ properties:
+ header_definition_type:
+ title: "Header Definition Type"
+ default: "From CSV"
+ const: "From CSV"
+ type: "string"
+ enum:
+ - "From CSV"
+ required:
+ - "header_definition_type"
+ - title: "Autogenerated"
+ type: "object"
+ properties:
+ header_definition_type:
+ title: "Header Definition Type"
+ default: "Autogenerated"
+ const: "Autogenerated"
+ type: "string"
+ enum:
+ - "Autogenerated"
+ required:
+ - "header_definition_type"
+ - title: "User Provided"
+ type: "object"
+ properties:
+ header_definition_type:
+ title: "Header Definition Type"
+ default: "User Provided"
+ const: "User Provided"
+ type: "string"
+ enum:
+ - "User Provided"
+ column_names:
+ title: "Column Names"
+ description: "The column names that will be used while\
+ \ emitting the CSV records"
+ type: "array"
+ items:
+ type: "string"
+ required:
+ - "column_names"
+ - "header_definition_type"
+ type: "object"
+ true_values:
+ title: "True Values"
+ description: "A set of case-sensitive strings that should be\
+ \ interpreted as true values."
+ default:
+ - "y"
+ - "yes"
+ - "t"
+ - "true"
+ - "on"
+ - "1"
+ type: "array"
+ items:
+ type: "string"
+ uniqueItems: true
+ false_values:
+ title: "False Values"
+ description: "A set of case-sensitive strings that should be\
+ \ interpreted as false values."
+ default:
+ - "n"
+ - "no"
+ - "f"
+ - "false"
+ - "off"
+ - "0"
+ type: "array"
+ items:
+ type: "string"
+ uniqueItems: true
+ required:
+ - "filetype"
+ - title: "Jsonl Format"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "jsonl"
+ const: "jsonl"
+ type: "string"
+ enum:
+ - "jsonl"
+ required:
+ - "filetype"
+ - title: "Parquet Format"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "parquet"
+ const: "parquet"
+ type: "string"
+ enum:
+ - "parquet"
+ decimal_as_float:
+ title: "Convert Decimal Fields to Floats"
+ description: "Whether to convert decimal fields to floats. There\
+ \ is a loss of precision when converting decimals to floats,\
+ \ so this is not recommended."
+ default: false
+ type: "boolean"
+ required:
+ - "filetype"
+ - title: "Document File Type Format (Experimental)"
+ type: "object"
+ properties:
+ filetype:
+ title: "Filetype"
+ default: "unstructured"
+ const: "unstructured"
+ type: "string"
+ enum:
+ - "unstructured"
+ skip_unprocessable_files:
+ title: "Skip Unprocessable Files"
+ description: "If true, skip files that cannot be parsed and\
+ \ pass the error message along as the _ab_source_file_parse_error\
+ \ field. If false, fail the sync."
+ default: true
+ always_show: true
+ type: "boolean"
+ strategy:
+ title: "Parsing Strategy"
+ description: "The strategy used to parse documents. `fast` extracts\
+ \ text directly from the document which doesn't work for all\
+ \ files. `ocr_only` is more reliable, but slower. `hi_res`\
+ \ is the most reliable, but requires an API key and a hosted\
+ \ instance of unstructured and can't be used with local mode.\
+ \ See the unstructured.io documentation for more details:\
+ \ https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf"
+ default: "auto"
+ always_show: true
+ order: 0
+ enum:
+ - "auto"
+ - "fast"
+ - "ocr_only"
+ - "hi_res"
+ type: "string"
+ processing:
+ title: "Processing"
+ description: "Processing configuration"
+ default:
+ mode: "local"
+ type: "object"
+ oneOf:
+ - title: "Local"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "local"
+ const: "local"
+ enum:
+ - "local"
+ type: "string"
+ description: "Process files locally, supporting `fast` and\
+ \ `ocr` modes. This is the default option."
+ required:
+ - "mode"
+ description: "Extract text from document formats (.pdf, .docx, .md,\
+ \ .pptx) and emit as one record per file."
+ required:
+ - "filetype"
+ schemaless:
+ title: "Schemaless"
+ description: "When enabled, syncs will not validate or structure records\
+ \ against the stream's schema."
+ default: false
+ type: "boolean"
required:
- - "app_id"
- - "app_api_key"
- order: 1
- start_date:
- type: "string"
- title: "Start Date"
- description: "The date from which you'd like to replicate data for OneSignal\
- \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\
- \ date will be replicated."
- examples:
- - "2020-11-16T00:00:00Z"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
- format: "date-time"
- order: 2
- outcome_names:
- type: "string"
- title: "Outcome Names"
- description: "Comma-separated list of names and the value (sum/count) for\
- \ the returned outcome data. See the docs for more details"
- examples:
- - "os__session_duration.count,os__click.count,CustomOutcomeName.sum"
- order: 3
- source-google-analytics-data-api:
- title: "Google Analytics (Data API) Spec"
- type: "object"
- required:
- - "property_ids"
- - "sourceType"
- properties:
+ - "name"
+ - "format"
credentials:
- order: 0
+ title: "Authentication"
+ description: "Credentials for connecting to the One Drive API"
type: "object"
- title: "Credentials"
- description: "Credentials for the service"
+ order: 0
oneOf:
- - title: "Authenticate via Google (Oauth)"
+ - title: "Authenticate via Microsoft (OAuth)"
+ description: "OAuthCredentials class to hold authentication details for\
+ \ Microsoft OAuth authentication.\nThis class uses pydantic for data\
+ \ validation and settings management."
type: "object"
- required:
- - "client_id"
- - "client_secret"
- - "refresh_token"
properties:
auth_type:
- type: "string"
+ title: "Auth Type"
+ default: "Client"
const: "Client"
- order: 0
enum:
- "Client"
+ type: "string"
+ tenant_id:
+ title: "Tenant ID"
+ description: "Tenant ID of the Microsoft OneDrive user"
+ airbyte_secret: true
+ type: "string"
client_id:
title: "Client ID"
+ description: "Client ID of your Microsoft developer application"
+ airbyte_secret: true
type: "string"
- description: "The Client ID of your Google Analytics developer application."
- order: 1
client_secret:
title: "Client Secret"
- type: "string"
- description: "The Client Secret of your Google Analytics developer\
- \ application."
+ description: "Client Secret of your Microsoft developer application"
airbyte_secret: true
- order: 2
+ type: "string"
refresh_token:
title: "Refresh Token"
- type: "string"
- description: "The token for obtaining a new access token."
+ description: "Refresh Token of your Microsoft developer application"
airbyte_secret: true
- order: 3
- x-speakeasy-param-sensitive: true
- access_token:
- title: "Access Token"
type: "string"
- description: "Access Token for making authenticated requests."
- airbyte_secret: true
- order: 4
- x-speakeasy-param-sensitive: true
- - type: "object"
- title: "Service Account Key Authentication"
required:
- - "credentials_json"
+ - "tenant_id"
+ - "client_id"
+ - "client_secret"
+ - "refresh_token"
+ - title: "Service Key Authentication"
+ description: "ServiceCredentials class for service key authentication.\n\
+ This class is structured similarly to OAuthCredentials but for a different\
+ \ authentication method."
+ type: "object"
properties:
auth_type:
- type: "string"
+ title: "Auth Type"
+ default: "Service"
const: "Service"
- order: 0
enum:
- "Service"
- credentials_json:
- title: "Service Account JSON Key"
type: "string"
- description: "The JSON key linked to the service account used for\
- \ authorization. For steps on obtaining this key, refer to the setup guide."
- examples:
- - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID,\
- \ \"private_key_id\": YOUR_PRIVATE_KEY, ... }"
+ tenant_id:
+ title: "Tenant ID"
+ description: "Tenant ID of the Microsoft OneDrive user"
airbyte_secret: true
- order: 1
- property_ids:
- title: "Property IDs"
- description: "A list of your Property IDs. The Property ID is a unique number\
- \ assigned to each property in Google Analytics, found in your GA4 property\
- \ URL. This ID allows the connector to track the specific events associated\
- \ with your property. Refer to the Google\
- \ Analytics documentation to locate your property ID."
- order: 1
- type: "array"
- items:
- type: "string"
- pattern: "^[0-9]*$"
+ type: "string"
+ user_principal_name:
+ title: "User Principal Name"
+ description: "Special characters such as a period, comma, space, and\
+ \ the at sign (@) are converted to underscores (_). More details:\
+ \ https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls"
+ airbyte_secret: true
+ type: "string"
+ client_id:
+ title: "Client ID"
+ description: "Client ID of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ client_secret:
+ title: "Client Secret"
+ description: "Client Secret of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ required:
+ - "tenant_id"
+ - "user_principal_name"
+ - "client_id"
+ - "client_secret"
+ drive_name:
+ title: "Drive Name"
+ description: "Name of the Microsoft OneDrive drive where the file(s) exist."
+ default: "OneDrive"
+ order: 2
+ type: "string"
+ search_scope:
+ title: "Search Scope"
+ description: "Specifies the location(s) to search for files. Valid options\
+ \ are 'ACCESSIBLE_DRIVES' to search in the selected OneDrive drive, 'SHARED_ITEMS'\
+ \ for shared items the user has access to, and 'ALL' to search both."
+ default: "ALL"
+ enum:
+ - "ACCESSIBLE_DRIVES"
+ - "SHARED_ITEMS"
+ - "ALL"
+ order: 3
+ type: "string"
+ folder_path:
+ title: "Folder Path"
+ description: "Path to a specific folder within the drives to search for\
+ \ files. Leave empty to search all folders of the drives. This does not\
+ \ apply to shared items."
+ default: "."
+ order: 4
+ type: "string"
+ required:
+ - "streams"
+ - "credentials"
+ source-punk-api:
+ title: "Punk Api Spec"
+ type: "object"
+ required:
+ - "brewed_before"
+ - "brewed_after"
+ - "sourceType"
+ properties:
+ id:
+ title: "Beers with specific ID"
+ type: "string"
+ description: "To extract specific data with Unique ID"
examples:
- - - "1738294"
- - "5729978930"
- date_ranges_start_date:
+ - 1
+ - 22
+ brewed_before:
+ title: "Brewed before data to get incremental reads"
type: "string"
- title: "Start Date"
- description: "The start date from which to replicate report data in the\
- \ format YYYY-MM-DD. Data generated before this date will not be included\
- \ in the report. Not applied to custom Cohort reports."
- format: "date"
- pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
- pattern_descriptor: "YYYY-MM-DD"
+ description: "To extract specific data with Unique ID"
+ pattern: "^[0-9]{2}-[0-9]{4}$"
examples:
- - "2021-01-01"
- order: 2
- custom_reports_array:
- title: "Custom Reports"
- description: "You can add your Custom Analytics report by creating one."
- order: 4
- type: "array"
- items:
- title: "Custom Report Config"
- type: "object"
- properties:
+ - "MM-YYYY"
+ brewed_after:
+ title: "Brewed after data to get incremental reads"
+ type: "string"
+ description: "To extract specific data with Unique ID"
+ pattern: "^[0-9]{2}-[0-9]{4}$"
+ examples:
+ - "MM-YYYY"
+ sourceType:
+ title: "punk-api"
+ const: "punk-api"
+ enum:
+ - "punk-api"
+ order: 0
+ type: "string"
+ source-punk-api-update:
+ title: "Punk Api Spec"
+ type: "object"
+ required:
+ - "brewed_before"
+ - "brewed_after"
+ properties:
+ id:
+ title: "Beers with specific ID"
+ type: "string"
+ description: "To extract specific data with Unique ID"
+ examples:
+ - 1
+ - 22
+ brewed_before:
+ title: "Brewed before data to get incremental reads"
+ type: "string"
+ description: "To extract specific data with Unique ID"
+ pattern: "^[0-9]{2}-[0-9]{4}$"
+ examples:
+ - "MM-YYYY"
+ brewed_after:
+ title: "Brewed after data to get incremental reads"
+ type: "string"
+ description: "To extract specific data with Unique ID"
+ pattern: "^[0-9]{2}-[0-9]{4}$"
+ examples:
+ - "MM-YYYY"
+ source-tiktok-marketing:
+ title: "TikTok Marketing Source Spec"
+ type: "object"
+ properties:
+ credentials:
+ title: "Authentication Method"
+ description: "Authentication method"
+ default: {}
+ order: 0
+ type: "object"
+ oneOf:
+ - title: "OAuth2.0"
+ type: "object"
+ properties:
+ auth_type:
+ title: "Auth Type"
+ const: "oauth2.0"
+ order: 0
+ type: "string"
+ enum:
+ - "oauth2.0"
+ app_id:
+ title: "App ID"
+ description: "The Developer Application App ID."
+ airbyte_secret: true
+ type: "string"
+ secret:
+ title: "Secret"
+ description: "The Developer Application Secret."
+ airbyte_secret: true
+ type: "string"
+ access_token:
+ title: "Access Token"
+ description: "Long-term Authorized Access Token."
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ advertiser_id:
+ title: "Advertiser ID"
+ description: "The Advertiser ID to filter reports and streams. Let\
+ \ this empty to retrieve all."
+ type: "string"
+ required:
+ - "app_id"
+ - "secret"
+ - "access_token"
+ - title: "Sandbox Access Token"
+ type: "object"
+ properties:
+ auth_type:
+ title: "Auth Type"
+ const: "sandbox_access_token"
+ order: 0
+ type: "string"
+ enum:
+ - "sandbox_access_token"
+ advertiser_id:
+ title: "Advertiser ID"
+ description: "The Advertiser ID which generated for the developer's\
+ \ Sandbox application."
+ type: "string"
+ access_token:
+ title: "Access Token"
+ description: "The long-term authorized access token."
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ required:
+ - "advertiser_id"
+ - "access_token"
+ start_date:
+ title: "Replication Start Date"
+ description: "The Start Date in format: YYYY-MM-DD. Any data before this\
+ \ date will not be replicated. If this parameter is not set, all data\
+ \ will be replicated."
+ default: "2016-09-01"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+ order: 1
+ type: "string"
+ format: "date"
+ end_date:
+ title: "End Date"
+ description: "The date until which you'd like to replicate data for all\
+ \ incremental streams, in the format YYYY-MM-DD. All data generated between\
+ \ start_date and this date will be replicated. Not setting this option\
+ \ will result in always syncing the data till the current date."
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+ order: 2
+ type: "string"
+ format: "date"
+ attribution_window:
+ title: "Attribution Window"
+ description: "The attribution window in days."
+ minimum: 0
+ maximum: 364
+ default: 3
+ order: 3
+ type: "integer"
+ include_deleted:
+ title: "Include Deleted Data in Reports"
+ description: "Set to active if you want to include deleted data in reports."
+ default: false
+ order: 4
+ type: "boolean"
+ sourceType:
+ title: "tiktok-marketing"
+ const: "tiktok-marketing"
+ enum:
+ - "tiktok-marketing"
+ order: 0
+ type: "string"
+ source-tiktok-marketing-update:
+ title: "TikTok Marketing Source Spec"
+ type: "object"
+ properties:
+ credentials:
+ title: "Authentication Method"
+ description: "Authentication method"
+ default: {}
+ order: 0
+ type: "object"
+ oneOf:
+ - title: "OAuth2.0"
+ type: "object"
+ properties:
+ auth_type:
+ title: "Auth Type"
+ const: "oauth2.0"
+ order: 0
+ type: "string"
+ enum:
+ - "oauth2.0"
+ app_id:
+ title: "App ID"
+ description: "The Developer Application App ID."
+ airbyte_secret: true
+ type: "string"
+ secret:
+ title: "Secret"
+ description: "The Developer Application Secret."
+ airbyte_secret: true
+ type: "string"
+ access_token:
+ title: "Access Token"
+ description: "Long-term Authorized Access Token."
+ airbyte_secret: true
+ type: "string"
+ advertiser_id:
+ title: "Advertiser ID"
+ description: "The Advertiser ID to filter reports and streams. Let\
+ \ this empty to retrieve all."
+ type: "string"
+ required:
+ - "app_id"
+ - "secret"
+ - "access_token"
+ - title: "Sandbox Access Token"
+ type: "object"
+ properties:
+ auth_type:
+ title: "Auth Type"
+ const: "sandbox_access_token"
+ order: 0
+ type: "string"
+ enum:
+ - "sandbox_access_token"
+ advertiser_id:
+ title: "Advertiser ID"
+ description: "The Advertiser ID which generated for the developer's\
+ \ Sandbox application."
+ type: "string"
+ access_token:
+ title: "Access Token"
+ description: "The long-term authorized access token."
+ airbyte_secret: true
+ type: "string"
+ required:
+ - "advertiser_id"
+ - "access_token"
+ start_date:
+ title: "Replication Start Date"
+ description: "The Start Date in format: YYYY-MM-DD. Any data before this\
+ \ date will not be replicated. If this parameter is not set, all data\
+ \ will be replicated."
+ default: "2016-09-01"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+ order: 1
+ type: "string"
+ format: "date"
+ end_date:
+ title: "End Date"
+ description: "The date until which you'd like to replicate data for all\
+ \ incremental streams, in the format YYYY-MM-DD. All data generated between\
+ \ start_date and this date will be replicated. Not setting this option\
+ \ will result in always syncing the data till the current date."
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+ order: 2
+ type: "string"
+ format: "date"
+ attribution_window:
+ title: "Attribution Window"
+ description: "The attribution window in days."
+ minimum: 0
+ maximum: 364
+ default: 3
+ order: 3
+ type: "integer"
+ include_deleted:
+ title: "Include Deleted Data in Reports"
+ description: "Set to active if you want to include deleted data in reports."
+ default: false
+ order: 4
+ type: "boolean"
+ source-aws-cloudtrail:
+ title: "Aws CloudTrail Spec"
+ type: "object"
+ required:
+ - "aws_key_id"
+ - "aws_secret_key"
+ - "aws_region_name"
+ - "start_date"
+ - "sourceType"
+ properties:
+ aws_key_id:
+ type: "string"
+ title: "Key ID"
+ description: "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key."
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ aws_secret_key:
+ type: "string"
+ title: "Secret Key"
+ description: "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key."
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ aws_region_name:
+ type: "string"
+ title: "Region Name"
+ description: "The default AWS Region to use, for example, us-west-1 or us-west-2.\
+ \ When specifying a Region inline during client initialization, this property\
+ \ is named region_name."
+ start_date:
+ type: "string"
+ title: "Start Date"
+ description: "The date you would like to replicate data. Data in AWS CloudTrail\
+ \ is available for last 90 days only. Format: YYYY-MM-DD."
+ examples:
+ - "2021-01-01"
+ default: "1970-01-01"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+ format: "date"
+ sourceType:
+ title: "aws-cloudtrail"
+ const: "aws-cloudtrail"
+ enum:
+ - "aws-cloudtrail"
+ order: 0
+ type: "string"
+ source-aws-cloudtrail-update:
+ title: "Aws CloudTrail Spec"
+ type: "object"
+ required:
+ - "aws_key_id"
+ - "aws_secret_key"
+ - "aws_region_name"
+ - "start_date"
+ properties:
+ aws_key_id:
+ type: "string"
+ title: "Key ID"
+ description: "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key."
+ airbyte_secret: true
+ aws_secret_key:
+ type: "string"
+ title: "Secret Key"
+ description: "AWS CloudTrail Access Key ID. See the docs for more information on how to obtain this key."
+ airbyte_secret: true
+ aws_region_name:
+ type: "string"
+ title: "Region Name"
+ description: "The default AWS Region to use, for example, us-west-1 or us-west-2.\
+ \ When specifying a Region inline during client initialization, this property\
+ \ is named region_name."
+ start_date:
+ type: "string"
+ title: "Start Date"
+ description: "The date you would like to replicate data. Data in AWS CloudTrail\
+ \ is available for last 90 days only. Format: YYYY-MM-DD."
+ examples:
+ - "2021-01-01"
+ default: "1970-01-01"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+ format: "date"
+ source-jira:
+ title: "Jira Spec"
+ type: "object"
+ required:
+ - "api_token"
+ - "domain"
+ - "email"
+ - "sourceType"
+ properties:
+ api_token:
+ type: "string"
+ title: "API Token"
+ description: "Jira API Token. See the docs for more information on how to generate this key. API Token\
+ \ is used for Authorization to your account by BasicAuth."
+ airbyte_secret: true
+ order: 0
+ x-speakeasy-param-sensitive: true
+ domain:
+ type: "string"
+ title: "Domain"
+ examples:
+ - ".atlassian.net"
+ - ".jira.com"
+ - "jira..com"
+ description: "The Domain for your Jira account, e.g. airbyteio.atlassian.net,\
+ \ airbyteio.jira.com, jira.your-domain.com"
+ order: 1
+ email:
+ type: "string"
+ title: "Email"
+ description: "The user email for your Jira account which you used to generate\
+ \ the API token. This field is used for Authorization to your account\
+ \ by BasicAuth."
+ order: 2
+ projects:
+ type: "array"
+ title: "Projects"
+ items:
+ type: "string"
+ examples:
+ - "PROJ1"
+ - "PROJ2"
+ description: "List of Jira project keys to replicate data for, or leave\
+ \ it empty if you want to replicate data for all projects."
+ order: 3
+ start_date:
+ type: "string"
+ title: "Start Date"
+ description: "The date from which you want to replicate data from Jira,\
+ \ use the format YYYY-MM-DDT00:00:00Z. Note that this field only applies\
+ \ to certain streams, and only data generated on or after the start date\
+ \ will be replicated. Or leave it empty if you want to replicate all data.\
+ \ For more information, refer to the documentation."
+ examples:
+ - "2021-03-01T00:00:00Z"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ format: "date-time"
+ order: 4
+ expand_issue_changelog:
+ type: "boolean"
+ title: "Expand Issue Changelog"
+ airbyte_hidden: true
+ description: "(DEPRECATED) Expand the changelog when replicating issues."
+ default: false
+ render_fields:
+ type: "boolean"
+ title: "Render Issue Fields"
+ airbyte_hidden: true
+ description: "(DEPRECATED) Render issue fields in HTML format in addition\
+ \ to Jira JSON-like format."
+ default: false
+ expand_issue_transition:
+ type: "boolean"
+ title: "Expand Issue Transitions"
+ airbyte_hidden: true
+ description: "(DEPRECATED) Expand the transitions when replicating issues."
+ default: false
+ issues_stream_expand_with:
+ type: "array"
+ items:
+ type: "string"
+ enum:
+ - "renderedFields"
+ - "transitions"
+ - "changelog"
+ title: "Expand Issues stream"
+ airbyte_hidden: true
+ description: "Select fields to Expand the `Issues` stream when replicating\
+ \ with: "
+ default: []
+ lookback_window_minutes:
+ title: "Lookback window"
+ description: "When set to N, the connector will always refresh resources\
+ \ created within the past N minutes. By default, updated objects that\
+ \ are not newly created are not incrementally synced."
+ examples:
+ - 60
+ default: 0
+ minimum: 0
+ maximum: 576000
+ type: "integer"
+ order: 5
+ enable_experimental_streams:
+ type: "boolean"
+ title: "Enable Experimental Streams"
+ description: "Allow the use of experimental streams which rely on undocumented\
+ \ Jira API endpoints. See https://docs.airbyte.com/integrations/sources/jira#experimental-tables\
+ \ for more info."
+ default: false
+ order: 6
+ sourceType:
+ title: "jira"
+ const: "jira"
+ enum:
+ - "jira"
+ order: 0
+ type: "string"
+ source-jira-update:
+ title: "Jira Spec"
+ type: "object"
+ required:
+ - "api_token"
+ - "domain"
+ - "email"
+ properties:
+ api_token:
+ type: "string"
+ title: "API Token"
+ description: "Jira API Token. See the docs for more information on how to generate this key. API Token\
+ \ is used for Authorization to your account by BasicAuth."
+ airbyte_secret: true
+ order: 0
+ domain:
+ type: "string"
+ title: "Domain"
+ examples:
+ - ".atlassian.net"
+ - ".jira.com"
+ - "jira..com"
+ description: "The Domain for your Jira account, e.g. airbyteio.atlassian.net,\
+ \ airbyteio.jira.com, jira.your-domain.com"
+ order: 1
+ email:
+ type: "string"
+ title: "Email"
+ description: "The user email for your Jira account which you used to generate\
+ \ the API token. This field is used for Authorization to your account\
+ \ by BasicAuth."
+ order: 2
+ projects:
+ type: "array"
+ title: "Projects"
+ items:
+ type: "string"
+ examples:
+ - "PROJ1"
+ - "PROJ2"
+ description: "List of Jira project keys to replicate data for, or leave\
+ \ it empty if you want to replicate data for all projects."
+ order: 3
+ start_date:
+ type: "string"
+ title: "Start Date"
+ description: "The date from which you want to replicate data from Jira,\
+ \ use the format YYYY-MM-DDT00:00:00Z. Note that this field only applies\
+ \ to certain streams, and only data generated on or after the start date\
+ \ will be replicated. Or leave it empty if you want to replicate all data.\
+ \ For more information, refer to the documentation."
+ examples:
+ - "2021-03-01T00:00:00Z"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ format: "date-time"
+ order: 4
+ expand_issue_changelog:
+ type: "boolean"
+ title: "Expand Issue Changelog"
+ airbyte_hidden: true
+ description: "(DEPRECATED) Expand the changelog when replicating issues."
+ default: false
+ render_fields:
+ type: "boolean"
+ title: "Render Issue Fields"
+ airbyte_hidden: true
+ description: "(DEPRECATED) Render issue fields in HTML format in addition\
+ \ to Jira JSON-like format."
+ default: false
+ expand_issue_transition:
+ type: "boolean"
+ title: "Expand Issue Transitions"
+ airbyte_hidden: true
+ description: "(DEPRECATED) Expand the transitions when replicating issues."
+ default: false
+ issues_stream_expand_with:
+ type: "array"
+ items:
+ type: "string"
+ enum:
+ - "renderedFields"
+ - "transitions"
+ - "changelog"
+ title: "Expand Issues stream"
+ airbyte_hidden: true
+ description: "Select fields to Expand the `Issues` stream when replicating\
+ \ with: "
+ default: []
+ lookback_window_minutes:
+ title: "Lookback window"
+ description: "When set to N, the connector will always refresh resources\
+ \ created within the past N minutes. By default, updated objects that\
+ \ are not newly created are not incrementally synced."
+ examples:
+ - 60
+ default: 0
+ minimum: 0
+ maximum: 576000
+ type: "integer"
+ order: 5
+ enable_experimental_streams:
+ type: "boolean"
+ title: "Enable Experimental Streams"
+ description: "Allow the use of experimental streams which rely on undocumented\
+ \ Jira API endpoints. See https://docs.airbyte.com/integrations/sources/jira#experimental-tables\
+ \ for more info."
+ default: false
+ order: 6
+ source-hubspot:
+ title: "HubSpot Source Spec"
+ type: "object"
+ required:
+ - "credentials"
+ - "sourceType"
+ properties:
+ start_date:
+ type: "string"
+ title: "Start date"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
+ \ data before this date will not be replicated. If not set, \"2006-06-01T00:00:00Z\"\
+ \ (Hubspot creation date) will be used as start date. It's recommended\
+ \ to provide relevant to your data start date value to optimize synchronization."
+ examples:
+ - "2017-01-25T00:00:00Z"
+ format: "date-time"
+ credentials:
+ title: "Authentication"
+ description: "Choose how to authenticate to HubSpot."
+ type: "object"
+ oneOf:
+ - type: "object"
+ title: "OAuth"
+ required:
+ - "client_id"
+ - "client_secret"
+ - "refresh_token"
+ - "credentials_title"
+ properties:
+ credentials_title:
+ type: "string"
+ title: "Auth Type"
+ description: "Name of the credentials"
+ const: "OAuth Credentials"
+ order: 0
+ enum:
+ - "OAuth Credentials"
+ client_id:
+ title: "Client ID"
+ description: "The Client ID of your HubSpot developer application.\
+ \ See the Hubspot docs if you need help finding this ID."
+ type: "string"
+ examples:
+ - "123456789000"
+ client_secret:
+ title: "Client Secret"
+ description: "The client secret for your HubSpot developer application.\
+ \ See the Hubspot docs if you need help finding this secret."
+ type: "string"
+ examples:
+ - "secret"
+ airbyte_secret: true
+ refresh_token:
+ title: "Refresh Token"
+ description: "Refresh token to renew an expired access token. See\
+ \ the Hubspot docs if you need help finding this token."
+ type: "string"
+ examples:
+ - "refresh_token"
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ - type: "object"
+ title: "Private App"
+ required:
+ - "access_token"
+ - "credentials_title"
+ properties:
+ credentials_title:
+ type: "string"
+ title: "Auth Type"
+ description: "Name of the credentials set"
+ const: "Private App Credentials"
+ order: 0
+ enum:
+ - "Private App Credentials"
+ access_token:
+ title: "Access token"
+ description: "HubSpot Access token. See the Hubspot docs if you need help finding this token."
+ type: "string"
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ enable_experimental_streams:
+ title: "Enable experimental streams"
+ description: "If enabled then experimental streams become available for\
+ \ sync."
+ type: "boolean"
+ default: false
+ sourceType:
+ title: "hubspot"
+ const: "hubspot"
+ enum:
+ - "hubspot"
+ order: 0
+ type: "string"
+ source-hubspot-update:
+ title: "HubSpot Source Spec"
+ type: "object"
+ required:
+ - "credentials"
+ properties:
+ start_date:
+ type: "string"
+ title: "Start date"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
+ \ data before this date will not be replicated. If not set, \"2006-06-01T00:00:00Z\"\
+ \ (Hubspot creation date) will be used as start date. It's recommended\
+ \ to provide relevant to your data start date value to optimize synchronization."
+ examples:
+ - "2017-01-25T00:00:00Z"
+ format: "date-time"
+ credentials:
+ title: "Authentication"
+ description: "Choose how to authenticate to HubSpot."
+ type: "object"
+ oneOf:
+ - type: "object"
+ title: "OAuth"
+ required:
+ - "client_id"
+ - "client_secret"
+ - "refresh_token"
+ - "credentials_title"
+ properties:
+ credentials_title:
+ type: "string"
+ title: "Auth Type"
+ description: "Name of the credentials"
+ const: "OAuth Credentials"
+ order: 0
+ enum:
+ - "OAuth Credentials"
+ client_id:
+ title: "Client ID"
+ description: "The Client ID of your HubSpot developer application.\
+ \ See the Hubspot docs if you need help finding this ID."
+ type: "string"
+ examples:
+ - "123456789000"
+ client_secret:
+ title: "Client Secret"
+ description: "The client secret for your HubSpot developer application.\
+ \ See the Hubspot docs if you need help finding this secret."
+ type: "string"
+ examples:
+ - "secret"
+ airbyte_secret: true
+ refresh_token:
+ title: "Refresh Token"
+ description: "Refresh token to renew an expired access token. See\
+ \ the Hubspot docs if you need help finding this token."
+ type: "string"
+ examples:
+ - "refresh_token"
+ airbyte_secret: true
+ - type: "object"
+ title: "Private App"
+ required:
+ - "access_token"
+ - "credentials_title"
+ properties:
+ credentials_title:
+ type: "string"
+ title: "Auth Type"
+ description: "Name of the credentials set"
+ const: "Private App Credentials"
+ order: 0
+ enum:
+ - "Private App Credentials"
+ access_token:
+ title: "Access token"
+ description: "HubSpot Access token. See the Hubspot docs if you need help finding this token."
+ type: "string"
+ airbyte_secret: true
+ enable_experimental_streams:
+ title: "Enable experimental streams"
+ description: "If enabled then experimental streams become available for\
+ \ sync."
+ type: "boolean"
+ default: false
+ source-rss:
+ title: "RSS Spec"
+ type: "object"
+ required:
+ - "url"
+ - "sourceType"
+ properties:
+ url:
+ type: "string"
+ description: "RSS Feed URL"
+ sourceType:
+ title: "rss"
+ const: "rss"
+ enum:
+ - "rss"
+ order: 0
+ type: "string"
+ source-rss-update:
+ title: "RSS Spec"
+ type: "object"
+ required:
+ - "url"
+ properties:
+ url:
+ type: "string"
+ description: "RSS Feed URL"
+ source-sap-fieldglass:
+ title: "Sap Fieldglass Spec"
+ type: "object"
+ required:
+ - "api_key"
+ - "sourceType"
+ properties:
+ api_key:
+ type: "string"
+ description: "API Key"
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ sourceType:
+ title: "sap-fieldglass"
+ const: "sap-fieldglass"
+ enum:
+ - "sap-fieldglass"
+ order: 0
+ type: "string"
+ source-sap-fieldglass-update:
+ title: "Sap Fieldglass Spec"
+ type: "object"
+ required:
+ - "api_key"
+ properties:
+ api_key:
+ type: "string"
+ description: "API Key"
+ airbyte_secret: true
+ source-twilio-taskrouter:
+ title: "Twilio Taskrouter Spec"
+ type: "object"
+ required:
+ - "account_sid"
+ - "auth_token"
+ - "sourceType"
+ properties:
+ account_sid:
+ title: "Account SID"
+ type: "string"
+ description: "Twilio Account ID"
+ airbyte_secret: true
+ auth_token:
+ type: "string"
+ description: "Twilio Auth Token"
+ airbyte_secret: true
+ title: "Auth Token"
+ x-speakeasy-param-sensitive: true
+ sourceType:
+ title: "twilio-taskrouter"
+ const: "twilio-taskrouter"
+ enum:
+ - "twilio-taskrouter"
+ order: 0
+ type: "string"
+ source-twilio-taskrouter-update:
+ title: "Twilio Taskrouter Spec"
+ type: "object"
+ required:
+ - "account_sid"
+ - "auth_token"
+ properties:
+ account_sid:
+ title: "Account SID"
+ type: "string"
+ description: "Twilio Account ID"
+ airbyte_secret: true
+ auth_token:
+ type: "string"
+ description: "Twilio Auth Token"
+ airbyte_secret: true
+ title: "Auth Token"
+ source-xkcd:
+ title: "Xkcd Spec"
+ type: "object"
+ properties:
+ sourceType:
+ title: "xkcd"
+ const: "xkcd"
+ enum:
+ - "xkcd"
+ order: 0
+ type: "string"
+ source-xkcd-update:
+ title: "Xkcd Spec"
+ type: "object"
+ properties: {}
+ source-zenloop:
+ title: "Zenloop Spec"
+ type: "object"
+ required:
+ - "api_token"
+ - "sourceType"
+ properties:
+ api_token:
+ type: "string"
+ description: "Zenloop API Token. You can get the API token in settings page\
+ \ here "
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ date_from:
+ type: "string"
+ description: "Zenloop date_from. Format: 2021-10-24T03:30:30Z or 2021-10-24.\
+ \ Leave empty if only data from current data should be synced"
+ examples:
+ - "2021-10-24T03:30:30Z"
+ survey_id:
+ type: "string"
+ description: "Zenloop Survey ID. Can be found here. Leave empty to pull answers from all surveys"
+ airbyte_secret: true
+ survey_group_id:
+ type: "string"
+ description: "Zenloop Survey Group ID. Can be found by pulling All Survey\
+ \ Groups via SurveyGroups stream. Leave empty to pull answers from all\
+ \ survey groups"
+ airbyte_secret: true
+ sourceType:
+ title: "zenloop"
+ const: "zenloop"
+ enum:
+ - "zenloop"
+ order: 0
+ type: "string"
+ source-zenloop-update:
+ title: "Zenloop Spec"
+ type: "object"
+ required:
+ - "api_token"
+ properties:
+ api_token:
+ type: "string"
+ description: "Zenloop API Token. You can get the API token in settings page\
+ \ here "
+ airbyte_secret: true
+ date_from:
+ type: "string"
+ description: "Zenloop date_from. Format: 2021-10-24T03:30:30Z or 2021-10-24.\
+ \ Leave empty if only data from current data should be synced"
+ examples:
+ - "2021-10-24T03:30:30Z"
+ survey_id:
+ type: "string"
+ description: "Zenloop Survey ID. Can be found here. Leave empty to pull answers from all surveys"
+ airbyte_secret: true
+ survey_group_id:
+ type: "string"
+ description: "Zenloop Survey Group ID. Can be found by pulling All Survey\
+ \ Groups via SurveyGroups stream. Leave empty to pull answers from all\
+ \ survey groups"
+ airbyte_secret: true
+ source-tempo:
+ title: "Tempo Spec"
+ type: "object"
+ required:
+ - "api_token"
+ - "sourceType"
+ properties:
+ api_token:
+ type: "string"
+ title: "API token"
+ description: "Tempo API Token. Go to Tempo>Settings, scroll down to Data\
+ \ Access and select API integration."
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ sourceType:
+ title: "tempo"
+ const: "tempo"
+ enum:
+ - "tempo"
+ order: 0
+ type: "string"
+ source-tempo-update:
+ title: "Tempo Spec"
+ type: "object"
+ required:
+ - "api_token"
+ properties:
+ api_token:
+ type: "string"
+ title: "API token"
+ description: "Tempo API Token. Go to Tempo>Settings, scroll down to Data\
+ \ Access and select API integration."
+ airbyte_secret: true
+ source-chargebee:
+ title: "Chargebee Spec"
+ type: "object"
+ required:
+ - "site"
+ - "site_api_key"
+ - "start_date"
+ - "sourceType"
+ properties:
+ site_api_key:
+ type: "string"
+ title: "API Key"
+ description: "Chargebee API Key. See the docs for more information on how to obtain this key."
+ airbyte_secret: true
+ order: 0
+ x-speakeasy-param-sensitive: true
+ site:
+ type: "string"
+ title: "Site"
+ description: "The site prefix for your Chargebee instance."
+ examples:
+ - "airbyte-test"
+ order: 1
+ start_date:
+ type: "string"
+ format: "date-time"
+ title: "Start Date"
+ description: "UTC date and time in the format 2017-01-25T00:00:00.000Z.\
+ \ Any data before this date will not be replicated."
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ examples:
+ - "2021-01-25T00:00:00Z"
+ order: 2
+ product_catalog:
+ type: "string"
+ title: "Product Catalog"
+ description: "Product Catalog version of your Chargebee site. Instructions\
+ \ on how to find your version you may find here under `API Version` section. If left blank, the product catalog\
+ \ version will be set to 2.0."
+ enum:
+ - "1.0"
+ - "2.0"
+ default: "2.0"
+ order: 3
+ sourceType:
+ title: "chargebee"
+ const: "chargebee"
+ enum:
+ - "chargebee"
+ order: 0
+ type: "string"
+ source-chargebee-update:
+ title: "Chargebee Spec"
+ type: "object"
+ required:
+ - "site"
+ - "site_api_key"
+ - "start_date"
+ properties:
+ site_api_key:
+ type: "string"
+ title: "API Key"
+ description: "Chargebee API Key. See the docs for more information on how to obtain this key."
+ airbyte_secret: true
+ order: 0
+ site:
+ type: "string"
+ title: "Site"
+ description: "The site prefix for your Chargebee instance."
+ examples:
+ - "airbyte-test"
+ order: 1
+ start_date:
+ type: "string"
+ format: "date-time"
+ title: "Start Date"
+ description: "UTC date and time in the format 2017-01-25T00:00:00.000Z.\
+ \ Any data before this date will not be replicated."
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ examples:
+ - "2021-01-25T00:00:00Z"
+ order: 2
+ product_catalog:
+ type: "string"
+ title: "Product Catalog"
+ description: "Product Catalog version of your Chargebee site. Instructions\
+ \ on how to find your version you may find here under `API Version` section. If left blank, the product catalog\
+ \ version will be set to 2.0."
+ enum:
+ - "1.0"
+ - "2.0"
+ default: "2.0"
+ order: 3
+ source-onesignal:
+ title: "OneSignal Source Spec"
+ type: "object"
+ required:
+ - "user_auth_key"
+ - "start_date"
+ - "outcome_names"
+ - "applications"
+ - "sourceType"
+ properties:
+ user_auth_key:
+ type: "string"
+ title: "User Auth Key"
+ description: "OneSignal User Auth Key, see the docs for more information on how to obtain this key."
+ airbyte_secret: true
+ order: 0
+ x-speakeasy-param-sensitive: true
+ applications:
+ type: "array"
+ title: "Applications"
+ description: "Applications keys, see the docs for more information on how to obtain this data"
+ items:
+ type: "object"
+ properties:
+ app_name:
+ type: "string"
+ title: "OneSignal App Name"
+ order: 0
+ app_id:
+ type: "string"
+ title: "OneSignal App ID"
+ order: 1
+ airbyte_secret: true
+ app_api_key:
+ type: "string"
+ title: "REST API Key"
+ order: 2
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ required:
+ - "app_id"
+ - "app_api_key"
+ order: 1
+ start_date:
+ type: "string"
+ title: "Start Date"
+ description: "The date from which you'd like to replicate data for OneSignal\
+ \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\
+ \ date will be replicated."
+ examples:
+ - "2020-11-16T00:00:00Z"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ format: "date-time"
+ order: 2
+ outcome_names:
+ type: "string"
+ title: "Outcome Names"
+ description: "Comma-separated list of names and the value (sum/count) for\
+ \ the returned outcome data. See the docs for more details"
+ examples:
+ - "os__session_duration.count,os__click.count,CustomOutcomeName.sum"
+ order: 3
+ sourceType:
+ title: "onesignal"
+ const: "onesignal"
+ enum:
+ - "onesignal"
+ order: 0
+ type: "string"
+ source-onesignal-update:
+ title: "OneSignal Source Spec"
+ type: "object"
+ required:
+ - "user_auth_key"
+ - "start_date"
+ - "outcome_names"
+ - "applications"
+ properties:
+ user_auth_key:
+ type: "string"
+ title: "User Auth Key"
+ description: "OneSignal User Auth Key, see the docs for more information on how to obtain this key."
+ airbyte_secret: true
+ order: 0
+ applications:
+ type: "array"
+ title: "Applications"
+ description: "Applications keys, see the docs for more information on how to obtain this data"
+ items:
+ type: "object"
+ properties:
+ app_name:
+ type: "string"
+ title: "OneSignal App Name"
+ order: 0
+ app_id:
+ type: "string"
+ title: "OneSignal App ID"
+ order: 1
+ airbyte_secret: true
+ app_api_key:
+ type: "string"
+ title: "REST API Key"
+ order: 2
+ airbyte_secret: true
+ required:
+ - "app_id"
+ - "app_api_key"
+ order: 1
+ start_date:
+ type: "string"
+ title: "Start Date"
+ description: "The date from which you'd like to replicate data for OneSignal\
+ \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\
+ \ date will be replicated."
+ examples:
+ - "2020-11-16T00:00:00Z"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+ format: "date-time"
+ order: 2
+ outcome_names:
+ type: "string"
+ title: "Outcome Names"
+ description: "Comma-separated list of names and the value (sum/count) for\
+ \ the returned outcome data. See the docs for more details"
+ examples:
+ - "os__session_duration.count,os__click.count,CustomOutcomeName.sum"
+ order: 3
+ source-google-analytics-data-api:
+ title: "Google Analytics (Data API) Spec"
+ type: "object"
+ required:
+ - "property_ids"
+ - "sourceType"
+ properties:
+ credentials:
+ order: 0
+ type: "object"
+ title: "Credentials"
+ description: "Credentials for the service"
+ oneOf:
+ - title: "Authenticate via Google (Oauth)"
+ type: "object"
+ required:
+ - "client_id"
+ - "client_secret"
+ - "refresh_token"
+ properties:
+ auth_type:
+ type: "string"
+ const: "Client"
+ order: 0
+ enum:
+ - "Client"
+ client_id:
+ title: "Client ID"
+ type: "string"
+ description: "The Client ID of your Google Analytics developer application."
+ order: 1
+ client_secret:
+ title: "Client Secret"
+ type: "string"
+ description: "The Client Secret of your Google Analytics developer\
+ \ application."
+ airbyte_secret: true
+ order: 2
+ refresh_token:
+ title: "Refresh Token"
+ type: "string"
+ description: "The token for obtaining a new access token."
+ airbyte_secret: true
+ order: 3
+ x-speakeasy-param-sensitive: true
+ access_token:
+ title: "Access Token"
+ type: "string"
+ description: "Access Token for making authenticated requests."
+ airbyte_secret: true
+ order: 4
+ x-speakeasy-param-sensitive: true
+ - type: "object"
+ title: "Service Account Key Authentication"
+ required:
+ - "credentials_json"
+ properties:
+ auth_type:
+ type: "string"
+ const: "Service"
+ order: 0
+ enum:
+ - "Service"
+ credentials_json:
+ title: "Service Account JSON Key"
+ type: "string"
+ description: "The JSON key linked to the service account used for\
+ \ authorization. For steps on obtaining this key, refer to the setup guide."
+ examples:
+ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID,\
+ \ \"private_key_id\": YOUR_PRIVATE_KEY, ... }"
+ airbyte_secret: true
+ order: 1
+ property_ids:
+ title: "Property IDs"
+ description: "A list of your Property IDs. The Property ID is a unique number\
+ \ assigned to each property in Google Analytics, found in your GA4 property\
+ \ URL. This ID allows the connector to track the specific events associated\
+ \ with your property. Refer to the Google\
+ \ Analytics documentation to locate your property ID."
+ order: 1
+ type: "array"
+ items:
+ type: "string"
+ pattern: "^[0-9]*$"
+ examples:
+ - - "1738294"
+ - "5729978930"
+ date_ranges_start_date:
+ type: "string"
+ title: "Start Date"
+ description: "The start date from which to replicate report data in the\
+ \ format YYYY-MM-DD. Data generated before this date will not be included\
+ \ in the report. Not applied to custom Cohort reports."
+ format: "date"
+ pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+ pattern_descriptor: "YYYY-MM-DD"
+ examples:
+ - "2021-01-01"
+ order: 2
+ custom_reports_array:
+ title: "Custom Reports"
+ description: "You can add your Custom Analytics report by creating one."
+ order: 4
+ type: "array"
+ items:
+ title: "Custom Report Config"
+ type: "object"
+ properties:
name:
title: "Name"
description: "The name of the custom report, this name would be used\
@@ -51332,7 +52177,9 @@ components:
type: "string"
profiles:
title: "Profile IDs"
- description: "Profile IDs you want to fetch data for. See docs for more details. Note: If Marketplace IDs are also selected,\
\ profiles will be selected if they match the Profile ID OR the Marketplace\
\ ID."
@@ -51464,7 +52311,9 @@ components:
type: "string"
profiles:
title: "Profile IDs"
- description: "Profile IDs you want to fetch data for. See docs for more details. Note: If Marketplace IDs are also selected,\
\ profiles will be selected if they match the Profile ID OR the Marketplace\
\ ID."
@@ -55193,12 +56042,12 @@ components:
title: "Initial Waiting Time in Seconds (Advanced)"
description: "The amount of time the connector will wait when it launches\
\ to determine if there is new data to sync or not. Defaults to\
- \ 300 seconds. Valid range: 120 seconds to 2400 seconds. Read about\
+ \ 300 seconds. Valid range: 120 seconds to 3600 seconds. Read about\
\ initial waiting time."
default: 300
min: 120
- max: 2400
+ max: 3600
order: 3
invalid_cdc_cursor_position_behavior:
type: "string"
@@ -55346,312 +56195,358 @@ components:
type: "string"
airbyte_secret: true
order: 4
- x-speakeasy-param-sensitive: true
- sourceType:
- title: "mssql"
- const: "mssql"
- enum:
- - "mssql"
- order: 0
- type: "string"
- source-mssql-update:
- title: "MSSQL Source Spec"
+ x-speakeasy-param-sensitive: true
+ sourceType:
+ title: "mssql"
+ const: "mssql"
+ enum:
+ - "mssql"
+ order: 0
+ type: "string"
+ source-mssql-update:
+ title: "MSSQL Source Spec"
+ type: "object"
+ required:
+ - "host"
+ - "port"
+ - "database"
+ - "username"
+ - "password"
+ properties:
+ host:
+ description: "The hostname of the database."
+ title: "Host"
+ type: "string"
+ order: 0
+ port:
+ description: "The port of the database."
+ title: "Port"
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ examples:
+ - "1433"
+ order: 1
+ database:
+ description: "The name of the database."
+ title: "Database"
+ type: "string"
+ examples:
+ - "master"
+ order: 2
+ schemas:
+ title: "Schemas"
+ description: "The list of schemas to sync from. Defaults to user. Case sensitive."
+ type: "array"
+ items:
+ type: "string"
+ minItems: 0
+ uniqueItems: true
+ default:
+ - "dbo"
+ order: 3
+ username:
+ description: "The username which is used to access the database."
+ title: "Username"
+ type: "string"
+ order: 4
+ password:
+ description: "The password associated with the username."
+ title: "Password"
+ type: "string"
+ airbyte_secret: true
+ order: 5
+ jdbc_url_params:
+ title: "JDBC URL Params"
+ description: "Additional properties to pass to the JDBC URL string when\
+ \ connecting to the database formatted as 'key=value' pairs separated\
+ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+ type: "string"
+ order: 6
+ ssl_method:
+ title: "SSL Method"
+ type: "object"
+ description: "The encryption method which is used when communicating with\
+ \ the database."
+ order: 7
+ oneOf:
+ - title: "Unencrypted"
+ description: "Data transfer will not be encrypted."
+ required:
+ - "ssl_method"
+ properties:
+ ssl_method:
+ type: "string"
+ const: "unencrypted"
+ enum:
+ - "unencrypted"
+ - title: "Encrypted (trust server certificate)"
+ description: "Use the certificate provided by the server without verification.\
+ \ (For testing purposes only!)"
+ required:
+ - "ssl_method"
+ properties:
+ ssl_method:
+ type: "string"
+ const: "encrypted_trust_server_certificate"
+ enum:
+ - "encrypted_trust_server_certificate"
+ - title: "Encrypted (verify certificate)"
+ description: "Verify and use the certificate provided by the server."
+ required:
+ - "ssl_method"
+ properties:
+ ssl_method:
+ type: "string"
+ const: "encrypted_verify_certificate"
+ enum:
+ - "encrypted_verify_certificate"
+ hostNameInCertificate:
+ title: "Host Name In Certificate"
+ type: "string"
+ description: "Specifies the host name of the server. The value of\
+ \ this property must match the subject property of the certificate."
+ order: 0
+ certificate:
+ title: "Certificate"
+ type: "string"
+ description: "certificate of the server, or of the CA that signed\
+ \ the server certificate"
+ order: 1
+ airbyte_secret: true
+ multiline: true
+ replication_method:
+ type: "object"
+ title: "Update Method"
+ description: "Configures how data is extracted from the database."
+ default: "CDC"
+ display_type: "radio"
+ order: 8
+ oneOf:
+ - title: "Read Changes using Change Data Capture (CDC)"
+ description: "Recommended - Incrementally reads new inserts, updates,\
+ \ and deletes using the SQL Server's change data capture feature. This must be enabled on your database."
+ required:
+ - "method"
+ properties:
+ method:
+ type: "string"
+ const: "CDC"
+ order: 0
+ enum:
+ - "CDC"
+ initial_waiting_seconds:
+ type: "integer"
+ title: "Initial Waiting Time in Seconds (Advanced)"
+ description: "The amount of time the connector will wait when it launches\
+ \ to determine if there is new data to sync or not. Defaults to\
+ \ 300 seconds. Valid range: 120 seconds to 3600 seconds. Read about\
+ \ initial waiting time."
+ default: 300
+ min: 120
+ max: 3600
+ order: 3
+ invalid_cdc_cursor_position_behavior:
+ type: "string"
+ title: "Invalid CDC position behavior (Advanced)"
+ description: "Determines whether Airbyte should fail or re-sync data\
+ \ in case of an stale/invalid cursor value into the WAL. If 'Fail\
+ \ sync' is chosen, a user will have to manually reset the connection\
+ \ before being able to continue syncing data. If 'Re-sync data'\
+ \ is chosen, Airbyte will automatically trigger a refresh but could\
+ \ lead to higher cloud costs and data loss."
+ enum:
+ - "Fail sync"
+ - "Re-sync data"
+ default: "Fail sync"
+ order: 4
+ queue_size:
+ type: "integer"
+ title: "Size of the queue (Advanced)"
+ description: "The size of the internal queue. This may interfere with\
+ \ memory consumption and efficiency of the connector, please be\
+ \ careful."
+ default: 10000
+ order: 5
+ min: 1000
+ max: 10000
+ - title: "Scan Changes with User Defined Cursor"
+ description: "Incrementally detects new inserts and updates using the\
+ \ cursor column chosen when configuring a connection (e.g. created_at,\
+ \ updated_at)."
+ required:
+ - "method"
+ properties:
+ method:
+ type: "string"
+ const: "STANDARD"
+ order: 0
+ enum:
+ - "STANDARD"
+ tunnel_method:
+ type: "object"
+ title: "SSH Tunnel Method"
+ description: "Whether to initiate an SSH tunnel before connecting to the\
+ \ database, and if so, which kind of authentication to use."
+ oneOf:
+ - title: "No Tunnel"
+ required:
+ - "tunnel_method"
+ properties:
+ tunnel_method:
+ description: "No ssh tunnel needed to connect to database"
+ type: "string"
+ const: "NO_TUNNEL"
+ order: 0
+ enum:
+ - "NO_TUNNEL"
+ - title: "SSH Key Authentication"
+ required:
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "ssh_key"
+ properties:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and ssh key"
+ type: "string"
+ const: "SSH_KEY_AUTH"
+ order: 0
+ enum:
+ - "SSH_KEY_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
+ type: "string"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host."
+ type: "string"
+ order: 3
+ ssh_key:
+ title: "SSH Private Key"
+ description: "OS-level user account ssh key credentials in RSA PEM\
+ \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
+ type: "string"
+ airbyte_secret: true
+ multiline: true
+ order: 4
+ - title: "Password Authentication"
+ required:
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "tunnel_user_password"
+ properties:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and password authentication"
+ type: "string"
+ const: "SSH_PASSWORD_AUTH"
+ order: 0
+ enum:
+ - "SSH_PASSWORD_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
+ type: "string"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host"
+ type: "string"
+ order: 3
+ tunnel_user_password:
+ title: "Password"
+ description: "OS-level password for logging into the jump server host"
+ type: "string"
+ airbyte_secret: true
+ order: 4
+ source-dynamodb:
+ title: "Dynamodb Source Spec"
type: "object"
- required:
- - "host"
- - "port"
- - "database"
- - "username"
- - "password"
properties:
- host:
- description: "The hostname of the database."
- title: "Host"
- type: "string"
+ credentials:
order: 0
- port:
- description: "The port of the database."
- title: "Port"
- type: "integer"
- minimum: 0
- maximum: 65536
- examples:
- - "1433"
- order: 1
- database:
- description: "The name of the database."
- title: "Database"
- type: "string"
- examples:
- - "master"
- order: 2
- schemas:
- title: "Schemas"
- description: "The list of schemas to sync from. Defaults to user. Case sensitive."
- type: "array"
- items:
- type: "string"
- minItems: 0
- uniqueItems: true
- default:
- - "dbo"
- order: 3
- username:
- description: "The username which is used to access the database."
- title: "Username"
- type: "string"
- order: 4
- password:
- description: "The password associated with the username."
- title: "Password"
- type: "string"
- airbyte_secret: true
- order: 5
- jdbc_url_params:
- title: "JDBC URL Params"
- description: "Additional properties to pass to the JDBC URL string when\
- \ connecting to the database formatted as 'key=value' pairs separated\
- \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
- type: "string"
- order: 6
- ssl_method:
- title: "SSL Method"
type: "object"
- description: "The encryption method which is used when communicating with\
- \ the database."
- order: 7
- oneOf:
- - title: "Unencrypted"
- description: "Data transfer will not be encrypted."
- required:
- - "ssl_method"
- properties:
- ssl_method:
- type: "string"
- const: "unencrypted"
- enum:
- - "unencrypted"
- - title: "Encrypted (trust server certificate)"
- description: "Use the certificate provided by the server without verification.\
- \ (For testing purposes only!)"
- required:
- - "ssl_method"
- properties:
- ssl_method:
- type: "string"
- const: "encrypted_trust_server_certificate"
- enum:
- - "encrypted_trust_server_certificate"
- - title: "Encrypted (verify certificate)"
- description: "Verify and use the certificate provided by the server."
- required:
- - "ssl_method"
- properties:
- ssl_method:
- type: "string"
- const: "encrypted_verify_certificate"
- enum:
- - "encrypted_verify_certificate"
- hostNameInCertificate:
- title: "Host Name In Certificate"
- type: "string"
- description: "Specifies the host name of the server. The value of\
- \ this property must match the subject property of the certificate."
- order: 0
- certificate:
- title: "Certificate"
- type: "string"
- description: "certificate of the server, or of the CA that signed\
- \ the server certificate"
- order: 1
- airbyte_secret: true
- multiline: true
- replication_method:
- type: "object"
- title: "Update Method"
- description: "Configures how data is extracted from the database."
- default: "CDC"
- display_type: "radio"
- order: 8
- oneOf:
- - title: "Read Changes using Change Data Capture (CDC)"
- description: "Recommended - Incrementally reads new inserts, updates,\
- \ and deletes using the SQL Server's change data capture feature. This must be enabled on your database."
- required:
- - "method"
- properties:
- method:
- type: "string"
- const: "CDC"
- order: 0
- enum:
- - "CDC"
- initial_waiting_seconds:
- type: "integer"
- title: "Initial Waiting Time in Seconds (Advanced)"
- description: "The amount of time the connector will wait when it launches\
- \ to determine if there is new data to sync or not. Defaults to\
- \ 300 seconds. Valid range: 120 seconds to 2400 seconds. Read about\
- \ initial waiting time."
- default: 300
- min: 120
- max: 2400
- order: 3
- invalid_cdc_cursor_position_behavior:
- type: "string"
- title: "Invalid CDC position behavior (Advanced)"
- description: "Determines whether Airbyte should fail or re-sync data\
- \ in case of an stale/invalid cursor value into the WAL. If 'Fail\
- \ sync' is chosen, a user will have to manually reset the connection\
- \ before being able to continue syncing data. If 'Re-sync data'\
- \ is chosen, Airbyte will automatically trigger a refresh but could\
- \ lead to higher cloud costs and data loss."
- enum:
- - "Fail sync"
- - "Re-sync data"
- default: "Fail sync"
- order: 4
- queue_size:
- type: "integer"
- title: "Size of the queue (Advanced)"
- description: "The size of the internal queue. This may interfere with\
- \ memory consumption and efficiency of the connector, please be\
- \ careful."
- default: 10000
- order: 5
- min: 1000
- max: 10000
- - title: "Scan Changes with User Defined Cursor"
- description: "Incrementally detects new inserts and updates using the\
- \ cursor column chosen when configuring a connection (e.g. created_at,\
- \ updated_at)."
- required:
- - "method"
- properties:
- method:
- type: "string"
- const: "STANDARD"
- order: 0
- enum:
- - "STANDARD"
- tunnel_method:
- type: "object"
- title: "SSH Tunnel Method"
- description: "Whether to initiate an SSH tunnel before connecting to the\
- \ database, and if so, which kind of authentication to use."
+ title: "Credentials"
+ description: "Credentials for the service"
oneOf:
- - title: "No Tunnel"
- required:
- - "tunnel_method"
- properties:
- tunnel_method:
- description: "No ssh tunnel needed to connect to database"
- type: "string"
- const: "NO_TUNNEL"
- order: 0
- enum:
- - "NO_TUNNEL"
- - title: "SSH Key Authentication"
+ - title: "Authenticate via Access Keys"
+ type:
+ - "null"
+ - "object"
required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "ssh_key"
+ - "access_key_id"
+ - "secret_access_key"
+ additionalProperties: true
properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and ssh key"
+ auth_type:
type: "string"
- const: "SSH_KEY_AUTH"
+ const: "User"
order: 0
enum:
- - "SSH_KEY_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
- type: "string"
+ - "User"
+ access_key_id:
order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
+ title: "Dynamodb Key Id"
+ type: "string"
+ description: "The access key id to access Dynamodb. Airbyte requires\
+ \ read permissions to the database"
+ airbyte_secret: true
examples:
- - "22"
+ - "A012345678910EXAMPLE"
+ x-speakeasy-param-sensitive: true
+ secret_access_key:
order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host."
- type: "string"
- order: 3
- ssh_key:
- title: "SSH Private Key"
- description: "OS-level user account ssh key credentials in RSA PEM\
- \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
+ title: "Dynamodb Access Key"
type: "string"
+ description: "The corresponding secret to the access key id."
airbyte_secret: true
- multiline: true
- order: 4
- - title: "Password Authentication"
- required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "tunnel_user_password"
+ examples:
+ - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
+ x-speakeasy-param-sensitive: true
+ - type: "object"
+ title: "Role Based Authentication"
+ additionalProperties: true
properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and password authentication"
+ auth_type:
type: "string"
- const: "SSH_PASSWORD_AUTH"
+ const: "Role"
order: 0
enum:
- - "SSH_PASSWORD_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
- type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
- examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host"
- type: "string"
- order: 3
- tunnel_user_password:
- title: "Password"
- description: "OS-level password for logging into the jump server host"
- type: "string"
- airbyte_secret: true
- order: 4
- source-dynamodb:
- title: "Dynamodb Source Spec"
- type: "object"
- required:
- - "access_key_id"
- - "secret_access_key"
- - "sourceType"
- properties:
+ - "Role"
endpoint:
title: "Dynamodb Endpoint"
type: "string"
@@ -55699,23 +56594,6 @@ components:
- "us-gov-west-1"
- "us-west-1"
- "us-west-2"
- access_key_id:
- title: "Dynamodb Key Id"
- type: "string"
- description: "The access key id to access Dynamodb. Airbyte requires read\
- \ permissions to the database"
- airbyte_secret: true
- examples:
- - "A012345678910EXAMPLE"
- x-speakeasy-param-sensitive: true
- secret_access_key:
- title: "Dynamodb Access Key"
- type: "string"
- description: "The corresponding secret to the access key id."
- airbyte_secret: true
- examples:
- - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
- x-speakeasy-param-sensitive: true
reserved_attribute_names:
title: "Reserved attribute names"
type: "string"
@@ -55723,6 +56601,11 @@ components:
airbyte_secret: true
examples:
- "name, field_name, field-name"
+ ignore_missing_read_permissions_tables:
+ title: "Ignore missing read permissions tables"
+ type: "boolean"
+ description: "Ignore tables with missing scan/read permissions"
+ default: false
sourceType:
title: "dynamodb"
const: "dynamodb"
@@ -55733,10 +56616,55 @@ components:
source-dynamodb-update:
title: "Dynamodb Source Spec"
type: "object"
- required:
- - "access_key_id"
- - "secret_access_key"
properties:
+ credentials:
+ order: 0
+ type: "object"
+ title: "Credentials"
+ description: "Credentials for the service"
+ oneOf:
+ - title: "Authenticate via Access Keys"
+ type:
+ - "null"
+ - "object"
+ required:
+ - "access_key_id"
+ - "secret_access_key"
+ additionalProperties: true
+ properties:
+ auth_type:
+ type: "string"
+ const: "User"
+ order: 0
+ enum:
+ - "User"
+ access_key_id:
+ order: 1
+ title: "Dynamodb Key Id"
+ type: "string"
+ description: "The access key id to access Dynamodb. Airbyte requires\
+ \ read permissions to the database"
+ airbyte_secret: true
+ examples:
+ - "A012345678910EXAMPLE"
+ secret_access_key:
+ order: 2
+ title: "Dynamodb Access Key"
+ type: "string"
+ description: "The corresponding secret to the access key id."
+ airbyte_secret: true
+ examples:
+ - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
+ - type: "object"
+ title: "Role Based Authentication"
+ additionalProperties: true
+ properties:
+ auth_type:
+ type: "string"
+ const: "Role"
+ order: 0
+ enum:
+ - "Role"
endpoint:
title: "Dynamodb Endpoint"
type: "string"
@@ -55784,21 +56712,6 @@ components:
- "us-gov-west-1"
- "us-west-1"
- "us-west-2"
- access_key_id:
- title: "Dynamodb Key Id"
- type: "string"
- description: "The access key id to access Dynamodb. Airbyte requires read\
- \ permissions to the database"
- airbyte_secret: true
- examples:
- - "A012345678910EXAMPLE"
- secret_access_key:
- title: "Dynamodb Access Key"
- type: "string"
- description: "The corresponding secret to the access key id."
- airbyte_secret: true
- examples:
- - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
reserved_attribute_names:
title: "Reserved attribute names"
type: "string"
@@ -55806,6 +56719,11 @@ components:
airbyte_secret: true
examples:
- "name, field_name, field-name"
+ ignore_missing_read_permissions_tables:
+ title: "Ignore missing read permissions tables"
+ type: "boolean"
+ description: "Ignore tables with missing scan/read permissions"
+ default: false
source-salesforce:
title: "Salesforce Source Spec"
type: "object"
@@ -58693,20 +59611,6 @@ components:
order: 0
trello:
title: null
- harvest:
- properties:
- credentials:
- properties:
- client_id:
- title: "Client ID"
- type: "string"
- description: "The Client ID of your Harvest developer application."
- client_secret:
- title: "Client Secret"
- type: "string"
- description: "The Client Secret of your Harvest developer application."
- airbyte_secret: true
- title: "Harvest Spec"
zendesk-chat:
properties:
credentials:
@@ -58838,6 +59742,21 @@ components:
airbyte_hidden: true
type: "string"
title: "Source Instagram"
+ azure-blob-storage:
+ properties:
+ credentials:
+ properties:
+ client_id:
+ title: "Client ID"
+ description: "Client ID of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ client_secret:
+ title: "Client Secret"
+ description: "Client Secret of your Microsoft developer application"
+ airbyte_secret: true
+ type: "string"
+ title: "SourceAzureBlobStorageSpec"
zendesk-sunshine:
properties:
credentials:
@@ -60560,6 +61479,11 @@ components:
description: "Specifies the host name of the server. The value of\
\ this property must match the subject property of the certificate."
order: 8
+ raw_data_schema:
+ type: "string"
+ description: "The schema to write raw tables into (default: airbyte_internal)"
+ title: "Raw Table Schema Name"
+ order: 7
tunnel_method:
type: "object"
title: "SSH Tunnel Method"
@@ -60774,6 +61698,11 @@ components:
description: "Specifies the host name of the server. The value of\
\ this property must match the subject property of the certificate."
order: 8
+ raw_data_schema:
+ type: "string"
+ description: "The schema to write raw tables into (default: airbyte_internal)"
+ title: "Raw Table Schema Name"
+ order: 7
tunnel_method:
type: "object"
title: "SSH Tunnel Method"
@@ -60930,6 +61859,20 @@ components:
title: "JDBC URL Params"
type: "string"
order: 6
+ raw_data_schema:
+ type: "string"
+ description: "The database to write raw tables into"
+ title: "Raw table database (defaults to airbyte_internal)"
+ order: 7
+ disable_type_dedupe:
+ type: "boolean"
+ default: false
+ description: "Disable Writing Final Tables. WARNING! The data format in\
+ \ _airbyte_data is likely stable but there are no guarantees that other\
+ \ metadata columns will remain the same in future versions"
+ title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
+ \ table schema might change between versions)"
+ order: 8
tunnel_method:
type: "object"
title: "SSH Tunnel Method"
@@ -61093,6 +62036,20 @@ components:
title: "JDBC URL Params"
type: "string"
order: 6
+ raw_data_schema:
+ type: "string"
+ description: "The database to write raw tables into"
+ title: "Raw table database (defaults to airbyte_internal)"
+ order: 7
+ disable_type_dedupe:
+ type: "boolean"
+ default: false
+ description: "Disable Writing Final Tables. WARNING! The data format in\
+ \ _airbyte_data is likely stable but there are no guarantees that other\
+ \ metadata columns will remain the same in future versions"
+ title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
+ \ table schema might change between versions)"
+ order: 8
tunnel_method:
type: "object"
title: "SSH Tunnel Method"
@@ -63871,20 +64828,10 @@ components:
- "Standard"
raw_data_schema:
type: "string"
- description: "The schema to write raw tables into"
+ description: "The schema to write raw tables into (default: airbyte_internal)."
title: "Destinations V2 Raw Table Schema"
order: 9
group: "connection"
- enable_incremental_final_table_updates:
- type: "boolean"
- default: false
- description: "When enabled your data will load into your final tables incrementally\
- \ while your data is still being synced. When Disabled (the default),\
- \ your data loads into your final tables once at the end of a sync. Note\
- \ that this option only applies if you elect to create Final tables"
- title: "Enable Loading Data Incrementally to Final Tables"
- order: 10
- group: "connection"
disable_type_dedupe:
type: "boolean"
default: false
@@ -64257,20 +65204,10 @@ components:
- "Standard"
raw_data_schema:
type: "string"
- description: "The schema to write raw tables into"
+ description: "The schema to write raw tables into (default: airbyte_internal)."
title: "Destinations V2 Raw Table Schema"
order: 9
group: "connection"
- enable_incremental_final_table_updates:
- type: "boolean"
- default: false
- description: "When enabled your data will load into your final tables incrementally\
- \ while your data is still being synced. When Disabled (the default),\
- \ your data loads into your final tables once at the end of a sync. Note\
- \ that this option only applies if you elect to create Final tables"
- title: "Enable Loading Data Incrementally to Final Tables"
- order: 10
- group: "connection"
disable_type_dedupe:
type: "boolean"
default: false
@@ -64988,7 +65925,435 @@ components:
- "embedding"
- "processing"
- "indexing"
- - "destinationType"
+ - "destinationType"
+ groups:
+ - id: "processing"
+ title: "Processing"
+ - id: "embedding"
+ title: "Embedding"
+ - id: "indexing"
+ title: "Indexing"
+ - id: "advanced"
+ title: "Advanced"
+ destination-qdrant-update:
+ title: "Destination Config"
+ description: "The configuration model for the Vector DB based destinations.\
+ \ This model is used to generate the UI for the destination configuration,\n\
+ as well as to provide type safety for the configuration passed to the destination.\n\
+ \nThe configuration model is composed of four parts:\n* Processing configuration\n\
+ * Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\
+ \nProcessing, embedding and advanced configuration are provided by this base\
+ \ class, while the indexing configuration is provided by the destination connector\
+ \ in the sub class."
+ type: "object"
+ properties:
+ embedding:
+ title: "Embedding"
+ description: "Embedding configuration"
+ group: "embedding"
+ type: "object"
+ oneOf:
+ - title: "OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "openai"
+ const: "openai"
+ enum:
+ - "openai"
+ type: "string"
+ openai_key:
+ title: "OpenAI API key"
+ airbyte_secret: true
+ type: "string"
+ required:
+ - "openai_key"
+ - "mode"
+ description: "Use the OpenAI API to embed text. This option is using the\
+ \ text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "Cohere"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "cohere"
+ const: "cohere"
+ enum:
+ - "cohere"
+ type: "string"
+ cohere_key:
+ title: "Cohere API key"
+ airbyte_secret: true
+ type: "string"
+ required:
+ - "cohere_key"
+ - "mode"
+ description: "Use the Cohere API to embed text."
+ - title: "Fake"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "fake"
+ const: "fake"
+ enum:
+ - "fake"
+ type: "string"
+ description: "Use a fake embedding made out of random vectors with 1536\
+ \ embedding dimensions. This is useful for testing the data pipeline\
+ \ without incurring any costs."
+ required:
+ - "mode"
+ - title: "Azure OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "azure_openai"
+ const: "azure_openai"
+ enum:
+ - "azure_openai"
+ type: "string"
+ openai_key:
+ title: "Azure OpenAI API key"
+ description: "The API key for your Azure OpenAI resource. You can\
+ \ find this in the Azure portal under your Azure OpenAI resource"
+ airbyte_secret: true
+ type: "string"
+ api_base:
+ title: "Resource base URL"
+ description: "The base URL for your Azure OpenAI resource. You can\
+ \ find this in the Azure portal under your Azure OpenAI resource"
+ examples:
+ - "https://your-resource-name.openai.azure.com"
+ type: "string"
+ deployment:
+ title: "Deployment"
+ description: "The deployment for your Azure OpenAI resource. You\
+ \ can find this in the Azure portal under your Azure OpenAI resource"
+ examples:
+ - "your-resource-name"
+ type: "string"
+ required:
+ - "openai_key"
+ - "api_base"
+ - "deployment"
+ - "mode"
+ description: "Use the Azure-hosted OpenAI API to embed text. This option\
+ \ is using the text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "OpenAI-compatible"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "openai_compatible"
+ const: "openai_compatible"
+ enum:
+ - "openai_compatible"
+ type: "string"
+ api_key:
+ title: "API key"
+ default: ""
+ airbyte_secret: true
+ type: "string"
+ base_url:
+ title: "Base URL"
+ description: "The base URL for your OpenAI-compatible service"
+ examples:
+ - "https://your-service-name.com"
+ type: "string"
+ model_name:
+ title: "Model name"
+ description: "The name of the model to use for embedding"
+ default: "text-embedding-ada-002"
+ examples:
+ - "text-embedding-ada-002"
+ type: "string"
+ dimensions:
+ title: "Embedding dimensions"
+ description: "The number of dimensions the embedding model is generating"
+ examples:
+ - 1536
+ - 384
+ type: "integer"
+ required:
+ - "base_url"
+ - "dimensions"
+ - "mode"
+ description: "Use a service that's compatible with the OpenAI API to embed\
+ \ text."
+ processing:
+ title: "ProcessingConfigModel"
+ type: "object"
+ properties:
+ chunk_size:
+ title: "Chunk size"
+ description: "Size of chunks in tokens to store in vector store (make\
+ \ sure it is not too big for the context if your LLM)"
+ maximum: 8191
+ minimum: 1
+ type: "integer"
+ chunk_overlap:
+ title: "Chunk overlap"
+ description: "Size of overlap between chunks in tokens to store in vector\
+ \ store to better capture relevant context"
+ default: 0
+ type: "integer"
+ text_fields:
+ title: "Text fields to embed"
+ description: "List of fields in the record that should be used to calculate\
+ \ the embedding. The field list is applied to all streams in the same\
+ \ way and non-existing fields are ignored. If none are defined, all\
+ \ fields are considered text fields. When specifying text fields,\
+ \ you can access nested fields in the record by using dot notation,\
+ \ e.g. `user.name` will access the `name` field in the `user` object.\
+ \ It's also possible to use wildcards to access all fields in an object,\
+ \ e.g. `users.*.name` will access all `names` fields in all entries\
+ \ of the `users` array."
+ default: []
+ always_show: true
+ examples:
+ - "text"
+ - "user.name"
+ - "users.*.name"
+ type: "array"
+ items:
+ type: "string"
+ metadata_fields:
+ title: "Fields to store as metadata"
+ description: "List of fields in the record that should be stored as\
+ \ metadata. The field list is applied to all streams in the same way\
+ \ and non-existing fields are ignored. If none are defined, all fields\
+ \ are considered metadata fields. When specifying text fields, you\
+ \ can access nested fields in the record by using dot notation, e.g.\
+ \ `user.name` will access the `name` field in the `user` object. It's\
+ \ also possible to use wildcards to access all fields in an object,\
+ \ e.g. `users.*.name` will access all `names` fields in all entries\
+ \ of the `users` array. When specifying nested paths, all matching\
+ \ values are flattened into an array set to a field named by the path."
+ default: []
+ always_show: true
+ examples:
+ - "age"
+ - "user"
+ - "user.name"
+ type: "array"
+ items:
+ type: "string"
+ text_splitter:
+ title: "Text splitter"
+ description: "Split text fields into chunks based on the specified method."
+ type: "object"
+ oneOf:
+ - title: "By Separator"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "separator"
+ const: "separator"
+ enum:
+ - "separator"
+ type: "string"
+ separators:
+ title: "Separators"
+ description: "List of separator strings to split text fields by.\
+ \ The separator itself needs to be wrapped in double quotes,\
+ \ e.g. to split by the dot character, use \".\". To split by\
+ \ a newline, use \"\\n\"."
+ default:
+ - "\"\\n\\n\""
+ - "\"\\n\""
+ - "\" \""
+ - "\"\""
+ type: "array"
+ items:
+ type: "string"
+ keep_separator:
+ title: "Keep separator"
+ description: "Whether to keep the separator in the resulting chunks"
+ default: false
+ type: "boolean"
+ description: "Split the text by the list of separators until the chunk\
+ \ size is reached, using the earlier mentioned separators where\
+ \ possible. This is useful for splitting text fields by paragraphs,\
+ \ sentences, words, etc."
+ required:
+ - "mode"
+ - title: "By Markdown header"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "markdown"
+ const: "markdown"
+ enum:
+ - "markdown"
+ type: "string"
+ split_level:
+ title: "Split level"
+ description: "Level of markdown headers to split text fields by.\
+ \ Headings down to the specified level will be used as split\
+ \ points"
+ default: 1
+ minimum: 1
+ maximum: 6
+ type: "integer"
+ description: "Split the text by Markdown headers down to the specified\
+ \ header level. If the chunk size fits multiple sections, they will\
+ \ be combined into a single chunk."
+ required:
+ - "mode"
+ - title: "By Programming Language"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "code"
+ const: "code"
+ enum:
+ - "code"
+ type: "string"
+ language:
+ title: "Language"
+ description: "Split code in suitable places based on the programming\
+ \ language"
+ enum:
+ - "cpp"
+ - "go"
+ - "java"
+ - "js"
+ - "php"
+ - "proto"
+ - "python"
+ - "rst"
+ - "ruby"
+ - "rust"
+ - "scala"
+ - "swift"
+ - "markdown"
+ - "latex"
+ - "html"
+ - "sol"
+ type: "string"
+ required:
+ - "language"
+ - "mode"
+ description: "Split the text by suitable delimiters based on the programming\
+ \ language. This is useful for splitting code into chunks."
+ field_name_mappings:
+ title: "Field name mappings"
+ description: "List of fields to rename. Not applicable for nested fields,\
+ \ but can be used to rename fields already flattened via dot notation."
+ default: []
+ type: "array"
+ items:
+ title: "FieldNameMappingConfigModel"
+ type: "object"
+ properties:
+ from_field:
+ title: "From field name"
+ description: "The field name in the source"
+ type: "string"
+ to_field:
+ title: "To field name"
+ description: "The field name to use in the destination"
+ type: "string"
+ required:
+ - "from_field"
+ - "to_field"
+ required:
+ - "chunk_size"
+ group: "processing"
+ omit_raw_text:
+ title: "Do not store raw text"
+ description: "Do not store the text that gets embedded along with the vector\
+ \ and the metadata in the destination. If set to true, only the vector\
+ \ and the metadata will be stored - in this case raw text for LLM use\
+ \ cases needs to be retrieved from another source."
+ default: false
+ group: "advanced"
+ type: "boolean"
+ indexing:
+ title: "Indexing"
+ type: "object"
+ properties:
+ url:
+ title: "Public Endpoint"
+ description: "Public Endpoint of the Qdrant cluser"
+ order: 0
+ type: "string"
+ auth_method:
+ title: "Authentication Method"
+ description: "Method to authenticate with the Qdrant Instance"
+ default: "api_key_auth"
+ type: "object"
+ order: 1
+ oneOf:
+ - title: "ApiKeyAuth"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "api_key_auth"
+ const: "api_key_auth"
+ enum:
+ - "api_key_auth"
+ type: "string"
+ api_key:
+ title: "API Key"
+ description: "API Key for the Qdrant instance"
+ airbyte_secret: true
+ type: "string"
+ required:
+ - "api_key"
+ - title: "NoAuth"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "no_auth"
+ const: "no_auth"
+ enum:
+ - "no_auth"
+ type: "string"
+ prefer_grpc:
+ title: "Prefer gRPC"
+ description: "Whether to prefer gRPC over HTTP. Set to true for Qdrant\
+ \ cloud clusters"
+ default: true
+ type: "boolean"
+ collection:
+ title: "Collection Name"
+ description: "The collection to load data into"
+ order: 2
+ type: "string"
+ distance_metric:
+ title: "Distance Metric"
+ description: "The Distance metric used to measure similarities among\
+ \ vectors. This field is only used if the collection defined in the\
+ \ does not exist yet and is created automatically by the connector."
+ default: "cos"
+ enum:
+ - "dot"
+ - "cos"
+ - "euc"
+ type: "string"
+ text_field:
+ title: "Text Field"
+ description: "The field in the payload that contains the embedded text"
+ default: "text"
+ type: "string"
+ required:
+ - "url"
+ - "collection"
+ group: "Indexing"
+ description: "Indexing configuration"
+ required:
+ - "embedding"
+ - "processing"
+ - "indexing"
groups:
- id: "processing"
title: "Processing"
@@ -64998,2088 +66363,3102 @@ components:
title: "Indexing"
- id: "advanced"
title: "Advanced"
- destination-qdrant-update:
- title: "Destination Config"
- description: "The configuration model for the Vector DB based destinations.\
- \ This model is used to generate the UI for the destination configuration,\n\
- as well as to provide type safety for the configuration passed to the destination.\n\
- \nThe configuration model is composed of four parts:\n* Processing configuration\n\
- * Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\
- \nProcessing, embedding and advanced configuration are provided by this base\
- \ class, while the indexing configuration is provided by the destination connector\
- \ in the sub class."
+ destination-snowflake:
+ title: "Snowflake Destination Spec"
type: "object"
+ required:
+ - "host"
+ - "role"
+ - "warehouse"
+ - "database"
+ - "schema"
+ - "username"
+ - "destinationType"
properties:
- embedding:
- title: "Embedding"
- description: "Embedding configuration"
- group: "embedding"
+ host:
+ description: "Enter your Snowflake account's locator (in the format ...snowflakecomputing.com)"
+ examples:
+ - "accountname.us-east-2.aws.snowflakecomputing.com"
+ - "accountname.snowflakecomputing.com"
+ type: "string"
+ title: "Host"
+ pattern: "^(http(s)?:\\/\\/)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\\
+ .snowflakecomputing\\.com)$"
+ pattern_descriptor: "{account_name}.snowflakecomputing.com or {accountname}.{aws_location}.aws.snowflakecomputing.com"
+ order: 0
+ role:
+ description: "Enter the role that you want to use to access Snowflake"
+ examples:
+ - "AIRBYTE_ROLE"
+ type: "string"
+ title: "Role"
+ order: 1
+ warehouse:
+ description: "Enter the name of the warehouse that you want to sync data into"
+ examples:
+ - "AIRBYTE_WAREHOUSE"
+ type: "string"
+ title: "Warehouse"
+ order: 2
+ database:
+ description: "Enter the name of the database you want to sync data into"
+ examples:
+ - "AIRBYTE_DATABASE"
+ type: "string"
+ title: "Database"
+ order: 3
+ schema:
+ description: "Enter the name of the default schema"
+ examples:
+ - "AIRBYTE_SCHEMA"
+ type: "string"
+ title: "Default Schema"
+ order: 4
+ username:
+ description: "Enter the name of the user you want to use to access the database"
+ examples:
+ - "AIRBYTE_USER"
+ type: "string"
+ title: "Username"
+ order: 5
+ credentials:
+ title: "Authorization Method"
+ description: ""
type: "object"
oneOf:
- - title: "OpenAI"
+ - title: "Key Pair Authentication"
type: "object"
+ order: 0
+ required:
+ - "private_key"
properties:
- mode:
- title: "Mode"
- default: "openai"
- const: "openai"
+ auth_type:
+ type: "string"
+ const: "Key Pair Authentication"
enum:
- - "openai"
+ - "Key Pair Authentication"
+ default: "Key Pair Authentication"
+ order: 0
+ private_key:
type: "string"
- openai_key:
- title: "OpenAI API key"
+ title: "Private Key"
+ description: "RSA Private key to use for Snowflake connection. See\
+ \ the docs for more information on how to obtain this key."
+ multiline: true
airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ private_key_password:
type: "string"
+ title: "Passphrase"
+ description: "Passphrase for private key"
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ - title: "Username and Password"
+ type: "object"
required:
- - "openai_key"
- - "mode"
- description: "Use the OpenAI API to embed text. This option is using the\
- \ text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "Cohere"
+ - "password"
+ order: 1
+ properties:
+ auth_type:
+ type: "string"
+ const: "Username and Password"
+ enum:
+ - "Username and Password"
+ default: "Username and Password"
+ order: 0
+ password:
+ description: "Enter the password associated with the username."
+ type: "string"
+ airbyte_secret: true
+ title: "Password"
+ order: 1
+ x-speakeasy-param-sensitive: true
+ - title: "OAuth2.0"
type: "object"
+ order: 2
+ required:
+ - "access_token"
+ - "refresh_token"
+ airbyte_hidden: true
properties:
- mode:
- title: "Mode"
- default: "cohere"
- const: "cohere"
+ auth_type:
+ type: "string"
+ const: "OAuth2.0"
+ enum:
+ - "OAuth2.0"
+ default: "OAuth2.0"
+ order: 0
+ client_id:
+ type: "string"
+ title: "Client ID"
+ description: "Enter your application's Client ID"
+ airbyte_secret: true
+ client_secret:
+ type: "string"
+ title: "Client Secret"
+ description: "Enter your application's Client secret"
+ airbyte_secret: true
+ access_token:
+ type: "string"
+ title: "Access Token"
+ description: "Enter you application's Access Token"
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ refresh_token:
+ type: "string"
+ title: "Refresh Token"
+ description: "Enter your application's Refresh Token"
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ order: 6
+ jdbc_url_params:
+ description: "Enter the additional properties to pass to the JDBC URL string\
+ \ when connecting to the database (formatted as key=value pairs separated\
+ \ by the symbol &). Example: key1=value1&key2=value2&key3=value3"
+ title: "JDBC URL Params"
+ type: "string"
+ order: 7
+ raw_data_schema:
+ type: "string"
+ description: "The schema to write raw tables into (default: airbyte_internal)"
+ title: "Raw Table Schema Name"
+ order: 10
+ disable_type_dedupe:
+ type: "boolean"
+ default: false
+ description: "Disable Writing Final Tables. WARNING! The data format in\
+ \ _airbyte_data is likely stable but there are no guarantees that other\
+ \ metadata columns will remain the same in future versions"
+ title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
+ \ table schema might change between versions)"
+ order: 11
+ enable_incremental_final_table_updates:
+ type: "boolean"
+ default: false
+ description: "When enabled your data will load into your final tables incrementally\
+ \ while your data is still being synced. When Disabled (the default),\
+ \ your data loads into your final tables once at the end of a sync. Note\
+ \ that this option only applies if you elect to create Final tables"
+ title: "Enable Loading Data Incrementally to Final Tables"
+ order: 12
+ retention_period_days:
+ type: "integer"
+ default: 1
+ description: "The number of days of Snowflake Time Travel to enable on the\
+ \ tables. See Snowflake's documentation for more information. Setting a nonzero\
+ \ value will incur increased storage costs in your Snowflake instance."
+ title: "Data Retention Period (days)"
+ order: 13
+ destinationType:
+ title: "snowflake"
+ const: "snowflake"
+ enum:
+ - "snowflake"
+ order: 0
+ type: "string"
+ destination-snowflake-update:
+ title: "Snowflake Destination Spec"
+ type: "object"
+ required:
+ - "host"
+ - "role"
+ - "warehouse"
+ - "database"
+ - "schema"
+ - "username"
+ properties:
+ host:
+ description: "Enter your Snowflake account's locator (in the format ...snowflakecomputing.com)"
+ examples:
+ - "accountname.us-east-2.aws.snowflakecomputing.com"
+ - "accountname.snowflakecomputing.com"
+ type: "string"
+ title: "Host"
+ pattern: "^(http(s)?:\\/\\/)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\\
+ .snowflakecomputing\\.com)$"
+ pattern_descriptor: "{account_name}.snowflakecomputing.com or {accountname}.{aws_location}.aws.snowflakecomputing.com"
+ order: 0
+ role:
+ description: "Enter the role that you want to use to access Snowflake"
+ examples:
+ - "AIRBYTE_ROLE"
+ type: "string"
+ title: "Role"
+ order: 1
+ warehouse:
+ description: "Enter the name of the warehouse that you want to sync data into"
+ examples:
+ - "AIRBYTE_WAREHOUSE"
+ type: "string"
+ title: "Warehouse"
+ order: 2
+ database:
+ description: "Enter the name of the database you want to sync data into"
+ examples:
+ - "AIRBYTE_DATABASE"
+ type: "string"
+ title: "Database"
+ order: 3
+ schema:
+ description: "Enter the name of the default schema"
+ examples:
+ - "AIRBYTE_SCHEMA"
+ type: "string"
+ title: "Default Schema"
+ order: 4
+ username:
+ description: "Enter the name of the user you want to use to access the database"
+ examples:
+ - "AIRBYTE_USER"
+ type: "string"
+ title: "Username"
+ order: 5
+ credentials:
+ title: "Authorization Method"
+ description: ""
+ type: "object"
+ oneOf:
+ - title: "Key Pair Authentication"
+ type: "object"
+ order: 0
+ required:
+ - "private_key"
+ properties:
+ auth_type:
+ type: "string"
+ const: "Key Pair Authentication"
+ enum:
+ - "Key Pair Authentication"
+ default: "Key Pair Authentication"
+ order: 0
+ private_key:
+ type: "string"
+ title: "Private Key"
+ description: "RSA Private key to use for Snowflake connection. See\
+ \ the docs for more information on how to obtain this key."
+ multiline: true
+ airbyte_secret: true
+ private_key_password:
+ type: "string"
+ title: "Passphrase"
+ description: "Passphrase for private key"
+ airbyte_secret: true
+ - title: "Username and Password"
+ type: "object"
+ required:
+ - "password"
+ order: 1
+ properties:
+ auth_type:
+ type: "string"
+ const: "Username and Password"
+ enum:
+ - "Username and Password"
+ default: "Username and Password"
+ order: 0
+ password:
+ description: "Enter the password associated with the username."
+ type: "string"
+ airbyte_secret: true
+ title: "Password"
+ order: 1
+ - title: "OAuth2.0"
+ type: "object"
+ order: 2
+ required:
+ - "access_token"
+ - "refresh_token"
+ airbyte_hidden: true
+ properties:
+ auth_type:
+ type: "string"
+ const: "OAuth2.0"
+ enum:
+ - "OAuth2.0"
+ default: "OAuth2.0"
+ order: 0
+ client_id:
+ type: "string"
+ title: "Client ID"
+ description: "Enter your application's Client ID"
+ airbyte_secret: true
+ client_secret:
+ type: "string"
+ title: "Client Secret"
+ description: "Enter your application's Client secret"
+ airbyte_secret: true
+ access_token:
+ type: "string"
+ title: "Access Token"
+ description: "Enter you application's Access Token"
+ airbyte_secret: true
+ refresh_token:
+ type: "string"
+ title: "Refresh Token"
+ description: "Enter your application's Refresh Token"
+ airbyte_secret: true
+ order: 6
+ jdbc_url_params:
+ description: "Enter the additional properties to pass to the JDBC URL string\
+ \ when connecting to the database (formatted as key=value pairs separated\
+ \ by the symbol &). Example: key1=value1&key2=value2&key3=value3"
+ title: "JDBC URL Params"
+ type: "string"
+ order: 7
+ raw_data_schema:
+ type: "string"
+ description: "The schema to write raw tables into (default: airbyte_internal)"
+ title: "Raw Table Schema Name"
+ order: 10
+ disable_type_dedupe:
+ type: "boolean"
+ default: false
+ description: "Disable Writing Final Tables. WARNING! The data format in\
+ \ _airbyte_data is likely stable but there are no guarantees that other\
+ \ metadata columns will remain the same in future versions"
+ title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
+ \ table schema might change between versions)"
+ order: 11
+ enable_incremental_final_table_updates:
+ type: "boolean"
+ default: false
+ description: "When enabled your data will load into your final tables incrementally\
+ \ while your data is still being synced. When Disabled (the default),\
+ \ your data loads into your final tables once at the end of a sync. Note\
+ \ that this option only applies if you elect to create Final tables"
+ title: "Enable Loading Data Incrementally to Final Tables"
+ order: 12
+ retention_period_days:
+ type: "integer"
+ default: 1
+ description: "The number of days of Snowflake Time Travel to enable on the\
+ \ tables. See Snowflake's documentation for more information. Setting a nonzero\
+ \ value will incur increased storage costs in your Snowflake instance."
+ title: "Data Retention Period (days)"
+ order: 13
+ destination-databricks:
+ title: "Databricks Lakehouse Destination Spec"
+ type: "object"
+ required:
+ - "accept_terms"
+ - "databricks_server_hostname"
+ - "databricks_http_path"
+ - "databricks_personal_access_token"
+ - "data_source"
+ - "destinationType"
+ properties:
+ accept_terms:
+ title: "Agree to the Databricks JDBC Driver Terms & Conditions"
+ type: "boolean"
+ description: "You must agree to the Databricks JDBC Driver Terms & Conditions to use this connector."
+ default: false
+ order: 1
+ databricks_server_hostname:
+ title: "Server Hostname"
+ type: "string"
+ description: "Databricks Cluster Server Hostname."
+ examples:
+ - "abc-12345678-wxyz.cloud.databricks.com"
+ order: 2
+ databricks_http_path:
+ title: "HTTP Path"
+ type: "string"
+ description: "Databricks Cluster HTTP Path."
+ examples:
+ - "sql/protocolvx/o/1234567489/0000-1111111-abcd90"
+ order: 3
+ databricks_port:
+ title: "Port"
+ type: "string"
+ description: "Databricks Cluster Port."
+ default: "443"
+ examples:
+ - "443"
+ order: 4
+ databricks_personal_access_token:
+ title: "Access Token"
+ type: "string"
+ description: "Databricks Personal Access Token for making authenticated\
+ \ requests."
+ examples:
+ - "dapi0123456789abcdefghij0123456789AB"
+ airbyte_secret: true
+ order: 5
+ x-speakeasy-param-sensitive: true
+ database:
+ title: "Databricks catalog"
+ description: "The name of the catalog. If not specified otherwise, the \"\
+ hive_metastore\" will be used."
+ type: "string"
+ order: 6
+ schema:
+ title: "Default Schema"
+ description: "The default schema tables are written. If not specified otherwise,\
+ \ the \"default\" will be used."
+ type: "string"
+ examples:
+ - "default"
+ default: "default"
+ order: 7
+ enable_schema_evolution:
+ title: "Support schema evolution for all streams."
+ type: "boolean"
+ description: "Support schema evolution for all streams. If \"false\", the\
+ \ connector might fail when a stream's schema changes."
+ default: false
+ order: 8
+ data_source:
+ title: "Data Source"
+ type: "object"
+ description: "Storage on which the delta lake is built."
+ default: "MANAGED_TABLES_STORAGE"
+ order: 9
+ oneOf:
+ - title: "[Recommended] Managed tables"
+ required:
+ - "data_source_type"
+ properties:
+ data_source_type:
+ type: "string"
+ const: "MANAGED_TABLES_STORAGE"
+ order: 0
+ enum:
+ - "MANAGED_TABLES_STORAGE"
+ - title: "Amazon S3"
+ required:
+ - "data_source_type"
+ - "s3_bucket_name"
+ - "s3_bucket_path"
+ - "s3_bucket_region"
+ - "s3_access_key_id"
+ - "s3_secret_access_key"
+ properties:
+ data_source_type:
+ type: "string"
+ const: "S3_STORAGE"
+ order: 1
enum:
- - "cohere"
+ - "S3_STORAGE"
+ s3_bucket_name:
+ title: "S3 Bucket Name"
type: "string"
- cohere_key:
- title: "Cohere API key"
- airbyte_secret: true
+ description: "The name of the S3 bucket to use for intermittent staging\
+ \ of the data."
+ examples:
+ - "airbyte.staging"
+ order: 2
+ s3_bucket_path:
+ title: "S3 Bucket Path"
type: "string"
- required:
- - "cohere_key"
- - "mode"
- description: "Use the Cohere API to embed text."
- - title: "Fake"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "fake"
- const: "fake"
- enum:
- - "fake"
+ description: "The directory under the S3 bucket where data will be\
+ \ written."
+ examples:
+ - "data_sync/test"
+ order: 3
+ s3_bucket_region:
+ title: "S3 Bucket Region"
type: "string"
- description: "Use a fake embedding made out of random vectors with 1536\
- \ embedding dimensions. This is useful for testing the data pipeline\
- \ without incurring any costs."
- required:
- - "mode"
- - title: "Azure OpenAI"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "azure_openai"
- const: "azure_openai"
+ default: ""
+ description: "The region of the S3 staging bucket to use if utilising\
+ \ a copy strategy."
enum:
- - "azure_openai"
+ - ""
+ - "us-east-1"
+ - "us-east-2"
+ - "us-west-1"
+ - "us-west-2"
+ - "af-south-1"
+ - "ap-east-1"
+ - "ap-south-1"
+ - "ap-northeast-1"
+ - "ap-northeast-2"
+ - "ap-northeast-3"
+ - "ap-southeast-1"
+ - "ap-southeast-2"
+ - "ca-central-1"
+ - "cn-north-1"
+ - "cn-northwest-1"
+ - "eu-central-1"
+ - "eu-north-1"
+ - "eu-south-1"
+ - "eu-west-1"
+ - "eu-west-2"
+ - "eu-west-3"
+ - "sa-east-1"
+ - "me-south-1"
+ - "us-gov-east-1"
+ - "us-gov-west-1"
+ order: 4
+ s3_access_key_id:
type: "string"
- openai_key:
- title: "Azure OpenAI API key"
- description: "The API key for your Azure OpenAI resource. You can\
- \ find this in the Azure portal under your Azure OpenAI resource"
+ description: "The Access Key Id granting allow one to access the above\
+ \ S3 staging bucket. Airbyte requires Read and Write permissions\
+ \ to the given bucket."
+ title: "S3 Access Key ID"
+ examples:
+ - "A012345678910EXAMPLE"
airbyte_secret: true
+ order: 5
+ x-speakeasy-param-sensitive: true
+ s3_secret_access_key:
+ title: "S3 Secret Access Key"
type: "string"
- api_base:
- title: "Resource base URL"
- description: "The base URL for your Azure OpenAI resource. You can\
- \ find this in the Azure portal under your Azure OpenAI resource"
+ description: "The corresponding secret to the above access key id."
examples:
- - "https://your-resource-name.openai.azure.com"
+ - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
+ airbyte_secret: true
+ order: 6
+ x-speakeasy-param-sensitive: true
+ file_name_pattern:
type: "string"
- deployment:
- title: "Deployment"
- description: "The deployment for your Azure OpenAI resource. You\
- \ can find this in the Azure portal under your Azure OpenAI resource"
+ description: "The pattern allows you to set the file-name format for\
+ \ the S3 staging file(s)"
+ title: "S3 Filename pattern"
examples:
- - "your-resource-name"
- type: "string"
+ - "{date}"
+ - "{date:yyyy_MM}"
+ - "{timestamp}"
+ - "{part_number}"
+ - "{sync_id}"
+ order: 7
+ - title: "Azure Blob Storage"
required:
- - "openai_key"
- - "api_base"
- - "deployment"
- - "mode"
- description: "Use the Azure-hosted OpenAI API to embed text. This option\
- \ is using the text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "OpenAI-compatible"
- type: "object"
+ - "data_source_type"
+ - "azure_blob_storage_account_name"
+ - "azure_blob_storage_container_name"
+ - "azure_blob_storage_sas_token"
properties:
- mode:
- title: "Mode"
- default: "openai_compatible"
- const: "openai_compatible"
- enum:
- - "openai_compatible"
+ data_source_type:
type: "string"
- api_key:
- title: "API key"
- default: ""
- airbyte_secret: true
+ const: "AZURE_BLOB_STORAGE"
+ order: 0
+ enum:
+ - "AZURE_BLOB_STORAGE"
+ azure_blob_storage_endpoint_domain_name:
+ title: "Endpoint Domain Name"
type: "string"
- base_url:
- title: "Base URL"
- description: "The base URL for your OpenAI-compatible service"
+ default: "blob.core.windows.net"
+ description: "This is Azure Blob Storage endpoint domain name. Leave\
+ \ default value (or leave it empty if run container from command\
+ \ line) to use Microsoft native from example."
examples:
- - "https://your-service-name.com"
+ - "blob.core.windows.net"
+ order: 1
+ azure_blob_storage_account_name:
+ title: "Azure Blob Storage Account Name"
type: "string"
- model_name:
- title: "Model name"
- description: "The name of the model to use for embedding"
- default: "text-embedding-ada-002"
+ description: "The account's name of the Azure Blob Storage."
examples:
- - "text-embedding-ada-002"
+ - "airbyte5storage"
+ order: 2
+ azure_blob_storage_container_name:
+ title: "Azure Blob Storage Container Name"
type: "string"
- dimensions:
- title: "Embedding dimensions"
- description: "The number of dimensions the embedding model is generating"
+ description: "The name of the Azure blob storage container."
examples:
- - 1536
- - 384
- type: "integer"
- required:
- - "base_url"
- - "dimensions"
- - "mode"
- description: "Use a service that's compatible with the OpenAI API to embed\
- \ text."
- processing:
- title: "ProcessingConfigModel"
- type: "object"
- properties:
- chunk_size:
- title: "Chunk size"
- description: "Size of chunks in tokens to store in vector store (make\
- \ sure it is not too big for the context if your LLM)"
- maximum: 8191
- minimum: 1
- type: "integer"
- chunk_overlap:
- title: "Chunk overlap"
- description: "Size of overlap between chunks in tokens to store in vector\
- \ store to better capture relevant context"
- default: 0
- type: "integer"
- text_fields:
- title: "Text fields to embed"
- description: "List of fields in the record that should be used to calculate\
- \ the embedding. The field list is applied to all streams in the same\
- \ way and non-existing fields are ignored. If none are defined, all\
- \ fields are considered text fields. When specifying text fields,\
- \ you can access nested fields in the record by using dot notation,\
- \ e.g. `user.name` will access the `name` field in the `user` object.\
- \ It's also possible to use wildcards to access all fields in an object,\
- \ e.g. `users.*.name` will access all `names` fields in all entries\
- \ of the `users` array."
- default: []
- always_show: true
- examples:
- - "text"
- - "user.name"
- - "users.*.name"
- type: "array"
- items:
- type: "string"
- metadata_fields:
- title: "Fields to store as metadata"
- description: "List of fields in the record that should be stored as\
- \ metadata. The field list is applied to all streams in the same way\
- \ and non-existing fields are ignored. If none are defined, all fields\
- \ are considered metadata fields. When specifying text fields, you\
- \ can access nested fields in the record by using dot notation, e.g.\
- \ `user.name` will access the `name` field in the `user` object. It's\
- \ also possible to use wildcards to access all fields in an object,\
- \ e.g. `users.*.name` will access all `names` fields in all entries\
- \ of the `users` array. When specifying nested paths, all matching\
- \ values are flattened into an array set to a field named by the path."
- default: []
- always_show: true
- examples:
- - "age"
- - "user"
- - "user.name"
- type: "array"
- items:
+ - "airbytetestcontainername"
+ order: 3
+ azure_blob_storage_sas_token:
+ title: "SAS Token"
type: "string"
- text_splitter:
- title: "Text splitter"
- description: "Split text fields into chunks based on the specified method."
- type: "object"
- oneOf:
- - title: "By Separator"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "separator"
- const: "separator"
- enum:
- - "separator"
- type: "string"
- separators:
- title: "Separators"
- description: "List of separator strings to split text fields by.\
- \ The separator itself needs to be wrapped in double quotes,\
- \ e.g. to split by the dot character, use \".\". To split by\
- \ a newline, use \"\\n\"."
- default:
- - "\"\\n\\n\""
- - "\"\\n\""
- - "\" \""
- - "\"\""
- type: "array"
- items:
- type: "string"
- keep_separator:
- title: "Keep separator"
- description: "Whether to keep the separator in the resulting chunks"
- default: false
- type: "boolean"
- description: "Split the text by the list of separators until the chunk\
- \ size is reached, using the earlier mentioned separators where\
- \ possible. This is useful for splitting text fields by paragraphs,\
- \ sentences, words, etc."
- required:
- - "mode"
- - title: "By Markdown header"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "markdown"
- const: "markdown"
- enum:
- - "markdown"
- type: "string"
- split_level:
- title: "Split level"
- description: "Level of markdown headers to split text fields by.\
- \ Headings down to the specified level will be used as split\
- \ points"
- default: 1
- minimum: 1
- maximum: 6
- type: "integer"
- description: "Split the text by Markdown headers down to the specified\
- \ header level. If the chunk size fits multiple sections, they will\
- \ be combined into a single chunk."
- required:
- - "mode"
- - title: "By Programming Language"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "code"
- const: "code"
- enum:
- - "code"
- type: "string"
- language:
- title: "Language"
- description: "Split code in suitable places based on the programming\
- \ language"
- enum:
- - "cpp"
- - "go"
- - "java"
- - "js"
- - "php"
- - "proto"
- - "python"
- - "rst"
- - "ruby"
- - "rust"
- - "scala"
- - "swift"
- - "markdown"
- - "latex"
- - "html"
- - "sol"
- type: "string"
- required:
- - "language"
- - "mode"
- description: "Split the text by suitable delimiters based on the programming\
- \ language. This is useful for splitting code into chunks."
- field_name_mappings:
- title: "Field name mappings"
- description: "List of fields to rename. Not applicable for nested fields,\
- \ but can be used to rename fields already flattened via dot notation."
- default: []
- type: "array"
- items:
- title: "FieldNameMappingConfigModel"
- type: "object"
- properties:
- from_field:
- title: "From field name"
- description: "The field name in the source"
- type: "string"
- to_field:
- title: "To field name"
- description: "The field name to use in the destination"
- type: "string"
- required:
- - "from_field"
- - "to_field"
- required:
- - "chunk_size"
- group: "processing"
- omit_raw_text:
- title: "Do not store raw text"
- description: "Do not store the text that gets embedded along with the vector\
- \ and the metadata in the destination. If set to true, only the vector\
- \ and the metadata will be stored - in this case raw text for LLM use\
- \ cases needs to be retrieved from another source."
- default: false
- group: "advanced"
+ airbyte_secret: true
+ description: "Shared access signature (SAS) token to grant limited\
+ \ access to objects in your storage account."
+ examples:
+ - "?sv=2016-05-31&ss=b&srt=sco&sp=rwdl&se=2018-06-27T10:05:50Z&st=2017-06-27T02:05:50Z&spr=https,http&sig=bgqQwoXwxzuD2GJfagRg7VOS8hzNr3QLT7rhS8OFRLQ%3D"
+ order: 4
+ x-speakeasy-param-sensitive: true
+ purge_staging_data:
+ title: "Purge Staging Files and Tables"
type: "boolean"
- indexing:
- title: "Indexing"
- type: "object"
- properties:
- url:
- title: "Public Endpoint"
- description: "Public Endpoint of the Qdrant cluser"
- order: 0
- type: "string"
- auth_method:
- title: "Authentication Method"
- description: "Method to authenticate with the Qdrant Instance"
- default: "api_key_auth"
- type: "object"
- order: 1
- oneOf:
- - title: "ApiKeyAuth"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "api_key_auth"
- const: "api_key_auth"
- enum:
- - "api_key_auth"
- type: "string"
- api_key:
- title: "API Key"
- description: "API Key for the Qdrant instance"
- airbyte_secret: true
- type: "string"
- required:
- - "api_key"
- - title: "NoAuth"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "no_auth"
- const: "no_auth"
- enum:
- - "no_auth"
- type: "string"
- prefer_grpc:
- title: "Prefer gRPC"
- description: "Whether to prefer gRPC over HTTP. Set to true for Qdrant\
- \ cloud clusters"
- default: true
- type: "boolean"
- collection:
- title: "Collection Name"
- description: "The collection to load data into"
- order: 2
- type: "string"
- distance_metric:
- title: "Distance Metric"
- description: "The Distance metric used to measure similarities among\
- \ vectors. This field is only used if the collection defined in the\
- \ does not exist yet and is created automatically by the connector."
- default: "cos"
- enum:
- - "dot"
- - "cos"
- - "euc"
- type: "string"
- text_field:
- title: "Text Field"
- description: "The field in the payload that contains the embedded text"
- default: "text"
- type: "string"
- required:
- - "url"
- - "collection"
- group: "Indexing"
- description: "Indexing configuration"
- required:
- - "embedding"
- - "processing"
- - "indexing"
- groups:
- - id: "processing"
- title: "Processing"
- - id: "embedding"
- title: "Embedding"
- - id: "indexing"
- title: "Indexing"
- - id: "advanced"
- title: "Advanced"
- destination-snowflake:
- title: "Snowflake Destination Spec"
+ description: "Default to 'true'. Switch it to 'false' for debugging purpose."
+ default: true
+ order: 10
+ destinationType:
+ title: "databricks"
+ const: "databricks"
+ enum:
+ - "databricks"
+ order: 0
+ type: "string"
+ destination-databricks-update:
+ title: "Databricks Lakehouse Destination Spec"
type: "object"
required:
- - "host"
- - "role"
- - "warehouse"
- - "database"
- - "schema"
- - "username"
- - "destinationType"
+ - "accept_terms"
+ - "databricks_server_hostname"
+ - "databricks_http_path"
+ - "databricks_personal_access_token"
+ - "data_source"
properties:
- host:
- description: "Enter your Snowflake account's locator (in the format ...snowflakecomputing.com)"
- examples:
- - "accountname.us-east-2.aws.snowflakecomputing.com"
- - "accountname.snowflakecomputing.com"
+ accept_terms:
+ title: "Agree to the Databricks JDBC Driver Terms & Conditions"
+ type: "boolean"
+ description: "You must agree to the Databricks JDBC Driver Terms & Conditions to use this connector."
+ default: false
+ order: 1
+ databricks_server_hostname:
+ title: "Server Hostname"
type: "string"
- title: "Host"
- pattern: "^(http(s)?:\\/\\/)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\\
- .snowflakecomputing\\.com)$"
- pattern_descriptor: "{account_name}.snowflakecomputing.com or {accountname}.{aws_location}.aws.snowflakecomputing.com"
- order: 0
- role:
- description: "Enter the role that you want to use to access Snowflake"
+ description: "Databricks Cluster Server Hostname."
examples:
- - "AIRBYTE_ROLE"
+ - "abc-12345678-wxyz.cloud.databricks.com"
+ order: 2
+ databricks_http_path:
+ title: "HTTP Path"
type: "string"
- title: "Role"
- order: 1
- warehouse:
- description: "Enter the name of the warehouse that you want to sync data into"
+ description: "Databricks Cluster HTTP Path."
examples:
- - "AIRBYTE_WAREHOUSE"
+ - "sql/protocolvx/o/1234567489/0000-1111111-abcd90"
+ order: 3
+ databricks_port:
+ title: "Port"
type: "string"
- title: "Warehouse"
- order: 2
- database:
- description: "Enter the name of the database you want to sync data into"
+ description: "Databricks Cluster Port."
+ default: "443"
examples:
- - "AIRBYTE_DATABASE"
+ - "443"
+ order: 4
+ databricks_personal_access_token:
+ title: "Access Token"
type: "string"
- title: "Database"
- order: 3
- schema:
- description: "Enter the name of the default schema"
+ description: "Databricks Personal Access Token for making authenticated\
+ \ requests."
examples:
- - "AIRBYTE_SCHEMA"
+ - "dapi0123456789abcdefghij0123456789AB"
+ airbyte_secret: true
+ order: 5
+ database:
+ title: "Databricks catalog"
+ description: "The name of the catalog. If not specified otherwise, the \"\
+ hive_metastore\" will be used."
type: "string"
+ order: 6
+ schema:
title: "Default Schema"
- order: 4
- username:
- description: "Enter the name of the user you want to use to access the database"
- examples:
- - "AIRBYTE_USER"
+ description: "The default schema tables are written. If not specified otherwise,\
+ \ the \"default\" will be used."
type: "string"
- title: "Username"
- order: 5
- credentials:
- title: "Authorization Method"
- description: ""
+ examples:
+ - "default"
+ default: "default"
+ order: 7
+ enable_schema_evolution:
+ title: "Support schema evolution for all streams."
+ type: "boolean"
+ description: "Support schema evolution for all streams. If \"false\", the\
+ \ connector might fail when a stream's schema changes."
+ default: false
+ order: 8
+ data_source:
+ title: "Data Source"
type: "object"
+ description: "Storage on which the delta lake is built."
+ default: "MANAGED_TABLES_STORAGE"
+ order: 9
oneOf:
- - title: "Key Pair Authentication"
- type: "object"
- order: 0
+ - title: "[Recommended] Managed tables"
required:
- - "private_key"
+ - "data_source_type"
properties:
- auth_type:
+ data_source_type:
type: "string"
- const: "Key Pair Authentication"
- enum:
- - "Key Pair Authentication"
- default: "Key Pair Authentication"
+ const: "MANAGED_TABLES_STORAGE"
order: 0
- private_key:
- type: "string"
- title: "Private Key"
- description: "RSA Private key to use for Snowflake connection. See\
- \ the docs for more information on how to obtain this key."
- multiline: true
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- private_key_password:
- type: "string"
- title: "Passphrase"
- description: "Passphrase for private key"
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- - title: "Username and Password"
- type: "object"
+ enum:
+ - "MANAGED_TABLES_STORAGE"
+ - title: "Amazon S3"
required:
- - "password"
- order: 1
+ - "data_source_type"
+ - "s3_bucket_name"
+ - "s3_bucket_path"
+ - "s3_bucket_region"
+ - "s3_access_key_id"
+ - "s3_secret_access_key"
properties:
- auth_type:
+ data_source_type:
type: "string"
- const: "Username and Password"
+ const: "S3_STORAGE"
+ order: 1
enum:
- - "Username and Password"
- default: "Username and Password"
- order: 0
- password:
- description: "Enter the password associated with the username."
+ - "S3_STORAGE"
+ s3_bucket_name:
+ title: "S3 Bucket Name"
+ type: "string"
+ description: "The name of the S3 bucket to use for intermittent staging\
+ \ of the data."
+ examples:
+ - "airbyte.staging"
+ order: 2
+ s3_bucket_path:
+ title: "S3 Bucket Path"
+ type: "string"
+ description: "The directory under the S3 bucket where data will be\
+ \ written."
+ examples:
+ - "data_sync/test"
+ order: 3
+ s3_bucket_region:
+ title: "S3 Bucket Region"
+ type: "string"
+ default: ""
+ description: "The region of the S3 staging bucket to use if utilising\
+ \ a copy strategy."
+ enum:
+ - ""
+ - "us-east-1"
+ - "us-east-2"
+ - "us-west-1"
+ - "us-west-2"
+ - "af-south-1"
+ - "ap-east-1"
+ - "ap-south-1"
+ - "ap-northeast-1"
+ - "ap-northeast-2"
+ - "ap-northeast-3"
+ - "ap-southeast-1"
+ - "ap-southeast-2"
+ - "ca-central-1"
+ - "cn-north-1"
+ - "cn-northwest-1"
+ - "eu-central-1"
+ - "eu-north-1"
+ - "eu-south-1"
+ - "eu-west-1"
+ - "eu-west-2"
+ - "eu-west-3"
+ - "sa-east-1"
+ - "me-south-1"
+ - "us-gov-east-1"
+ - "us-gov-west-1"
+ order: 4
+ s3_access_key_id:
+ type: "string"
+ description: "The Access Key Id granting allow one to access the above\
+ \ S3 staging bucket. Airbyte requires Read and Write permissions\
+ \ to the given bucket."
+ title: "S3 Access Key ID"
+ examples:
+ - "A012345678910EXAMPLE"
+ airbyte_secret: true
+ order: 5
+ s3_secret_access_key:
+ title: "S3 Secret Access Key"
type: "string"
+ description: "The corresponding secret to the above access key id."
+ examples:
+ - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
airbyte_secret: true
- title: "Password"
- order: 1
- x-speakeasy-param-sensitive: true
- - title: "OAuth2.0"
- type: "object"
- order: 2
+ order: 6
+ file_name_pattern:
+ type: "string"
+ description: "The pattern allows you to set the file-name format for\
+ \ the S3 staging file(s)"
+ title: "S3 Filename pattern"
+ examples:
+ - "{date}"
+ - "{date:yyyy_MM}"
+ - "{timestamp}"
+ - "{part_number}"
+ - "{sync_id}"
+ order: 7
+ - title: "Azure Blob Storage"
required:
- - "access_token"
- - "refresh_token"
- airbyte_hidden: true
+ - "data_source_type"
+ - "azure_blob_storage_account_name"
+ - "azure_blob_storage_container_name"
+ - "azure_blob_storage_sas_token"
properties:
- auth_type:
+ data_source_type:
type: "string"
- const: "OAuth2.0"
- enum:
- - "OAuth2.0"
- default: "OAuth2.0"
+ const: "AZURE_BLOB_STORAGE"
order: 0
- client_id:
+ enum:
+ - "AZURE_BLOB_STORAGE"
+ azure_blob_storage_endpoint_domain_name:
+ title: "Endpoint Domain Name"
type: "string"
- title: "Client ID"
- description: "Enter your application's Client ID"
- airbyte_secret: true
- client_secret:
+ default: "blob.core.windows.net"
+ description: "This is Azure Blob Storage endpoint domain name. Leave\
+ \ default value (or leave it empty if run container from command\
+ \ line) to use Microsoft native from example."
+ examples:
+ - "blob.core.windows.net"
+ order: 1
+ azure_blob_storage_account_name:
+ title: "Azure Blob Storage Account Name"
type: "string"
- title: "Client Secret"
- description: "Enter your application's Client secret"
- airbyte_secret: true
- access_token:
+ description: "The account's name of the Azure Blob Storage."
+ examples:
+ - "airbyte5storage"
+ order: 2
+ azure_blob_storage_container_name:
+ title: "Azure Blob Storage Container Name"
type: "string"
- title: "Access Token"
- description: "Enter you application's Access Token"
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- refresh_token:
+ description: "The name of the Azure blob storage container."
+ examples:
+ - "airbytetestcontainername"
+ order: 3
+ azure_blob_storage_sas_token:
+ title: "SAS Token"
type: "string"
- title: "Refresh Token"
- description: "Enter your application's Refresh Token"
airbyte_secret: true
- x-speakeasy-param-sensitive: true
- order: 6
- jdbc_url_params:
- description: "Enter the additional properties to pass to the JDBC URL string\
- \ when connecting to the database (formatted as key=value pairs separated\
- \ by the symbol &). Example: key1=value1&key2=value2&key3=value3"
- title: "JDBC URL Params"
- type: "string"
- order: 7
- raw_data_schema:
- type: "string"
- description: "The schema to write raw tables into (default: airbyte_internal)"
- title: "Raw Table Schema Name"
- order: 10
- disable_type_dedupe:
- type: "boolean"
- default: false
- description: "Disable Writing Final Tables. WARNING! The data format in\
- \ _airbyte_data is likely stable but there are no guarantees that other\
- \ metadata columns will remain the same in future versions"
- title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
- \ table schema might change between versions)"
- order: 11
- enable_incremental_final_table_updates:
+ description: "Shared access signature (SAS) token to grant limited\
+ \ access to objects in your storage account."
+ examples:
+ - "?sv=2016-05-31&ss=b&srt=sco&sp=rwdl&se=2018-06-27T10:05:50Z&st=2017-06-27T02:05:50Z&spr=https,http&sig=bgqQwoXwxzuD2GJfagRg7VOS8hzNr3QLT7rhS8OFRLQ%3D"
+ order: 4
+ purge_staging_data:
+ title: "Purge Staging Files and Tables"
type: "boolean"
- default: false
- description: "When enabled your data will load into your final tables incrementally\
- \ while your data is still being synced. When Disabled (the default),\
- \ your data loads into your final tables once at the end of a sync. Note\
- \ that this option only applies if you elect to create Final tables"
- title: "Enable Loading Data Incrementally to Final Tables"
- order: 12
- retention_period_days:
- type: "integer"
- default: 1
- description: "The number of days of Snowflake Time Travel to enable on the\
- \ tables. See Snowflake's documentation for more information. Setting a nonzero\
- \ value will incur increased storage costs in your Snowflake instance."
- title: "Data Retention Period (days)"
- order: 13
- destinationType:
- title: "snowflake"
- const: "snowflake"
- enum:
- - "snowflake"
- order: 0
- type: "string"
- destination-snowflake-update:
- title: "Snowflake Destination Spec"
+ description: "Default to 'true'. Switch it to 'false' for debugging purpose."
+ default: true
+ order: 10
+ destination-oracle:
+ title: "Oracle Destination Spec"
type: "object"
required:
- "host"
- - "role"
- - "warehouse"
- - "database"
- - "schema"
+ - "port"
- "username"
+ - "sid"
+ - "destinationType"
properties:
host:
- description: "Enter your Snowflake account's locator (in the format ...snowflakecomputing.com)"
- examples:
- - "accountname.us-east-2.aws.snowflakecomputing.com"
- - "accountname.snowflakecomputing.com"
- type: "string"
title: "Host"
- pattern: "^(http(s)?:\\/\\/)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\\
- .snowflakecomputing\\.com)$"
- pattern_descriptor: "{account_name}.snowflakecomputing.com or {accountname}.{aws_location}.aws.snowflakecomputing.com"
+ description: "The hostname of the database."
+ type: "string"
order: 0
- role:
- description: "Enter the role that you want to use to access Snowflake"
+ port:
+ title: "Port"
+ description: "The port of the database."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 1521
examples:
- - "AIRBYTE_ROLE"
- type: "string"
- title: "Role"
+ - "1521"
order: 1
- warehouse:
- description: "Enter the name of the warehouse that you want to sync data into"
- examples:
- - "AIRBYTE_WAREHOUSE"
+ sid:
+ title: "SID"
+ description: "The System Identifier uniquely distinguishes the instance\
+ \ from any other instance on the same computer."
type: "string"
- title: "Warehouse"
order: 2
- database:
- description: "Enter the name of the database you want to sync data into"
- examples:
- - "AIRBYTE_DATABASE"
+ username:
+ title: "User"
+ description: "The username to access the database. This user must have CREATE\
+ \ USER privileges in the database."
type: "string"
- title: "Database"
order: 3
- schema:
- description: "Enter the name of the default schema"
- examples:
- - "AIRBYTE_SCHEMA"
+ password:
+ title: "Password"
+ description: "The password associated with the username."
type: "string"
- title: "Default Schema"
+ airbyte_secret: true
order: 4
- username:
- description: "Enter the name of the user you want to use to access the database"
- examples:
- - "AIRBYTE_USER"
+ x-speakeasy-param-sensitive: true
+ jdbc_url_params:
+ description: "Additional properties to pass to the JDBC URL string when\
+ \ connecting to the database formatted as 'key=value' pairs separated\
+ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+ title: "JDBC URL Params"
type: "string"
- title: "Username"
order: 5
- credentials:
- title: "Authorization Method"
- description: ""
+ schema:
+ title: "Default Schema"
+ description: "The default schema is used as the target schema for all statements\
+ \ issued from the connection that do not explicitly specify a schema name.\
+ \ The usual value for this field is \"airbyte\". In Oracle, schemas and\
+ \ users are the same thing, so the \"user\" parameter is used as the login\
+ \ credentials and this is used for the default Airbyte message schema."
+ type: "string"
+ examples:
+ - "airbyte"
+ default: "airbyte"
+ order: 6
+ raw_data_schema:
+ type: "string"
+ description: "The schema to write raw tables into (default: airbyte_internal)"
+ title: "Raw Table Schema Name"
+ order: 7
+ tunnel_method:
type: "object"
+ title: "SSH Tunnel Method"
+ description: "Whether to initiate an SSH tunnel before connecting to the\
+ \ database, and if so, which kind of authentication to use."
oneOf:
- - title: "Key Pair Authentication"
- type: "object"
- order: 0
+ - title: "No Tunnel"
required:
- - "private_key"
+ - "tunnel_method"
properties:
- auth_type:
+ tunnel_method:
+ description: "No ssh tunnel needed to connect to database"
type: "string"
- const: "Key Pair Authentication"
- enum:
- - "Key Pair Authentication"
- default: "Key Pair Authentication"
+ const: "NO_TUNNEL"
order: 0
- private_key:
- type: "string"
- title: "Private Key"
- description: "RSA Private key to use for Snowflake connection. See\
- \ the docs for more information on how to obtain this key."
- multiline: true
- airbyte_secret: true
- private_key_password:
- type: "string"
- title: "Passphrase"
- description: "Passphrase for private key"
- airbyte_secret: true
- - title: "Username and Password"
- type: "object"
+ enum:
+ - "NO_TUNNEL"
+ - title: "SSH Key Authentication"
required:
- - "password"
- order: 1
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "ssh_key"
properties:
- auth_type:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and ssh key"
type: "string"
- const: "Username and Password"
- enum:
- - "Username and Password"
- default: "Username and Password"
+ const: "SSH_KEY_AUTH"
order: 0
- password:
- description: "Enter the password associated with the username."
+ enum:
+ - "SSH_KEY_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
+ type: "string"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host."
+ type: "string"
+ order: 3
+ ssh_key:
+ title: "SSH Private Key"
+ description: "OS-level user account ssh key credentials in RSA PEM\
+ \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
type: "string"
airbyte_secret: true
- title: "Password"
- order: 1
- - title: "OAuth2.0"
- type: "object"
- order: 2
+ multiline: true
+ order: 4
+ x-speakeasy-param-sensitive: true
+ - title: "Password Authentication"
required:
- - "access_token"
- - "refresh_token"
- airbyte_hidden: true
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "tunnel_user_password"
properties:
- auth_type:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and password authentication"
type: "string"
- const: "OAuth2.0"
- enum:
- - "OAuth2.0"
- default: "OAuth2.0"
+ const: "SSH_PASSWORD_AUTH"
order: 0
- client_id:
- type: "string"
- title: "Client ID"
- description: "Enter your application's Client ID"
- airbyte_secret: true
- client_secret:
+ enum:
+ - "SSH_PASSWORD_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
type: "string"
- title: "Client Secret"
- description: "Enter your application's Client secret"
- airbyte_secret: true
- access_token:
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host"
type: "string"
- title: "Access Token"
- description: "Enter you application's Access Token"
- airbyte_secret: true
- refresh_token:
+ order: 3
+ tunnel_user_password:
+ title: "Password"
+ description: "OS-level password for logging into the jump server host"
type: "string"
- title: "Refresh Token"
- description: "Enter your application's Refresh Token"
airbyte_secret: true
- order: 6
- jdbc_url_params:
- description: "Enter the additional properties to pass to the JDBC URL string\
- \ when connecting to the database (formatted as key=value pairs separated\
- \ by the symbol &). Example: key1=value1&key2=value2&key3=value3"
- title: "JDBC URL Params"
- type: "string"
- order: 7
- raw_data_schema:
+ order: 4
+ x-speakeasy-param-sensitive: true
+ destinationType:
+ title: "oracle"
+ const: "oracle"
+ enum:
+ - "oracle"
+ order: 0
type: "string"
- description: "The schema to write raw tables into (default: airbyte_internal)"
- title: "Raw Table Schema Name"
- order: 10
- disable_type_dedupe:
- type: "boolean"
- default: false
- description: "Disable Writing Final Tables. WARNING! The data format in\
- \ _airbyte_data is likely stable but there are no guarantees that other\
- \ metadata columns will remain the same in future versions"
- title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
- \ table schema might change between versions)"
- order: 11
- enable_incremental_final_table_updates:
- type: "boolean"
- default: false
- description: "When enabled your data will load into your final tables incrementally\
- \ while your data is still being synced. When Disabled (the default),\
- \ your data loads into your final tables once at the end of a sync. Note\
- \ that this option only applies if you elect to create Final tables"
- title: "Enable Loading Data Incrementally to Final Tables"
- order: 12
- retention_period_days:
- type: "integer"
- default: 1
- description: "The number of days of Snowflake Time Travel to enable on the\
- \ tables. See Snowflake's documentation for more information. Setting a nonzero\
- \ value will incur increased storage costs in your Snowflake instance."
- title: "Data Retention Period (days)"
- order: 13
- destination-databricks:
- title: "Databricks Lakehouse Destination Spec"
+ destination-oracle-update:
+ title: "Oracle Destination Spec"
type: "object"
required:
- - "accept_terms"
- - "databricks_server_hostname"
- - "databricks_http_path"
- - "databricks_personal_access_token"
- - "data_source"
- - "destinationType"
+ - "host"
+ - "port"
+ - "username"
+ - "sid"
properties:
- accept_terms:
- title: "Agree to the Databricks JDBC Driver Terms & Conditions"
- type: "boolean"
- description: "You must agree to the Databricks JDBC Driver Terms & Conditions to use this connector."
- default: false
- order: 1
- databricks_server_hostname:
- title: "Server Hostname"
+ host:
+ title: "Host"
+ description: "The hostname of the database."
type: "string"
- description: "Databricks Cluster Server Hostname."
+ order: 0
+ port:
+ title: "Port"
+ description: "The port of the database."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 1521
examples:
- - "abc-12345678-wxyz.cloud.databricks.com"
+ - "1521"
+ order: 1
+ sid:
+ title: "SID"
+ description: "The System Identifier uniquely distinguishes the instance\
+ \ from any other instance on the same computer."
+ type: "string"
order: 2
- databricks_http_path:
- title: "HTTP Path"
+ username:
+ title: "User"
+ description: "The username to access the database. This user must have CREATE\
+ \ USER privileges in the database."
type: "string"
- description: "Databricks Cluster HTTP Path."
- examples:
- - "sql/protocolvx/o/1234567489/0000-1111111-abcd90"
order: 3
- databricks_port:
- title: "Port"
+ password:
+ title: "Password"
+ description: "The password associated with the username."
type: "string"
- description: "Databricks Cluster Port."
- default: "443"
- examples:
- - "443"
+ airbyte_secret: true
order: 4
- databricks_personal_access_token:
- title: "Access Token"
+ jdbc_url_params:
+ description: "Additional properties to pass to the JDBC URL string when\
+ \ connecting to the database formatted as 'key=value' pairs separated\
+ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+ title: "JDBC URL Params"
type: "string"
- description: "Databricks Personal Access Token for making authenticated\
- \ requests."
- examples:
- - "dapi0123456789abcdefghij0123456789AB"
- airbyte_secret: true
order: 5
- x-speakeasy-param-sensitive: true
- database:
- title: "Databricks catalog"
- description: "The name of the catalog. If not specified otherwise, the \"\
- hive_metastore\" will be used."
- type: "string"
- order: 6
schema:
title: "Default Schema"
- description: "The default schema tables are written. If not specified otherwise,\
- \ the \"default\" will be used."
+ description: "The default schema is used as the target schema for all statements\
+ \ issued from the connection that do not explicitly specify a schema name.\
+ \ The usual value for this field is \"airbyte\". In Oracle, schemas and\
+ \ users are the same thing, so the \"user\" parameter is used as the login\
+ \ credentials and this is used for the default Airbyte message schema."
type: "string"
examples:
- - "default"
- default: "default"
+ - "airbyte"
+ default: "airbyte"
+ order: 6
+ raw_data_schema:
+ type: "string"
+ description: "The schema to write raw tables into (default: airbyte_internal)"
+ title: "Raw Table Schema Name"
order: 7
- enable_schema_evolution:
- title: "Support schema evolution for all streams."
- type: "boolean"
- description: "Support schema evolution for all streams. If \"false\", the\
- \ connector might fail when a stream's schema changes."
- default: false
- order: 8
- data_source:
- title: "Data Source"
+ tunnel_method:
type: "object"
- description: "Storage on which the delta lake is built."
- default: "MANAGED_TABLES_STORAGE"
- order: 9
+ title: "SSH Tunnel Method"
+ description: "Whether to initiate an SSH tunnel before connecting to the\
+ \ database, and if so, which kind of authentication to use."
oneOf:
- - title: "[Recommended] Managed tables"
+ - title: "No Tunnel"
required:
- - "data_source_type"
+ - "tunnel_method"
properties:
- data_source_type:
+ tunnel_method:
+ description: "No ssh tunnel needed to connect to database"
type: "string"
- const: "MANAGED_TABLES_STORAGE"
+ const: "NO_TUNNEL"
order: 0
enum:
- - "MANAGED_TABLES_STORAGE"
- - title: "Amazon S3"
+ - "NO_TUNNEL"
+ - title: "SSH Key Authentication"
required:
- - "data_source_type"
- - "s3_bucket_name"
- - "s3_bucket_path"
- - "s3_bucket_region"
- - "s3_access_key_id"
- - "s3_secret_access_key"
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "ssh_key"
properties:
- data_source_type:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and ssh key"
type: "string"
- const: "S3_STORAGE"
- order: 1
+ const: "SSH_KEY_AUTH"
+ order: 0
enum:
- - "S3_STORAGE"
- s3_bucket_name:
- title: "S3 Bucket Name"
+ - "SSH_KEY_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
type: "string"
- description: "The name of the S3 bucket to use for intermittent staging\
- \ of the data."
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
examples:
- - "airbyte.staging"
+ - "22"
order: 2
- s3_bucket_path:
- title: "S3 Bucket Path"
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host."
type: "string"
- description: "The directory under the S3 bucket where data will be\
- \ written."
- examples:
- - "data_sync/test"
order: 3
- s3_bucket_region:
- title: "S3 Bucket Region"
+ ssh_key:
+ title: "SSH Private Key"
+ description: "OS-level user account ssh key credentials in RSA PEM\
+ \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
type: "string"
- default: ""
- description: "The region of the S3 staging bucket to use if utilising\
- \ a copy strategy."
- enum:
- - ""
- - "us-east-1"
- - "us-east-2"
- - "us-west-1"
- - "us-west-2"
- - "af-south-1"
- - "ap-east-1"
- - "ap-south-1"
- - "ap-northeast-1"
- - "ap-northeast-2"
- - "ap-northeast-3"
- - "ap-southeast-1"
- - "ap-southeast-2"
- - "ca-central-1"
- - "cn-north-1"
- - "cn-northwest-1"
- - "eu-central-1"
- - "eu-north-1"
- - "eu-south-1"
- - "eu-west-1"
- - "eu-west-2"
- - "eu-west-3"
- - "sa-east-1"
- - "me-south-1"
- - "us-gov-east-1"
- - "us-gov-west-1"
+ airbyte_secret: true
+ multiline: true
order: 4
- s3_access_key_id:
+ - title: "Password Authentication"
+ required:
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "tunnel_user_password"
+ properties:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and password authentication"
type: "string"
- description: "The Access Key Id granting allow one to access the above\
- \ S3 staging bucket. Airbyte requires Read and Write permissions\
- \ to the given bucket."
- title: "S3 Access Key ID"
+ const: "SSH_PASSWORD_AUTH"
+ order: 0
+ enum:
+ - "SSH_PASSWORD_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
+ type: "string"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
examples:
- - "A012345678910EXAMPLE"
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host"
+ type: "string"
+ order: 3
+ tunnel_user_password:
+ title: "Password"
+ description: "OS-level password for logging into the jump server host"
+ type: "string"
airbyte_secret: true
- order: 5
- x-speakeasy-param-sensitive: true
- s3_secret_access_key:
- title: "S3 Secret Access Key"
+ order: 4
+ destination-aws-datalake:
+ title: "AWS Datalake Destination Spec"
+ type: "object"
+ required:
+ - "credentials"
+ - "region"
+ - "bucket_name"
+ - "lakeformation_database_name"
+ - "destinationType"
+ properties:
+ aws_account_id:
+ type: "string"
+ title: "AWS Account Id"
+ description: "target aws account id"
+ examples:
+ - "111111111111"
+ order: 1
+ credentials:
+ title: "Authentication mode"
+ description: "Choose How to Authenticate to AWS."
+ type: "object"
+ oneOf:
+ - type: "object"
+ title: "IAM Role"
+ required:
+ - "role_arn"
+ - "credentials_title"
+ properties:
+ credentials_title:
type: "string"
- description: "The corresponding secret to the above access key id."
- examples:
- - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
+ title: "Credentials Title"
+ description: "Name of the credentials"
+ const: "IAM Role"
+ enum:
+ - "IAM Role"
+ default: "IAM Role"
+ order: 0
+ role_arn:
+ title: "Target Role Arn"
+ type: "string"
+ description: "Will assume this role to write data to s3"
+ airbyte_secret: false
+ - type: "object"
+ title: "IAM User"
+ required:
+ - "credentials_title"
+ - "aws_access_key_id"
+ - "aws_secret_access_key"
+ properties:
+ credentials_title:
+ type: "string"
+ title: "Credentials Title"
+ description: "Name of the credentials"
+ const: "IAM User"
+ enum:
+ - "IAM User"
+ default: "IAM User"
+ order: 0
+ aws_access_key_id:
+ title: "Access Key Id"
+ type: "string"
+ description: "AWS User Access Key Id"
airbyte_secret: true
- order: 6
x-speakeasy-param-sensitive: true
- file_name_pattern:
+ aws_secret_access_key:
+ title: "Secret Access Key"
type: "string"
- description: "The pattern allows you to set the file-name format for\
- \ the S3 staging file(s)"
- title: "S3 Filename pattern"
- examples:
- - "{date}"
- - "{date:yyyy_MM}"
- - "{timestamp}"
- - "{part_number}"
- - "{sync_id}"
- order: 7
- - title: "Azure Blob Storage"
+ description: "Secret Access Key"
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ order: 2
+ region:
+ title: "S3 Bucket Region"
+ type: "string"
+ default: ""
+ description: "The region of the S3 bucket. See here for all region codes."
+ enum:
+ - ""
+ - "af-south-1"
+ - "ap-east-1"
+ - "ap-northeast-1"
+ - "ap-northeast-2"
+ - "ap-northeast-3"
+ - "ap-south-1"
+ - "ap-south-2"
+ - "ap-southeast-1"
+ - "ap-southeast-2"
+ - "ap-southeast-3"
+ - "ap-southeast-4"
+ - "ca-central-1"
+ - "ca-west-1"
+ - "cn-north-1"
+ - "cn-northwest-1"
+ - "eu-central-1"
+ - "eu-central-2"
+ - "eu-north-1"
+ - "eu-south-1"
+ - "eu-south-2"
+ - "eu-west-1"
+ - "eu-west-2"
+ - "eu-west-3"
+ - "il-central-1"
+ - "me-central-1"
+ - "me-south-1"
+ - "sa-east-1"
+ - "us-east-1"
+ - "us-east-2"
+ - "us-gov-east-1"
+ - "us-gov-west-1"
+ - "us-west-1"
+ - "us-west-2"
+ order: 3
+ bucket_name:
+ title: "S3 Bucket Name"
+ type: "string"
+ description: "The name of the S3 bucket. Read more here."
+ order: 4
+ bucket_prefix:
+ title: "Target S3 Bucket Prefix"
+ type: "string"
+ description: "S3 prefix"
+ order: 5
+ lakeformation_database_name:
+ title: "Lake Formation Database Name"
+ type: "string"
+ description: "The default database this destination will use to create tables\
+ \ in per stream. Can be changed per connection by customizing the namespace."
+ order: 6
+ lakeformation_database_default_tag_key:
+ title: "Lake Formation Database Tag Key"
+ description: "Add a default tag key to databases created by this destination"
+ examples:
+ - "pii_level"
+ type: "string"
+ order: 7
+ x-speakeasy-param-sensitive: true
+ lakeformation_database_default_tag_values:
+ title: "Lake Formation Database Tag Values"
+ description: "Add default values for the `Tag Key` to databases created\
+ \ by this destination. Comma separate for multiple values."
+ examples:
+ - "private,public"
+ type: "string"
+ order: 8
+ lakeformation_governed_tables:
+ title: "Lake Formation Governed Tables"
+ description: "Whether to create tables as LF governed tables."
+ type: "boolean"
+ default: false
+ order: 9
+ format:
+ title: "Output Format *"
+ type: "object"
+ description: "Format of the data output."
+ oneOf:
+ - title: "JSON Lines: Newline-delimited JSON"
required:
- - "data_source_type"
- - "azure_blob_storage_account_name"
- - "azure_blob_storage_container_name"
- - "azure_blob_storage_sas_token"
+ - "format_type"
properties:
- data_source_type:
+ format_type:
+ title: "Format Type *"
type: "string"
- const: "AZURE_BLOB_STORAGE"
- order: 0
enum:
- - "AZURE_BLOB_STORAGE"
- azure_blob_storage_endpoint_domain_name:
- title: "Endpoint Domain Name"
- type: "string"
- default: "blob.core.windows.net"
- description: "This is Azure Blob Storage endpoint domain name. Leave\
- \ default value (or leave it empty if run container from command\
- \ line) to use Microsoft native from example."
- examples:
- - "blob.core.windows.net"
- order: 1
- azure_blob_storage_account_name:
- title: "Azure Blob Storage Account Name"
+ - "JSONL"
+ default: "JSONL"
+ compression_codec:
+ title: "Compression Codec (Optional)"
+ description: "The compression algorithm used to compress data."
type: "string"
- description: "The account's name of the Azure Blob Storage."
- examples:
- - "airbyte5storage"
- order: 2
- azure_blob_storage_container_name:
- title: "Azure Blob Storage Container Name"
+ enum:
+ - "UNCOMPRESSED"
+ - "GZIP"
+ default: "UNCOMPRESSED"
+ - title: "Parquet: Columnar Storage"
+ required:
+ - "format_type"
+ properties:
+ format_type:
+ title: "Format Type *"
type: "string"
- description: "The name of the Azure blob storage container."
- examples:
- - "airbytetestcontainername"
- order: 3
- azure_blob_storage_sas_token:
- title: "SAS Token"
+ enum:
+ - "Parquet"
+ default: "Parquet"
+ compression_codec:
+ title: "Compression Codec (Optional)"
+ description: "The compression algorithm used to compress data."
type: "string"
- airbyte_secret: true
- description: "Shared access signature (SAS) token to grant limited\
- \ access to objects in your storage account."
- examples:
- - "?sv=2016-05-31&ss=b&srt=sco&sp=rwdl&se=2018-06-27T10:05:50Z&st=2017-06-27T02:05:50Z&spr=https,http&sig=bgqQwoXwxzuD2GJfagRg7VOS8hzNr3QLT7rhS8OFRLQ%3D"
- order: 4
- x-speakeasy-param-sensitive: true
- purge_staging_data:
- title: "Purge Staging Files and Tables"
- type: "boolean"
- description: "Default to 'true'. Switch it to 'false' for debugging purpose."
- default: true
+ enum:
+ - "UNCOMPRESSED"
+ - "SNAPPY"
+ - "GZIP"
+ - "ZSTD"
+ default: "SNAPPY"
order: 10
+ partitioning:
+ title: "Choose how to partition data"
+ description: "Partition data by cursor fields when a cursor field is a date"
+ type: "string"
+ enum:
+ - "NO PARTITIONING"
+ - "DATE"
+ - "YEAR"
+ - "MONTH"
+ - "DAY"
+ - "YEAR/MONTH"
+ - "YEAR/MONTH/DAY"
+ default: "NO PARTITIONING"
+ order: 11
+ glue_catalog_float_as_decimal:
+ title: "Glue Catalog: Float as Decimal"
+ description: "Cast float/double as decimal(38,18). This can help achieve\
+ \ higher accuracy and represent numbers correctly as received from the\
+ \ source."
+ type: "boolean"
+ default: false
+ order: 12
destinationType:
- title: "databricks"
- const: "databricks"
+ title: "aws-datalake"
+ const: "aws-datalake"
enum:
- - "databricks"
+ - "aws-datalake"
order: 0
type: "string"
- destination-databricks-update:
- title: "Databricks Lakehouse Destination Spec"
+ destination-aws-datalake-update:
+ title: "AWS Datalake Destination Spec"
type: "object"
required:
- - "accept_terms"
- - "databricks_server_hostname"
- - "databricks_http_path"
- - "databricks_personal_access_token"
- - "data_source"
+ - "credentials"
+ - "region"
+ - "bucket_name"
+ - "lakeformation_database_name"
properties:
- accept_terms:
- title: "Agree to the Databricks JDBC Driver Terms & Conditions"
- type: "boolean"
- description: "You must agree to the Databricks JDBC Driver Terms & Conditions to use this connector."
- default: false
- order: 1
- databricks_server_hostname:
- title: "Server Hostname"
+ aws_account_id:
type: "string"
- description: "Databricks Cluster Server Hostname."
+ title: "AWS Account Id"
+ description: "target aws account id"
examples:
- - "abc-12345678-wxyz.cloud.databricks.com"
+ - "111111111111"
+ order: 1
+ credentials:
+ title: "Authentication mode"
+ description: "Choose How to Authenticate to AWS."
+ type: "object"
+ oneOf:
+ - type: "object"
+ title: "IAM Role"
+ required:
+ - "role_arn"
+ - "credentials_title"
+ properties:
+ credentials_title:
+ type: "string"
+ title: "Credentials Title"
+ description: "Name of the credentials"
+ const: "IAM Role"
+ enum:
+ - "IAM Role"
+ default: "IAM Role"
+ order: 0
+ role_arn:
+ title: "Target Role Arn"
+ type: "string"
+ description: "Will assume this role to write data to s3"
+ airbyte_secret: false
+ - type: "object"
+ title: "IAM User"
+ required:
+ - "credentials_title"
+ - "aws_access_key_id"
+ - "aws_secret_access_key"
+ properties:
+ credentials_title:
+ type: "string"
+ title: "Credentials Title"
+ description: "Name of the credentials"
+ const: "IAM User"
+ enum:
+ - "IAM User"
+ default: "IAM User"
+ order: 0
+ aws_access_key_id:
+ title: "Access Key Id"
+ type: "string"
+ description: "AWS User Access Key Id"
+ airbyte_secret: true
+ aws_secret_access_key:
+ title: "Secret Access Key"
+ type: "string"
+ description: "Secret Access Key"
+ airbyte_secret: true
order: 2
- databricks_http_path:
- title: "HTTP Path"
+ region:
+ title: "S3 Bucket Region"
type: "string"
- description: "Databricks Cluster HTTP Path."
- examples:
- - "sql/protocolvx/o/1234567489/0000-1111111-abcd90"
+ default: ""
+ description: "The region of the S3 bucket. See here for all region codes."
+ enum:
+ - ""
+ - "af-south-1"
+ - "ap-east-1"
+ - "ap-northeast-1"
+ - "ap-northeast-2"
+ - "ap-northeast-3"
+ - "ap-south-1"
+ - "ap-south-2"
+ - "ap-southeast-1"
+ - "ap-southeast-2"
+ - "ap-southeast-3"
+ - "ap-southeast-4"
+ - "ca-central-1"
+ - "ca-west-1"
+ - "cn-north-1"
+ - "cn-northwest-1"
+ - "eu-central-1"
+ - "eu-central-2"
+ - "eu-north-1"
+ - "eu-south-1"
+ - "eu-south-2"
+ - "eu-west-1"
+ - "eu-west-2"
+ - "eu-west-3"
+ - "il-central-1"
+ - "me-central-1"
+ - "me-south-1"
+ - "sa-east-1"
+ - "us-east-1"
+ - "us-east-2"
+ - "us-gov-east-1"
+ - "us-gov-west-1"
+ - "us-west-1"
+ - "us-west-2"
order: 3
- databricks_port:
- title: "Port"
+ bucket_name:
+ title: "S3 Bucket Name"
type: "string"
- description: "Databricks Cluster Port."
- default: "443"
- examples:
- - "443"
+ description: "The name of the S3 bucket. Read more here."
order: 4
- databricks_personal_access_token:
- title: "Access Token"
+ bucket_prefix:
+ title: "Target S3 Bucket Prefix"
type: "string"
- description: "Databricks Personal Access Token for making authenticated\
- \ requests."
- examples:
- - "dapi0123456789abcdefghij0123456789AB"
- airbyte_secret: true
+ description: "S3 prefix"
order: 5
- database:
- title: "Databricks catalog"
- description: "The name of the catalog. If not specified otherwise, the \"\
- hive_metastore\" will be used."
+ lakeformation_database_name:
+ title: "Lake Formation Database Name"
type: "string"
+ description: "The default database this destination will use to create tables\
+ \ in per stream. Can be changed per connection by customizing the namespace."
order: 6
- schema:
- title: "Default Schema"
- description: "The default schema tables are written. If not specified otherwise,\
- \ the \"default\" will be used."
- type: "string"
+ lakeformation_database_default_tag_key:
+ title: "Lake Formation Database Tag Key"
+ description: "Add a default tag key to databases created by this destination"
examples:
- - "default"
- default: "default"
+ - "pii_level"
+ type: "string"
order: 7
- enable_schema_evolution:
- title: "Support schema evolution for all streams."
+ lakeformation_database_default_tag_values:
+ title: "Lake Formation Database Tag Values"
+ description: "Add default values for the `Tag Key` to databases created\
+ \ by this destination. Comma separate for multiple values."
+ examples:
+ - "private,public"
+ type: "string"
+ order: 8
+ lakeformation_governed_tables:
+ title: "Lake Formation Governed Tables"
+ description: "Whether to create tables as LF governed tables."
type: "boolean"
- description: "Support schema evolution for all streams. If \"false\", the\
- \ connector might fail when a stream's schema changes."
default: false
- order: 8
- data_source:
- title: "Data Source"
- type: "object"
- description: "Storage on which the delta lake is built."
- default: "MANAGED_TABLES_STORAGE"
order: 9
+ format:
+ title: "Output Format *"
+ type: "object"
+ description: "Format of the data output."
oneOf:
- - title: "[Recommended] Managed tables"
+ - title: "JSON Lines: Newline-delimited JSON"
required:
- - "data_source_type"
+ - "format_type"
properties:
- data_source_type:
+ format_type:
+ title: "Format Type *"
type: "string"
- const: "MANAGED_TABLES_STORAGE"
- order: 0
enum:
- - "MANAGED_TABLES_STORAGE"
- - title: "Amazon S3"
+ - "JSONL"
+ default: "JSONL"
+ compression_codec:
+ title: "Compression Codec (Optional)"
+ description: "The compression algorithm used to compress data."
+ type: "string"
+ enum:
+ - "UNCOMPRESSED"
+ - "GZIP"
+ default: "UNCOMPRESSED"
+ - title: "Parquet: Columnar Storage"
required:
- - "data_source_type"
- - "s3_bucket_name"
- - "s3_bucket_path"
- - "s3_bucket_region"
- - "s3_access_key_id"
- - "s3_secret_access_key"
+ - "format_type"
properties:
- data_source_type:
+ format_type:
+ title: "Format Type *"
type: "string"
- const: "S3_STORAGE"
- order: 1
enum:
- - "S3_STORAGE"
- s3_bucket_name:
- title: "S3 Bucket Name"
+ - "Parquet"
+ default: "Parquet"
+ compression_codec:
+ title: "Compression Codec (Optional)"
+ description: "The compression algorithm used to compress data."
type: "string"
- description: "The name of the S3 bucket to use for intermittent staging\
- \ of the data."
- examples:
- - "airbyte.staging"
- order: 2
- s3_bucket_path:
- title: "S3 Bucket Path"
+ enum:
+ - "UNCOMPRESSED"
+ - "SNAPPY"
+ - "GZIP"
+ - "ZSTD"
+ default: "SNAPPY"
+ order: 10
+ partitioning:
+ title: "Choose how to partition data"
+ description: "Partition data by cursor fields when a cursor field is a date"
+ type: "string"
+ enum:
+ - "NO PARTITIONING"
+ - "DATE"
+ - "YEAR"
+ - "MONTH"
+ - "DAY"
+ - "YEAR/MONTH"
+ - "YEAR/MONTH/DAY"
+ default: "NO PARTITIONING"
+ order: 11
+ glue_catalog_float_as_decimal:
+ title: "Glue Catalog: Float as Decimal"
+ description: "Cast float/double as decimal(38,18). This can help achieve\
+ \ higher accuracy and represent numbers correctly as received from the\
+ \ source."
+ type: "boolean"
+ default: false
+ order: 12
+ destination-milvus:
+ title: "Destination Config"
+ description: "The configuration model for the Vector DB based destinations.\
+ \ This model is used to generate the UI for the destination configuration,\n\
+ as well as to provide type safety for the configuration passed to the destination.\n\
+ \nThe configuration model is composed of four parts:\n* Processing configuration\n\
+ * Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\
+ \nProcessing, embedding and advanced configuration are provided by this base\
+ \ class, while the indexing configuration is provided by the destination connector\
+ \ in the sub class."
+ type: "object"
+ properties:
+ embedding:
+ title: "Embedding"
+ description: "Embedding configuration"
+ group: "embedding"
+ type: "object"
+ oneOf:
+ - title: "OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "openai"
+ const: "openai"
+ enum:
+ - "openai"
type: "string"
- description: "The directory under the S3 bucket where data will be\
- \ written."
- examples:
- - "data_sync/test"
- order: 3
- s3_bucket_region:
- title: "S3 Bucket Region"
+ openai_key:
+ title: "OpenAI API key"
+ airbyte_secret: true
type: "string"
- default: ""
- description: "The region of the S3 staging bucket to use if utilising\
- \ a copy strategy."
+ x-speakeasy-param-sensitive: true
+ required:
+ - "openai_key"
+ - "mode"
+ description: "Use the OpenAI API to embed text. This option is using the\
+ \ text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "Cohere"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "cohere"
+ const: "cohere"
enum:
- - ""
- - "us-east-1"
- - "us-east-2"
- - "us-west-1"
- - "us-west-2"
- - "af-south-1"
- - "ap-east-1"
- - "ap-south-1"
- - "ap-northeast-1"
- - "ap-northeast-2"
- - "ap-northeast-3"
- - "ap-southeast-1"
- - "ap-southeast-2"
- - "ca-central-1"
- - "cn-north-1"
- - "cn-northwest-1"
- - "eu-central-1"
- - "eu-north-1"
- - "eu-south-1"
- - "eu-west-1"
- - "eu-west-2"
- - "eu-west-3"
- - "sa-east-1"
- - "me-south-1"
- - "us-gov-east-1"
- - "us-gov-west-1"
- order: 4
- s3_access_key_id:
+ - "cohere"
+ type: "string"
+ cohere_key:
+ title: "Cohere API key"
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ required:
+ - "cohere_key"
+ - "mode"
+ description: "Use the Cohere API to embed text."
+ - title: "Fake"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "fake"
+ const: "fake"
+ enum:
+ - "fake"
+ type: "string"
+ description: "Use a fake embedding made out of random vectors with 1536\
+ \ embedding dimensions. This is useful for testing the data pipeline\
+ \ without incurring any costs."
+ required:
+ - "mode"
+ - title: "Azure OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "azure_openai"
+ const: "azure_openai"
+ enum:
+ - "azure_openai"
type: "string"
- description: "The Access Key Id granting allow one to access the above\
- \ S3 staging bucket. Airbyte requires Read and Write permissions\
- \ to the given bucket."
- title: "S3 Access Key ID"
- examples:
- - "A012345678910EXAMPLE"
+ openai_key:
+ title: "Azure OpenAI API key"
+ description: "The API key for your Azure OpenAI resource. You can\
+ \ find this in the Azure portal under your Azure OpenAI resource"
airbyte_secret: true
- order: 5
- s3_secret_access_key:
- title: "S3 Secret Access Key"
type: "string"
- description: "The corresponding secret to the above access key id."
+ x-speakeasy-param-sensitive: true
+ api_base:
+ title: "Resource base URL"
+ description: "The base URL for your Azure OpenAI resource. You can\
+ \ find this in the Azure portal under your Azure OpenAI resource"
examples:
- - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
- airbyte_secret: true
- order: 6
- file_name_pattern:
+ - "https://your-resource-name.openai.azure.com"
type: "string"
- description: "The pattern allows you to set the file-name format for\
- \ the S3 staging file(s)"
- title: "S3 Filename pattern"
+ deployment:
+ title: "Deployment"
+ description: "The deployment for your Azure OpenAI resource. You\
+ \ can find this in the Azure portal under your Azure OpenAI resource"
examples:
- - "{date}"
- - "{date:yyyy_MM}"
- - "{timestamp}"
- - "{part_number}"
- - "{sync_id}"
- order: 7
- - title: "Azure Blob Storage"
+ - "your-resource-name"
+ type: "string"
required:
- - "data_source_type"
- - "azure_blob_storage_account_name"
- - "azure_blob_storage_container_name"
- - "azure_blob_storage_sas_token"
+ - "openai_key"
+ - "api_base"
+ - "deployment"
+ - "mode"
+ description: "Use the Azure-hosted OpenAI API to embed text. This option\
+ \ is using the text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "OpenAI-compatible"
+ type: "object"
properties:
- data_source_type:
- type: "string"
- const: "AZURE_BLOB_STORAGE"
- order: 0
+ mode:
+ title: "Mode"
+ default: "openai_compatible"
+ const: "openai_compatible"
enum:
- - "AZURE_BLOB_STORAGE"
- azure_blob_storage_endpoint_domain_name:
- title: "Endpoint Domain Name"
+ - "openai_compatible"
type: "string"
- default: "blob.core.windows.net"
- description: "This is Azure Blob Storage endpoint domain name. Leave\
- \ default value (or leave it empty if run container from command\
- \ line) to use Microsoft native from example."
- examples:
- - "blob.core.windows.net"
- order: 1
- azure_blob_storage_account_name:
- title: "Azure Blob Storage Account Name"
+ api_key:
+ title: "API key"
+ default: ""
+ airbyte_secret: true
type: "string"
- description: "The account's name of the Azure Blob Storage."
+ x-speakeasy-param-sensitive: true
+ base_url:
+ title: "Base URL"
+ description: "The base URL for your OpenAI-compatible service"
examples:
- - "airbyte5storage"
- order: 2
- azure_blob_storage_container_name:
- title: "Azure Blob Storage Container Name"
+ - "https://your-service-name.com"
type: "string"
- description: "The name of the Azure blob storage container."
+ model_name:
+ title: "Model name"
+ description: "The name of the model to use for embedding"
+ default: "text-embedding-ada-002"
examples:
- - "airbytetestcontainername"
- order: 3
- azure_blob_storage_sas_token:
- title: "SAS Token"
+ - "text-embedding-ada-002"
type: "string"
- airbyte_secret: true
- description: "Shared access signature (SAS) token to grant limited\
- \ access to objects in your storage account."
+ dimensions:
+ title: "Embedding dimensions"
+ description: "The number of dimensions the embedding model is generating"
examples:
- - "?sv=2016-05-31&ss=b&srt=sco&sp=rwdl&se=2018-06-27T10:05:50Z&st=2017-06-27T02:05:50Z&spr=https,http&sig=bgqQwoXwxzuD2GJfagRg7VOS8hzNr3QLT7rhS8OFRLQ%3D"
- order: 4
- purge_staging_data:
- title: "Purge Staging Files and Tables"
+ - 1536
+ - 384
+ type: "integer"
+ required:
+ - "base_url"
+ - "dimensions"
+ - "mode"
+ description: "Use a service that's compatible with the OpenAI API to embed\
+ \ text."
+ processing:
+ title: "ProcessingConfigModel"
+ type: "object"
+ properties:
+ chunk_size:
+ title: "Chunk size"
+ description: "Size of chunks in tokens to store in vector store (make\
+ \ sure it is not too big for the context if your LLM)"
+ maximum: 8191
+ minimum: 1
+ type: "integer"
+ chunk_overlap:
+ title: "Chunk overlap"
+ description: "Size of overlap between chunks in tokens to store in vector\
+ \ store to better capture relevant context"
+ default: 0
+ type: "integer"
+ text_fields:
+ title: "Text fields to embed"
+ description: "List of fields in the record that should be used to calculate\
+ \ the embedding. The field list is applied to all streams in the same\
+ \ way and non-existing fields are ignored. If none are defined, all\
+ \ fields are considered text fields. When specifying text fields,\
+ \ you can access nested fields in the record by using dot notation,\
+ \ e.g. `user.name` will access the `name` field in the `user` object.\
+ \ It's also possible to use wildcards to access all fields in an object,\
+ \ e.g. `users.*.name` will access all `names` fields in all entries\
+ \ of the `users` array."
+ default: []
+ always_show: true
+ examples:
+ - "text"
+ - "user.name"
+ - "users.*.name"
+ type: "array"
+ items:
+ type: "string"
+ metadata_fields:
+ title: "Fields to store as metadata"
+ description: "List of fields in the record that should be stored as\
+ \ metadata. The field list is applied to all streams in the same way\
+ \ and non-existing fields are ignored. If none are defined, all fields\
+ \ are considered metadata fields. When specifying text fields, you\
+ \ can access nested fields in the record by using dot notation, e.g.\
+ \ `user.name` will access the `name` field in the `user` object. It's\
+ \ also possible to use wildcards to access all fields in an object,\
+ \ e.g. `users.*.name` will access all `names` fields in all entries\
+ \ of the `users` array. When specifying nested paths, all matching\
+ \ values are flattened into an array set to a field named by the path."
+ default: []
+ always_show: true
+ examples:
+ - "age"
+ - "user"
+ - "user.name"
+ type: "array"
+ items:
+ type: "string"
+ text_splitter:
+ title: "Text splitter"
+ description: "Split text fields into chunks based on the specified method."
+ type: "object"
+ oneOf:
+ - title: "By Separator"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "separator"
+ const: "separator"
+ enum:
+ - "separator"
+ type: "string"
+ separators:
+ title: "Separators"
+ description: "List of separator strings to split text fields by.\
+ \ The separator itself needs to be wrapped in double quotes,\
+ \ e.g. to split by the dot character, use \".\". To split by\
+ \ a newline, use \"\\n\"."
+ default:
+ - "\"\\n\\n\""
+ - "\"\\n\""
+ - "\" \""
+ - "\"\""
+ type: "array"
+ items:
+ type: "string"
+ keep_separator:
+ title: "Keep separator"
+ description: "Whether to keep the separator in the resulting chunks"
+ default: false
+ type: "boolean"
+ description: "Split the text by the list of separators until the chunk\
+ \ size is reached, using the earlier mentioned separators where\
+ \ possible. This is useful for splitting text fields by paragraphs,\
+ \ sentences, words, etc."
+ required:
+ - "mode"
+ - title: "By Markdown header"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "markdown"
+ const: "markdown"
+ enum:
+ - "markdown"
+ type: "string"
+ split_level:
+ title: "Split level"
+ description: "Level of markdown headers to split text fields by.\
+ \ Headings down to the specified level will be used as split\
+ \ points"
+ default: 1
+ minimum: 1
+ maximum: 6
+ type: "integer"
+ description: "Split the text by Markdown headers down to the specified\
+ \ header level. If the chunk size fits multiple sections, they will\
+ \ be combined into a single chunk."
+ required:
+ - "mode"
+ - title: "By Programming Language"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "code"
+ const: "code"
+ enum:
+ - "code"
+ type: "string"
+ language:
+ title: "Language"
+ description: "Split code in suitable places based on the programming\
+ \ language"
+ enum:
+ - "cpp"
+ - "go"
+ - "java"
+ - "js"
+ - "php"
+ - "proto"
+ - "python"
+ - "rst"
+ - "ruby"
+ - "rust"
+ - "scala"
+ - "swift"
+ - "markdown"
+ - "latex"
+ - "html"
+ - "sol"
+ type: "string"
+ required:
+ - "language"
+ - "mode"
+ description: "Split the text by suitable delimiters based on the programming\
+ \ language. This is useful for splitting code into chunks."
+ field_name_mappings:
+ title: "Field name mappings"
+ description: "List of fields to rename. Not applicable for nested fields,\
+ \ but can be used to rename fields already flattened via dot notation."
+ default: []
+ type: "array"
+ items:
+ title: "FieldNameMappingConfigModel"
+ type: "object"
+ properties:
+ from_field:
+ title: "From field name"
+ description: "The field name in the source"
+ type: "string"
+ to_field:
+ title: "To field name"
+ description: "The field name to use in the destination"
+ type: "string"
+ required:
+ - "from_field"
+ - "to_field"
+ required:
+ - "chunk_size"
+ group: "processing"
+ omit_raw_text:
+ title: "Do not store raw text"
+ description: "Do not store the text that gets embedded along with the vector\
+ \ and the metadata in the destination. If set to true, only the vector\
+ \ and the metadata will be stored - in this case raw text for LLM use\
+ \ cases needs to be retrieved from another source."
+ default: false
+ group: "advanced"
type: "boolean"
- description: "Default to 'true'. Switch it to 'false' for debugging purpose."
- default: true
- order: 10
- destination-oracle:
- title: "Oracle Destination Spec"
- type: "object"
- required:
- - "host"
- - "port"
- - "username"
- - "sid"
- - "destinationType"
- properties:
- host:
- title: "Host"
- description: "The hostname of the database."
- type: "string"
+ indexing:
+ title: "Indexing"
+ type: "object"
+ properties:
+ host:
+ title: "Public Endpoint"
+ description: "The public endpoint of the Milvus instance. "
+ order: 1
+ examples:
+ - "https://my-instance.zone.zillizcloud.com"
+ - "tcp://host.docker.internal:19530"
+ - "tcp://my-local-milvus:19530"
+ type: "string"
+ db:
+ title: "Database Name"
+ description: "The database to connect to"
+ default: ""
+ type: "string"
+ collection:
+ title: "Collection Name"
+ description: "The collection to load data into"
+ order: 3
+ type: "string"
+ auth:
+ title: "Authentication"
+ description: "Authentication method"
+ type: "object"
+ order: 2
+ oneOf:
+ - title: "API Token"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "token"
+ const: "token"
+ enum:
+ - "token"
+ type: "string"
+ token:
+ title: "API Token"
+ description: "API Token for the Milvus instance"
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ required:
+ - "token"
+ - "mode"
+ description: "Authenticate using an API token (suitable for Zilliz\
+ \ Cloud)"
+ - title: "Username/Password"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "username_password"
+ const: "username_password"
+ enum:
+ - "username_password"
+ type: "string"
+ username:
+ title: "Username"
+ description: "Username for the Milvus instance"
+ order: 1
+ type: "string"
+ password:
+ title: "Password"
+ description: "Password for the Milvus instance"
+ airbyte_secret: true
+ order: 2
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ required:
+ - "username"
+ - "password"
+ - "mode"
+ description: "Authenticate using username and password (suitable for\
+ \ self-managed Milvus clusters)"
+ - title: "No auth"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "no_auth"
+ const: "no_auth"
+ enum:
+ - "no_auth"
+ type: "string"
+ description: "Do not authenticate (suitable for locally running test\
+ \ clusters, do not use for clusters with public IP addresses)"
+ required:
+ - "mode"
+ vector_field:
+ title: "Vector Field"
+ description: "The field in the entity that contains the vector"
+ default: "vector"
+ type: "string"
+ text_field:
+ title: "Text Field"
+ description: "The field in the entity that contains the embedded text"
+ default: "text"
+ type: "string"
+ required:
+ - "host"
+ - "collection"
+ - "auth"
+ group: "indexing"
+ description: "Indexing configuration"
+ destinationType:
+ title: "milvus"
+ const: "milvus"
+ enum:
+ - "milvus"
order: 0
- port:
- title: "Port"
- description: "The port of the database."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 1521
- examples:
- - "1521"
- order: 1
- sid:
- title: "SID"
- description: "The System Identifier uniquely distinguishes the instance\
- \ from any other instance on the same computer."
- type: "string"
- order: 2
- username:
- title: "User"
- description: "The username to access the database. This user must have CREATE\
- \ USER privileges in the database."
- type: "string"
- order: 3
- password:
- title: "Password"
- description: "The password associated with the username."
- type: "string"
- airbyte_secret: true
- order: 4
- x-speakeasy-param-sensitive: true
- jdbc_url_params:
- description: "Additional properties to pass to the JDBC URL string when\
- \ connecting to the database formatted as 'key=value' pairs separated\
- \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
- title: "JDBC URL Params"
- type: "string"
- order: 5
- schema:
- title: "Default Schema"
- description: "The default schema is used as the target schema for all statements\
- \ issued from the connection that do not explicitly specify a schema name.\
- \ The usual value for this field is \"airbyte\". In Oracle, schemas and\
- \ users are the same thing, so the \"user\" parameter is used as the login\
- \ credentials and this is used for the default Airbyte message schema."
type: "string"
- examples:
- - "airbyte"
- default: "airbyte"
- order: 6
- tunnel_method:
+ required:
+ - "embedding"
+ - "processing"
+ - "indexing"
+ - "destinationType"
+ groups:
+ - id: "processing"
+ title: "Processing"
+ - id: "embedding"
+ title: "Embedding"
+ - id: "indexing"
+ title: "Indexing"
+ - id: "advanced"
+ title: "Advanced"
+ destination-milvus-update:
+ title: "Destination Config"
+ description: "The configuration model for the Vector DB based destinations.\
+ \ This model is used to generate the UI for the destination configuration,\n\
+ as well as to provide type safety for the configuration passed to the destination.\n\
+ \nThe configuration model is composed of four parts:\n* Processing configuration\n\
+ * Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\
+ \nProcessing, embedding and advanced configuration are provided by this base\
+ \ class, while the indexing configuration is provided by the destination connector\
+ \ in the sub class."
+ type: "object"
+ properties:
+ embedding:
+ title: "Embedding"
+ description: "Embedding configuration"
+ group: "embedding"
type: "object"
- title: "SSH Tunnel Method"
- description: "Whether to initiate an SSH tunnel before connecting to the\
- \ database, and if so, which kind of authentication to use."
oneOf:
- - title: "No Tunnel"
- required:
- - "tunnel_method"
+ - title: "OpenAI"
+ type: "object"
properties:
- tunnel_method:
- description: "No ssh tunnel needed to connect to database"
+ mode:
+ title: "Mode"
+ default: "openai"
+ const: "openai"
+ enum:
+ - "openai"
type: "string"
- const: "NO_TUNNEL"
- order: 0
+ openai_key:
+ title: "OpenAI API key"
+ airbyte_secret: true
+ type: "string"
+ required:
+ - "openai_key"
+ - "mode"
+ description: "Use the OpenAI API to embed text. This option is using the\
+ \ text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "Cohere"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "cohere"
+ const: "cohere"
enum:
- - "NO_TUNNEL"
- - title: "SSH Key Authentication"
+ - "cohere"
+ type: "string"
+ cohere_key:
+ title: "Cohere API key"
+ airbyte_secret: true
+ type: "string"
required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "ssh_key"
+ - "cohere_key"
+ - "mode"
+ description: "Use the Cohere API to embed text."
+ - title: "Fake"
+ type: "object"
properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and ssh key"
+ mode:
+ title: "Mode"
+ default: "fake"
+ const: "fake"
+ enum:
+ - "fake"
type: "string"
- const: "SSH_KEY_AUTH"
- order: 0
+ description: "Use a fake embedding made out of random vectors with 1536\
+ \ embedding dimensions. This is useful for testing the data pipeline\
+ \ without incurring any costs."
+ required:
+ - "mode"
+ - title: "Azure OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "azure_openai"
+ const: "azure_openai"
enum:
- - "SSH_KEY_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
+ - "azure_openai"
type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
+ openai_key:
+ title: "Azure OpenAI API key"
+ description: "The API key for your Azure OpenAI resource. You can\
+ \ find this in the Azure portal under your Azure OpenAI resource"
+ airbyte_secret: true
+ type: "string"
+ api_base:
+ title: "Resource base URL"
+ description: "The base URL for your Azure OpenAI resource. You can\
+ \ find this in the Azure portal under your Azure OpenAI resource"
examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host."
+ - "https://your-resource-name.openai.azure.com"
type: "string"
- order: 3
- ssh_key:
- title: "SSH Private Key"
- description: "OS-level user account ssh key credentials in RSA PEM\
- \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
+ deployment:
+ title: "Deployment"
+ description: "The deployment for your Azure OpenAI resource. You\
+ \ can find this in the Azure portal under your Azure OpenAI resource"
+ examples:
+ - "your-resource-name"
type: "string"
- airbyte_secret: true
- multiline: true
- order: 4
- x-speakeasy-param-sensitive: true
- - title: "Password Authentication"
required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "tunnel_user_password"
+ - "openai_key"
+ - "api_base"
+ - "deployment"
+ - "mode"
+ description: "Use the Azure-hosted OpenAI API to embed text. This option\
+ \ is using the text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "OpenAI-compatible"
+ type: "object"
properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and password authentication"
- type: "string"
- const: "SSH_PASSWORD_AUTH"
- order: 0
+ mode:
+ title: "Mode"
+ default: "openai_compatible"
+ const: "openai_compatible"
enum:
- - "SSH_PASSWORD_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
+ - "openai_compatible"
type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
+ api_key:
+ title: "API key"
+ default: ""
+ airbyte_secret: true
+ type: "string"
+ base_url:
+ title: "Base URL"
+ description: "The base URL for your OpenAI-compatible service"
examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host"
+ - "https://your-service-name.com"
type: "string"
- order: 3
- tunnel_user_password:
- title: "Password"
- description: "OS-level password for logging into the jump server host"
+ model_name:
+ title: "Model name"
+ description: "The name of the model to use for embedding"
+ default: "text-embedding-ada-002"
+ examples:
+ - "text-embedding-ada-002"
type: "string"
- airbyte_secret: true
- order: 4
- x-speakeasy-param-sensitive: true
- destinationType:
- title: "oracle"
- const: "oracle"
- enum:
- - "oracle"
- order: 0
- type: "string"
- destination-oracle-update:
- title: "Oracle Destination Spec"
- type: "object"
+ dimensions:
+ title: "Embedding dimensions"
+ description: "The number of dimensions the embedding model is generating"
+ examples:
+ - 1536
+ - 384
+ type: "integer"
+ required:
+ - "base_url"
+ - "dimensions"
+ - "mode"
+ description: "Use a service that's compatible with the OpenAI API to embed\
+ \ text."
+ processing:
+ title: "ProcessingConfigModel"
+ type: "object"
+ properties:
+ chunk_size:
+ title: "Chunk size"
+ description: "Size of chunks in tokens to store in vector store (make\
+ \ sure it is not too big for the context if your LLM)"
+ maximum: 8191
+ minimum: 1
+ type: "integer"
+ chunk_overlap:
+ title: "Chunk overlap"
+ description: "Size of overlap between chunks in tokens to store in vector\
+ \ store to better capture relevant context"
+ default: 0
+ type: "integer"
+ text_fields:
+ title: "Text fields to embed"
+ description: "List of fields in the record that should be used to calculate\
+ \ the embedding. The field list is applied to all streams in the same\
+ \ way and non-existing fields are ignored. If none are defined, all\
+ \ fields are considered text fields. When specifying text fields,\
+ \ you can access nested fields in the record by using dot notation,\
+ \ e.g. `user.name` will access the `name` field in the `user` object.\
+ \ It's also possible to use wildcards to access all fields in an object,\
+ \ e.g. `users.*.name` will access all `names` fields in all entries\
+ \ of the `users` array."
+ default: []
+ always_show: true
+ examples:
+ - "text"
+ - "user.name"
+ - "users.*.name"
+ type: "array"
+ items:
+ type: "string"
+ metadata_fields:
+ title: "Fields to store as metadata"
+ description: "List of fields in the record that should be stored as\
+ \ metadata. The field list is applied to all streams in the same way\
+ \ and non-existing fields are ignored. If none are defined, all fields\
+ \ are considered metadata fields. When specifying text fields, you\
+ \ can access nested fields in the record by using dot notation, e.g.\
+ \ `user.name` will access the `name` field in the `user` object. It's\
+ \ also possible to use wildcards to access all fields in an object,\
+ \ e.g. `users.*.name` will access all `names` fields in all entries\
+ \ of the `users` array. When specifying nested paths, all matching\
+ \ values are flattened into an array set to a field named by the path."
+ default: []
+ always_show: true
+ examples:
+ - "age"
+ - "user"
+ - "user.name"
+ type: "array"
+ items:
+ type: "string"
+ text_splitter:
+ title: "Text splitter"
+ description: "Split text fields into chunks based on the specified method."
+ type: "object"
+ oneOf:
+ - title: "By Separator"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "separator"
+ const: "separator"
+ enum:
+ - "separator"
+ type: "string"
+ separators:
+ title: "Separators"
+ description: "List of separator strings to split text fields by.\
+ \ The separator itself needs to be wrapped in double quotes,\
+ \ e.g. to split by the dot character, use \".\". To split by\
+ \ a newline, use \"\\n\"."
+ default:
+ - "\"\\n\\n\""
+ - "\"\\n\""
+ - "\" \""
+ - "\"\""
+ type: "array"
+ items:
+ type: "string"
+ keep_separator:
+ title: "Keep separator"
+ description: "Whether to keep the separator in the resulting chunks"
+ default: false
+ type: "boolean"
+ description: "Split the text by the list of separators until the chunk\
+ \ size is reached, using the earlier mentioned separators where\
+ \ possible. This is useful for splitting text fields by paragraphs,\
+ \ sentences, words, etc."
+ required:
+ - "mode"
+ - title: "By Markdown header"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "markdown"
+ const: "markdown"
+ enum:
+ - "markdown"
+ type: "string"
+ split_level:
+ title: "Split level"
+ description: "Level of markdown headers to split text fields by.\
+ \ Headings down to the specified level will be used as split\
+ \ points"
+ default: 1
+ minimum: 1
+ maximum: 6
+ type: "integer"
+ description: "Split the text by Markdown headers down to the specified\
+ \ header level. If the chunk size fits multiple sections, they will\
+ \ be combined into a single chunk."
+ required:
+ - "mode"
+ - title: "By Programming Language"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "code"
+ const: "code"
+ enum:
+ - "code"
+ type: "string"
+ language:
+ title: "Language"
+ description: "Split code in suitable places based on the programming\
+ \ language"
+ enum:
+ - "cpp"
+ - "go"
+ - "java"
+ - "js"
+ - "php"
+ - "proto"
+ - "python"
+ - "rst"
+ - "ruby"
+ - "rust"
+ - "scala"
+ - "swift"
+ - "markdown"
+ - "latex"
+ - "html"
+ - "sol"
+ type: "string"
+ required:
+ - "language"
+ - "mode"
+ description: "Split the text by suitable delimiters based on the programming\
+ \ language. This is useful for splitting code into chunks."
+ field_name_mappings:
+ title: "Field name mappings"
+ description: "List of fields to rename. Not applicable for nested fields,\
+ \ but can be used to rename fields already flattened via dot notation."
+ default: []
+ type: "array"
+ items:
+ title: "FieldNameMappingConfigModel"
+ type: "object"
+ properties:
+ from_field:
+ title: "From field name"
+ description: "The field name in the source"
+ type: "string"
+ to_field:
+ title: "To field name"
+ description: "The field name to use in the destination"
+ type: "string"
+ required:
+ - "from_field"
+ - "to_field"
+ required:
+ - "chunk_size"
+ group: "processing"
+ omit_raw_text:
+ title: "Do not store raw text"
+ description: "Do not store the text that gets embedded along with the vector\
+ \ and the metadata in the destination. If set to true, only the vector\
+ \ and the metadata will be stored - in this case raw text for LLM use\
+ \ cases needs to be retrieved from another source."
+ default: false
+ group: "advanced"
+ type: "boolean"
+ indexing:
+ title: "Indexing"
+ type: "object"
+ properties:
+ host:
+ title: "Public Endpoint"
+ description: "The public endpoint of the Milvus instance. "
+ order: 1
+ examples:
+ - "https://my-instance.zone.zillizcloud.com"
+ - "tcp://host.docker.internal:19530"
+ - "tcp://my-local-milvus:19530"
+ type: "string"
+ db:
+ title: "Database Name"
+ description: "The database to connect to"
+ default: ""
+ type: "string"
+ collection:
+ title: "Collection Name"
+ description: "The collection to load data into"
+ order: 3
+ type: "string"
+ auth:
+ title: "Authentication"
+ description: "Authentication method"
+ type: "object"
+ order: 2
+ oneOf:
+ - title: "API Token"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "token"
+ const: "token"
+ enum:
+ - "token"
+ type: "string"
+ token:
+ title: "API Token"
+ description: "API Token for the Milvus instance"
+ airbyte_secret: true
+ type: "string"
+ required:
+ - "token"
+ - "mode"
+ description: "Authenticate using an API token (suitable for Zilliz\
+ \ Cloud)"
+ - title: "Username/Password"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "username_password"
+ const: "username_password"
+ enum:
+ - "username_password"
+ type: "string"
+ username:
+ title: "Username"
+ description: "Username for the Milvus instance"
+ order: 1
+ type: "string"
+ password:
+ title: "Password"
+ description: "Password for the Milvus instance"
+ airbyte_secret: true
+ order: 2
+ type: "string"
+ required:
+ - "username"
+ - "password"
+ - "mode"
+ description: "Authenticate using username and password (suitable for\
+ \ self-managed Milvus clusters)"
+ - title: "No auth"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "no_auth"
+ const: "no_auth"
+ enum:
+ - "no_auth"
+ type: "string"
+ description: "Do not authenticate (suitable for locally running test\
+ \ clusters, do not use for clusters with public IP addresses)"
+ required:
+ - "mode"
+ vector_field:
+ title: "Vector Field"
+ description: "The field in the entity that contains the vector"
+ default: "vector"
+ type: "string"
+ text_field:
+ title: "Text Field"
+ description: "The field in the entity that contains the embedded text"
+ default: "text"
+ type: "string"
+ required:
+ - "host"
+ - "collection"
+ - "auth"
+ group: "indexing"
+ description: "Indexing configuration"
required:
- - "host"
- - "port"
- - "username"
- - "sid"
- properties:
- host:
- title: "Host"
- description: "The hostname of the database."
- type: "string"
- order: 0
- port:
- title: "Port"
- description: "The port of the database."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 1521
- examples:
- - "1521"
- order: 1
- sid:
- title: "SID"
- description: "The System Identifier uniquely distinguishes the instance\
- \ from any other instance on the same computer."
- type: "string"
- order: 2
- username:
- title: "User"
- description: "The username to access the database. This user must have CREATE\
- \ USER privileges in the database."
- type: "string"
- order: 3
- password:
- title: "Password"
- description: "The password associated with the username."
- type: "string"
- airbyte_secret: true
- order: 4
- jdbc_url_params:
- description: "Additional properties to pass to the JDBC URL string when\
- \ connecting to the database formatted as 'key=value' pairs separated\
- \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
- title: "JDBC URL Params"
- type: "string"
- order: 5
- schema:
- title: "Default Schema"
- description: "The default schema is used as the target schema for all statements\
- \ issued from the connection that do not explicitly specify a schema name.\
- \ The usual value for this field is \"airbyte\". In Oracle, schemas and\
- \ users are the same thing, so the \"user\" parameter is used as the login\
- \ credentials and this is used for the default Airbyte message schema."
- type: "string"
- examples:
- - "airbyte"
- default: "airbyte"
- order: 6
- tunnel_method:
- type: "object"
- title: "SSH Tunnel Method"
- description: "Whether to initiate an SSH tunnel before connecting to the\
- \ database, and if so, which kind of authentication to use."
- oneOf:
- - title: "No Tunnel"
- required:
- - "tunnel_method"
- properties:
- tunnel_method:
- description: "No ssh tunnel needed to connect to database"
- type: "string"
- const: "NO_TUNNEL"
- order: 0
- enum:
- - "NO_TUNNEL"
- - title: "SSH Key Authentication"
- required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "ssh_key"
- properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and ssh key"
- type: "string"
- const: "SSH_KEY_AUTH"
- order: 0
- enum:
- - "SSH_KEY_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
- type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
- examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host."
- type: "string"
- order: 3
- ssh_key:
- title: "SSH Private Key"
- description: "OS-level user account ssh key credentials in RSA PEM\
- \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
- type: "string"
- airbyte_secret: true
- multiline: true
- order: 4
- - title: "Password Authentication"
- required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "tunnel_user_password"
- properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and password authentication"
- type: "string"
- const: "SSH_PASSWORD_AUTH"
- order: 0
- enum:
- - "SSH_PASSWORD_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
- type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
- examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host"
- type: "string"
- order: 3
- tunnel_user_password:
- title: "Password"
- description: "OS-level password for logging into the jump server host"
- type: "string"
- airbyte_secret: true
- order: 4
- destination-aws-datalake:
- title: "AWS Datalake Destination Spec"
+ - "embedding"
+ - "processing"
+ - "indexing"
+ groups:
+ - id: "processing"
+ title: "Processing"
+ - id: "embedding"
+ title: "Embedding"
+ - id: "indexing"
+ title: "Indexing"
+ - id: "advanced"
+ title: "Advanced"
+ destination-google-sheets:
+ title: "Destination Google Sheets"
type: "object"
required:
+ - "spreadsheet_id"
- "credentials"
- - "region"
- - "bucket_name"
- - "lakeformation_database_name"
- "destinationType"
properties:
- aws_account_id:
+ spreadsheet_id:
type: "string"
- title: "AWS Account Id"
- description: "target aws account id"
+ title: "Spreadsheet Link"
+ description: "The link to your spreadsheet. See this\
+ \ guide for more details."
examples:
- - "111111111111"
- order: 1
+ - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG/edit"
credentials:
- title: "Authentication mode"
- description: "Choose How to Authenticate to AWS."
- type: "object"
- oneOf:
- - type: "object"
- title: "IAM Role"
- required:
- - "role_arn"
- - "credentials_title"
- properties:
- credentials_title:
- type: "string"
- title: "Credentials Title"
- description: "Name of the credentials"
- const: "IAM Role"
- enum:
- - "IAM Role"
- default: "IAM Role"
- order: 0
- role_arn:
- title: "Target Role Arn"
- type: "string"
- description: "Will assume this role to write data to s3"
- airbyte_secret: false
- - type: "object"
- title: "IAM User"
- required:
- - "credentials_title"
- - "aws_access_key_id"
- - "aws_secret_access_key"
- properties:
- credentials_title:
- type: "string"
- title: "Credentials Title"
- description: "Name of the credentials"
- const: "IAM User"
- enum:
- - "IAM User"
- default: "IAM User"
- order: 0
- aws_access_key_id:
- title: "Access Key Id"
- type: "string"
- description: "AWS User Access Key Id"
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- aws_secret_access_key:
- title: "Secret Access Key"
- type: "string"
- description: "Secret Access Key"
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- order: 2
- region:
- title: "S3 Bucket Region"
- type: "string"
- default: ""
- description: "The region of the S3 bucket. See here for all region codes."
- enum:
- - ""
- - "af-south-1"
- - "ap-east-1"
- - "ap-northeast-1"
- - "ap-northeast-2"
- - "ap-northeast-3"
- - "ap-south-1"
- - "ap-south-2"
- - "ap-southeast-1"
- - "ap-southeast-2"
- - "ap-southeast-3"
- - "ap-southeast-4"
- - "ca-central-1"
- - "ca-west-1"
- - "cn-north-1"
- - "cn-northwest-1"
- - "eu-central-1"
- - "eu-central-2"
- - "eu-north-1"
- - "eu-south-1"
- - "eu-south-2"
- - "eu-west-1"
- - "eu-west-2"
- - "eu-west-3"
- - "il-central-1"
- - "me-central-1"
- - "me-south-1"
- - "sa-east-1"
- - "us-east-1"
- - "us-east-2"
- - "us-gov-east-1"
- - "us-gov-west-1"
- - "us-west-1"
- - "us-west-2"
- order: 3
- bucket_name:
- title: "S3 Bucket Name"
- type: "string"
- description: "The name of the S3 bucket. Read more here."
- order: 4
- bucket_prefix:
- title: "Target S3 Bucket Prefix"
- type: "string"
- description: "S3 prefix"
- order: 5
- lakeformation_database_name:
- title: "Lake Formation Database Name"
- type: "string"
- description: "The default database this destination will use to create tables\
- \ in per stream. Can be changed per connection by customizing the namespace."
- order: 6
- lakeformation_database_default_tag_key:
- title: "Lake Formation Database Tag Key"
- description: "Add a default tag key to databases created by this destination"
- examples:
- - "pii_level"
- type: "string"
- order: 7
- x-speakeasy-param-sensitive: true
- lakeformation_database_default_tag_values:
- title: "Lake Formation Database Tag Values"
- description: "Add default values for the `Tag Key` to databases created\
- \ by this destination. Comma separate for multiple values."
- examples:
- - "private,public"
- type: "string"
- order: 8
- lakeformation_governed_tables:
- title: "Lake Formation Governed Tables"
- description: "Whether to create tables as LF governed tables."
- type: "boolean"
- default: false
- order: 9
- format:
- title: "Output Format *"
type: "object"
- description: "Format of the data output."
- oneOf:
- - title: "JSON Lines: Newline-delimited JSON"
- required:
- - "format_type"
- properties:
- format_type:
- title: "Format Type *"
- type: "string"
- enum:
- - "JSONL"
- default: "JSONL"
- compression_codec:
- title: "Compression Codec (Optional)"
- description: "The compression algorithm used to compress data."
- type: "string"
- enum:
- - "UNCOMPRESSED"
- - "GZIP"
- default: "UNCOMPRESSED"
- - title: "Parquet: Columnar Storage"
- required:
- - "format_type"
- properties:
- format_type:
- title: "Format Type *"
- type: "string"
- enum:
- - "Parquet"
- default: "Parquet"
- compression_codec:
- title: "Compression Codec (Optional)"
- description: "The compression algorithm used to compress data."
- type: "string"
- enum:
- - "UNCOMPRESSED"
- - "SNAPPY"
- - "GZIP"
- - "ZSTD"
- default: "SNAPPY"
- order: 10
- partitioning:
- title: "Choose how to partition data"
- description: "Partition data by cursor fields when a cursor field is a date"
- type: "string"
- enum:
- - "NO PARTITIONING"
- - "DATE"
- - "YEAR"
- - "MONTH"
- - "DAY"
- - "YEAR/MONTH"
- - "YEAR/MONTH/DAY"
- default: "NO PARTITIONING"
- order: 11
- glue_catalog_float_as_decimal:
- title: "Glue Catalog: Float as Decimal"
- description: "Cast float/double as decimal(38,18). This can help achieve\
- \ higher accuracy and represent numbers correctly as received from the\
- \ source."
- type: "boolean"
- default: false
- order: 12
+ title: "Authentication via Google (OAuth)"
+ description: "Google API Credentials for connecting to Google Sheets and\
+ \ Google Drive APIs"
+ required:
+ - "client_id"
+ - "client_secret"
+ - "refresh_token"
+ properties:
+ client_id:
+ title: "Client ID"
+ type: "string"
+ description: "The Client ID of your Google Sheets developer application."
+ airbyte_secret: true
+ client_secret:
+ title: "Client Secret"
+ type: "string"
+ description: "The Client Secret of your Google Sheets developer application."
+ airbyte_secret: true
+ refresh_token:
+ title: "Refresh Token"
+ type: "string"
+ description: "The token for obtaining new access token."
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
destinationType:
- title: "aws-datalake"
- const: "aws-datalake"
+ title: "google-sheets"
+ const: "google-sheets"
enum:
- - "aws-datalake"
+ - "google-sheets"
order: 0
type: "string"
- destination-aws-datalake-update:
- title: "AWS Datalake Destination Spec"
+ destination-google-sheets-update:
+ title: "Destination Google Sheets"
type: "object"
required:
+ - "spreadsheet_id"
- "credentials"
- - "region"
- - "bucket_name"
- - "lakeformation_database_name"
properties:
- aws_account_id:
+ spreadsheet_id:
type: "string"
- title: "AWS Account Id"
- description: "target aws account id"
+ title: "Spreadsheet Link"
+ description: "The link to your spreadsheet. See this\
+ \ guide for more details."
examples:
- - "111111111111"
- order: 1
+ - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG/edit"
credentials:
- title: "Authentication mode"
- description: "Choose How to Authenticate to AWS."
+ type: "object"
+ title: "Authentication via Google (OAuth)"
+ description: "Google API Credentials for connecting to Google Sheets and\
+ \ Google Drive APIs"
+ required:
+ - "client_id"
+ - "client_secret"
+ - "refresh_token"
+ properties:
+ client_id:
+ title: "Client ID"
+ type: "string"
+ description: "The Client ID of your Google Sheets developer application."
+ airbyte_secret: true
+ client_secret:
+ title: "Client Secret"
+ type: "string"
+ description: "The Client Secret of your Google Sheets developer application."
+ airbyte_secret: true
+ refresh_token:
+ title: "Refresh Token"
+ type: "string"
+ description: "The token for obtaining new access token."
+ airbyte_secret: true
+ destination-astra:
+ title: "Destination Config"
+ description: "The configuration model for the Vector DB based destinations.\
+ \ This model is used to generate the UI for the destination configuration,\n\
+ as well as to provide type safety for the configuration passed to the destination.\n\
+ \nThe configuration model is composed of four parts:\n* Processing configuration\n\
+ * Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\
+ \nProcessing, embedding and advanced configuration are provided by this base\
+ \ class, while the indexing configuration is provided by the destination connector\
+ \ in the sub class."
+ type: "object"
+ properties:
+ embedding:
+ title: "Embedding"
+ description: "Embedding configuration"
+ group: "embedding"
type: "object"
oneOf:
- - type: "object"
- title: "IAM Role"
+ - title: "OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "openai"
+ const: "openai"
+ enum:
+ - "openai"
+ type: "string"
+ openai_key:
+ title: "OpenAI API key"
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
required:
- - "role_arn"
- - "credentials_title"
+ - "openai_key"
+ - "mode"
+ description: "Use the OpenAI API to embed text. This option is using the\
+ \ text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "Cohere"
+ type: "object"
properties:
- credentials_title:
- type: "string"
- title: "Credentials Title"
- description: "Name of the credentials"
- const: "IAM Role"
+ mode:
+ title: "Mode"
+ default: "cohere"
+ const: "cohere"
enum:
- - "IAM Role"
- default: "IAM Role"
- order: 0
- role_arn:
- title: "Target Role Arn"
+ - "cohere"
type: "string"
- description: "Will assume this role to write data to s3"
- airbyte_secret: false
- - type: "object"
- title: "IAM User"
+ cohere_key:
+ title: "Cohere API key"
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
required:
- - "credentials_title"
- - "aws_access_key_id"
- - "aws_secret_access_key"
+ - "cohere_key"
+ - "mode"
+ description: "Use the Cohere API to embed text."
+ - title: "Fake"
+ type: "object"
properties:
- credentials_title:
+ mode:
+ title: "Mode"
+ default: "fake"
+ const: "fake"
+ enum:
+ - "fake"
type: "string"
- title: "Credentials Title"
- description: "Name of the credentials"
- const: "IAM User"
+ description: "Use a fake embedding made out of random vectors with 1536\
+ \ embedding dimensions. This is useful for testing the data pipeline\
+ \ without incurring any costs."
+ required:
+ - "mode"
+ - title: "Azure OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "azure_openai"
+ const: "azure_openai"
enum:
- - "IAM User"
- default: "IAM User"
- order: 0
- aws_access_key_id:
- title: "Access Key Id"
+ - "azure_openai"
type: "string"
- description: "AWS User Access Key Id"
+ openai_key:
+ title: "Azure OpenAI API key"
+ description: "The API key for your Azure OpenAI resource. You can\
+ \ find this in the Azure portal under your Azure OpenAI resource"
airbyte_secret: true
- aws_secret_access_key:
- title: "Secret Access Key"
type: "string"
- description: "Secret Access Key"
- airbyte_secret: true
- order: 2
- region:
- title: "S3 Bucket Region"
- type: "string"
- default: ""
- description: "The region of the S3 bucket. See here for all region codes."
- enum:
- - ""
- - "af-south-1"
- - "ap-east-1"
- - "ap-northeast-1"
- - "ap-northeast-2"
- - "ap-northeast-3"
- - "ap-south-1"
- - "ap-south-2"
- - "ap-southeast-1"
- - "ap-southeast-2"
- - "ap-southeast-3"
- - "ap-southeast-4"
- - "ca-central-1"
- - "ca-west-1"
- - "cn-north-1"
- - "cn-northwest-1"
- - "eu-central-1"
- - "eu-central-2"
- - "eu-north-1"
- - "eu-south-1"
- - "eu-south-2"
- - "eu-west-1"
- - "eu-west-2"
- - "eu-west-3"
- - "il-central-1"
- - "me-central-1"
- - "me-south-1"
- - "sa-east-1"
- - "us-east-1"
- - "us-east-2"
- - "us-gov-east-1"
- - "us-gov-west-1"
- - "us-west-1"
- - "us-west-2"
- order: 3
- bucket_name:
- title: "S3 Bucket Name"
- type: "string"
- description: "The name of the S3 bucket. Read more here."
- order: 4
- bucket_prefix:
- title: "Target S3 Bucket Prefix"
- type: "string"
- description: "S3 prefix"
- order: 5
- lakeformation_database_name:
- title: "Lake Formation Database Name"
- type: "string"
- description: "The default database this destination will use to create tables\
- \ in per stream. Can be changed per connection by customizing the namespace."
- order: 6
- lakeformation_database_default_tag_key:
- title: "Lake Formation Database Tag Key"
- description: "Add a default tag key to databases created by this destination"
- examples:
- - "pii_level"
- type: "string"
- order: 7
- lakeformation_database_default_tag_values:
- title: "Lake Formation Database Tag Values"
- description: "Add default values for the `Tag Key` to databases created\
- \ by this destination. Comma separate for multiple values."
- examples:
- - "private,public"
- type: "string"
- order: 8
- lakeformation_governed_tables:
- title: "Lake Formation Governed Tables"
- description: "Whether to create tables as LF governed tables."
- type: "boolean"
- default: false
- order: 9
- format:
- title: "Output Format *"
- type: "object"
- description: "Format of the data output."
- oneOf:
- - title: "JSON Lines: Newline-delimited JSON"
+ x-speakeasy-param-sensitive: true
+ api_base:
+ title: "Resource base URL"
+ description: "The base URL for your Azure OpenAI resource. You can\
+ \ find this in the Azure portal under your Azure OpenAI resource"
+ examples:
+ - "https://your-resource-name.openai.azure.com"
+ type: "string"
+ deployment:
+ title: "Deployment"
+ description: "The deployment for your Azure OpenAI resource. You\
+ \ can find this in the Azure portal under your Azure OpenAI resource"
+ examples:
+ - "your-resource-name"
+ type: "string"
required:
- - "format_type"
+ - "openai_key"
+ - "api_base"
+ - "deployment"
+ - "mode"
+ description: "Use the Azure-hosted OpenAI API to embed text. This option\
+ \ is using the text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "OpenAI-compatible"
+ type: "object"
properties:
- format_type:
- title: "Format Type *"
- type: "string"
+ mode:
+ title: "Mode"
+ default: "openai_compatible"
+ const: "openai_compatible"
enum:
- - "JSONL"
- default: "JSONL"
- compression_codec:
- title: "Compression Codec (Optional)"
- description: "The compression algorithm used to compress data."
+ - "openai_compatible"
type: "string"
- enum:
- - "UNCOMPRESSED"
- - "GZIP"
- default: "UNCOMPRESSED"
- - title: "Parquet: Columnar Storage"
+ api_key:
+ title: "API key"
+ default: ""
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ base_url:
+ title: "Base URL"
+ description: "The base URL for your OpenAI-compatible service"
+ examples:
+ - "https://your-service-name.com"
+ type: "string"
+ model_name:
+ title: "Model name"
+ description: "The name of the model to use for embedding"
+ default: "text-embedding-ada-002"
+ examples:
+ - "text-embedding-ada-002"
+ type: "string"
+ dimensions:
+ title: "Embedding dimensions"
+ description: "The number of dimensions the embedding model is generating"
+ examples:
+ - 1536
+ - 384
+ type: "integer"
required:
- - "format_type"
- properties:
- format_type:
- title: "Format Type *"
+ - "base_url"
+ - "dimensions"
+ - "mode"
+ description: "Use a service that's compatible with the OpenAI API to embed\
+ \ text."
+ processing:
+ title: "ProcessingConfigModel"
+ type: "object"
+ properties:
+ chunk_size:
+ title: "Chunk size"
+ description: "Size of chunks in tokens to store in vector store (make\
+ \ sure it is not too big for the context if your LLM)"
+ maximum: 8191
+ minimum: 1
+ type: "integer"
+ chunk_overlap:
+ title: "Chunk overlap"
+ description: "Size of overlap between chunks in tokens to store in vector\
+ \ store to better capture relevant context"
+ default: 0
+ type: "integer"
+ text_fields:
+ title: "Text fields to embed"
+ description: "List of fields in the record that should be used to calculate\
+ \ the embedding. The field list is applied to all streams in the same\
+ \ way and non-existing fields are ignored. If none are defined, all\
+ \ fields are considered text fields. When specifying text fields,\
+ \ you can access nested fields in the record by using dot notation,\
+ \ e.g. `user.name` will access the `name` field in the `user` object.\
+ \ It's also possible to use wildcards to access all fields in an object,\
+ \ e.g. `users.*.name` will access all `names` fields in all entries\
+ \ of the `users` array."
+ default: []
+ always_show: true
+ examples:
+ - "text"
+ - "user.name"
+ - "users.*.name"
+ type: "array"
+ items:
type: "string"
- enum:
- - "Parquet"
- default: "Parquet"
- compression_codec:
- title: "Compression Codec (Optional)"
- description: "The compression algorithm used to compress data."
+ metadata_fields:
+ title: "Fields to store as metadata"
+ description: "List of fields in the record that should be stored as\
+ \ metadata. The field list is applied to all streams in the same way\
+ \ and non-existing fields are ignored. If none are defined, all fields\
+ \ are considered metadata fields. When specifying text fields, you\
+ \ can access nested fields in the record by using dot notation, e.g.\
+ \ `user.name` will access the `name` field in the `user` object. It's\
+ \ also possible to use wildcards to access all fields in an object,\
+ \ e.g. `users.*.name` will access all `names` fields in all entries\
+ \ of the `users` array. When specifying nested paths, all matching\
+ \ values are flattened into an array set to a field named by the path."
+ default: []
+ always_show: true
+ examples:
+ - "age"
+ - "user"
+ - "user.name"
+ type: "array"
+ items:
type: "string"
- enum:
- - "UNCOMPRESSED"
- - "SNAPPY"
- - "GZIP"
- - "ZSTD"
- default: "SNAPPY"
- order: 10
- partitioning:
- title: "Choose how to partition data"
- description: "Partition data by cursor fields when a cursor field is a date"
- type: "string"
- enum:
- - "NO PARTITIONING"
- - "DATE"
- - "YEAR"
- - "MONTH"
- - "DAY"
- - "YEAR/MONTH"
- - "YEAR/MONTH/DAY"
- default: "NO PARTITIONING"
- order: 11
- glue_catalog_float_as_decimal:
- title: "Glue Catalog: Float as Decimal"
- description: "Cast float/double as decimal(38,18). This can help achieve\
- \ higher accuracy and represent numbers correctly as received from the\
- \ source."
- type: "boolean"
+ text_splitter:
+ title: "Text splitter"
+ description: "Split text fields into chunks based on the specified method."
+ type: "object"
+ oneOf:
+ - title: "By Separator"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "separator"
+ const: "separator"
+ enum:
+ - "separator"
+ type: "string"
+ separators:
+ title: "Separators"
+ description: "List of separator strings to split text fields by.\
+ \ The separator itself needs to be wrapped in double quotes,\
+ \ e.g. to split by the dot character, use \".\". To split by\
+ \ a newline, use \"\\n\"."
+ default:
+ - "\"\\n\\n\""
+ - "\"\\n\""
+ - "\" \""
+ - "\"\""
+ type: "array"
+ items:
+ type: "string"
+ keep_separator:
+ title: "Keep separator"
+ description: "Whether to keep the separator in the resulting chunks"
+ default: false
+ type: "boolean"
+ description: "Split the text by the list of separators until the chunk\
+ \ size is reached, using the earlier mentioned separators where\
+ \ possible. This is useful for splitting text fields by paragraphs,\
+ \ sentences, words, etc."
+ required:
+ - "mode"
+ - title: "By Markdown header"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "markdown"
+ const: "markdown"
+ enum:
+ - "markdown"
+ type: "string"
+ split_level:
+ title: "Split level"
+ description: "Level of markdown headers to split text fields by.\
+ \ Headings down to the specified level will be used as split\
+ \ points"
+ default: 1
+ minimum: 1
+ maximum: 6
+ type: "integer"
+ description: "Split the text by Markdown headers down to the specified\
+ \ header level. If the chunk size fits multiple sections, they will\
+ \ be combined into a single chunk."
+ required:
+ - "mode"
+ - title: "By Programming Language"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "code"
+ const: "code"
+ enum:
+ - "code"
+ type: "string"
+ language:
+ title: "Language"
+ description: "Split code in suitable places based on the programming\
+ \ language"
+ enum:
+ - "cpp"
+ - "go"
+ - "java"
+ - "js"
+ - "php"
+ - "proto"
+ - "python"
+ - "rst"
+ - "ruby"
+ - "rust"
+ - "scala"
+ - "swift"
+ - "markdown"
+ - "latex"
+ - "html"
+ - "sol"
+ type: "string"
+ required:
+ - "language"
+ - "mode"
+ description: "Split the text by suitable delimiters based on the programming\
+ \ language. This is useful for splitting code into chunks."
+ field_name_mappings:
+ title: "Field name mappings"
+ description: "List of fields to rename. Not applicable for nested fields,\
+ \ but can be used to rename fields already flattened via dot notation."
+ default: []
+ type: "array"
+ items:
+ title: "FieldNameMappingConfigModel"
+ type: "object"
+ properties:
+ from_field:
+ title: "From field name"
+ description: "The field name in the source"
+ type: "string"
+ to_field:
+ title: "To field name"
+ description: "The field name to use in the destination"
+ type: "string"
+ required:
+ - "from_field"
+ - "to_field"
+ required:
+ - "chunk_size"
+ group: "processing"
+ omit_raw_text:
+ title: "Do not store raw text"
+ description: "Do not store the text that gets embedded along with the vector\
+ \ and the metadata in the destination. If set to true, only the vector\
+ \ and the metadata will be stored - in this case raw text for LLM use\
+ \ cases needs to be retrieved from another source."
default: false
- order: 12
- destination-milvus:
+ group: "advanced"
+ type: "boolean"
+ indexing:
+ title: "Indexing"
+ type: "object"
+ properties:
+ astra_db_app_token:
+ title: "Astra DB Application Token"
+ description: "The application token authorizes a user to connect to\
+ \ a specific Astra DB database. It is created when the user clicks\
+ \ the Generate Token button on the Overview tab of the Database page\
+ \ in the Astra UI."
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ astra_db_endpoint:
+ title: "Astra DB Endpoint"
+ description: "The endpoint specifies which Astra DB database queries\
+ \ are sent to. It can be copied from the Database Details section\
+ \ of the Overview tab of the Database page in the Astra UI."
+ pattern: "^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\\
+ .]*?\\.apps\\.astra\\.datastax\\.com"
+ examples:
+ - "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com"
+ type: "string"
+ astra_db_keyspace:
+ title: "Astra DB Keyspace"
+ description: "Keyspaces (or Namespaces) serve as containers for organizing\
+ \ data within a database. You can create a new keyspace uisng the\
+ \ Data Explorer tab in the Astra UI. The keyspace default_keyspace\
+ \ is created for you when you create a Vector Database in Astra DB."
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ collection:
+ title: "Astra DB collection"
+ description: "Collections hold data. They are analagous to tables in\
+ \ traditional Cassandra terminology. This tool will create the collection\
+ \ with the provided name automatically if it does not already exist.\
+ \ Alternatively, you can create one thorugh the Data Explorer tab\
+ \ in the Astra UI."
+ type: "string"
+ required:
+ - "astra_db_app_token"
+ - "astra_db_endpoint"
+ - "astra_db_keyspace"
+ - "collection"
+ description: "Astra DB gives developers the APIs, real-time data and ecosystem\
+ \ integrations to put accurate RAG and Gen AI apps with fewer hallucinations\
+ \ in production."
+ group: "indexing"
+ destinationType:
+ title: "astra"
+ const: "astra"
+ enum:
+ - "astra"
+ order: 0
+ type: "string"
+ required:
+ - "embedding"
+ - "processing"
+ - "indexing"
+ - "destinationType"
+ groups:
+ - id: "processing"
+ title: "Processing"
+ - id: "embedding"
+ title: "Embedding"
+ - id: "indexing"
+ title: "Indexing"
+ - id: "advanced"
+ title: "Advanced"
+ destination-astra-update:
title: "Destination Config"
description: "The configuration model for the Vector DB based destinations.\
\ This model is used to generate the UI for the destination configuration,\n\
@@ -67111,7 +69490,6 @@ components:
title: "OpenAI API key"
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
required:
- "openai_key"
- "mode"
@@ -67131,7 +69509,6 @@ components:
title: "Cohere API key"
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
required:
- "cohere_key"
- "mode"
@@ -67167,7 +69544,6 @@ components:
\ find this in the Azure portal under your Azure OpenAI resource"
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
api_base:
title: "Resource base URL"
description: "The base URL for your Azure OpenAI resource. You can\
@@ -67204,7 +69580,6 @@ components:
default: ""
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
base_url:
title: "Base URL"
description: "The base URL for your OpenAI-compatible service"
@@ -67427,122 +69802,52 @@ components:
title: "Indexing"
type: "object"
properties:
- host:
- title: "Public Endpoint"
- description: "The public endpoint of the Milvus instance. "
- order: 1
+ astra_db_app_token:
+ title: "Astra DB Application Token"
+ description: "The application token authorizes a user to connect to\
+ \ a specific Astra DB database. It is created when the user clicks\
+ \ the Generate Token button on the Overview tab of the Database page\
+ \ in the Astra UI."
+ airbyte_secret: true
+ type: "string"
+ astra_db_endpoint:
+ title: "Astra DB Endpoint"
+ description: "The endpoint specifies which Astra DB database queries\
+ \ are sent to. It can be copied from the Database Details section\
+ \ of the Overview tab of the Database page in the Astra UI."
+ pattern: "^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\\
+ .]*?\\.apps\\.astra\\.datastax\\.com"
examples:
- - "https://my-instance.zone.zillizcloud.com"
- - "tcp://host.docker.internal:19530"
- - "tcp://my-local-milvus:19530"
+ - "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com"
type: "string"
- db:
- title: "Database Name"
- description: "The database to connect to"
- default: ""
+ astra_db_keyspace:
+ title: "Astra DB Keyspace"
+ description: "Keyspaces (or Namespaces) serve as containers for organizing\
+ \ data within a database. You can create a new keyspace uisng the\
+ \ Data Explorer tab in the Astra UI. The keyspace default_keyspace\
+ \ is created for you when you create a Vector Database in Astra DB."
type: "string"
collection:
- title: "Collection Name"
- description: "The collection to load data into"
- order: 3
- type: "string"
- auth:
- title: "Authentication"
- description: "Authentication method"
- type: "object"
- order: 2
- oneOf:
- - title: "API Token"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "token"
- const: "token"
- enum:
- - "token"
- type: "string"
- token:
- title: "API Token"
- description: "API Token for the Milvus instance"
- airbyte_secret: true
- type: "string"
- x-speakeasy-param-sensitive: true
- required:
- - "token"
- - "mode"
- description: "Authenticate using an API token (suitable for Zilliz\
- \ Cloud)"
- - title: "Username/Password"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "username_password"
- const: "username_password"
- enum:
- - "username_password"
- type: "string"
- username:
- title: "Username"
- description: "Username for the Milvus instance"
- order: 1
- type: "string"
- password:
- title: "Password"
- description: "Password for the Milvus instance"
- airbyte_secret: true
- order: 2
- type: "string"
- x-speakeasy-param-sensitive: true
- required:
- - "username"
- - "password"
- - "mode"
- description: "Authenticate using username and password (suitable for\
- \ self-managed Milvus clusters)"
- - title: "No auth"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "no_auth"
- const: "no_auth"
- enum:
- - "no_auth"
- type: "string"
- description: "Do not authenticate (suitable for locally running test\
- \ clusters, do not use for clusters with public IP addresses)"
- required:
- - "mode"
- vector_field:
- title: "Vector Field"
- description: "The field in the entity that contains the vector"
- default: "vector"
- type: "string"
- text_field:
- title: "Text Field"
- description: "The field in the entity that contains the embedded text"
- default: "text"
+ title: "Astra DB collection"
+ description: "Collections hold data. They are analagous to tables in\
+ \ traditional Cassandra terminology. This tool will create the collection\
+ \ with the provided name automatically if it does not already exist.\
+ \ Alternatively, you can create one thorugh the Data Explorer tab\
+ \ in the Astra UI."
type: "string"
required:
- - "host"
+ - "astra_db_app_token"
+ - "astra_db_endpoint"
+ - "astra_db_keyspace"
- "collection"
- - "auth"
+ description: "Astra DB gives developers the APIs, real-time data and ecosystem\
+ \ integrations to put accurate RAG and Gen AI apps with fewer hallucinations\
+ \ in production."
group: "indexing"
- description: "Indexing configuration"
- destinationType:
- title: "milvus"
- const: "milvus"
- enum:
- - "milvus"
- order: 0
- type: "string"
required:
- "embedding"
- "processing"
- "indexing"
- - "destinationType"
groups:
- id: "processing"
title: "Processing"
@@ -67552,7 +69857,348 @@ components:
title: "Indexing"
- id: "advanced"
title: "Advanced"
- destination-milvus-update:
+ destination-teradata:
+ title: "Teradata Destination Spec"
+ type: "object"
+ required:
+ - "host"
+ - "username"
+ - "destinationType"
+ properties:
+ host:
+ title: "Host"
+ description: "Hostname of the database."
+ type: "string"
+ order: 0
+ username:
+ title: "User"
+ description: "Username to use to access the database."
+ type: "string"
+ order: 1
+ password:
+ title: "Password"
+ description: "Password associated with the username."
+ type: "string"
+ airbyte_secret: true
+ order: 2
+ x-speakeasy-param-sensitive: true
+ schema:
+ title: "Default Schema"
+ description: "The default schema tables are written to if the source does\
+ \ not specify a namespace. The usual value for this field is \"public\"\
+ ."
+ type: "string"
+ examples:
+ - "airbyte_td"
+ default: "airbyte_td"
+ order: 3
+ ssl:
+ title: "SSL Connection"
+ description: "Encrypt data using SSL. When activating SSL, please select\
+ \ one of the connection modes."
+ type: "boolean"
+ default: false
+ order: 5
+ ssl_mode:
+ title: "SSL modes"
+ description: "SSL connection modes. \n disable - Chose this mode\
+ \ to disable encryption of communication between Airbyte and destination\
+ \ database\n allow - Chose this mode to enable encryption only\
+ \ when required by the destination database\n prefer - Chose this\
+ \ mode to allow unencrypted connection only if the destination database\
+ \ does not support encryption\n require - Chose this mode to always\
+ \ require encryption. If the destination database server does not support\
+ \ encryption, connection will fail\n verify-ca - Chose this mode\
+ \ to always require encryption and to verify that the destination database\
+ \ server has a valid SSL certificate\n verify-full - This is the\
+ \ most secure mode. Chose this mode to always require encryption and to\
+ \ verify the identity of the destination database server\n See more information\
+ \ - in the docs."
+ type: "object"
+ order: 6
+ oneOf:
+ - title: "disable"
+ additionalProperties: false
+ description: "Disable SSL."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "disable"
+ enum:
+ - "disable"
+ default: "disable"
+ order: 0
+ - title: "allow"
+ additionalProperties: false
+ description: "Allow SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "allow"
+ enum:
+ - "allow"
+ default: "allow"
+ order: 0
+ - title: "prefer"
+ additionalProperties: false
+ description: "Prefer SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "prefer"
+ enum:
+ - "prefer"
+ default: "prefer"
+ order: 0
+ - title: "require"
+ additionalProperties: false
+ description: "Require SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "require"
+ enum:
+ - "require"
+ default: "require"
+ order: 0
+ - title: "verify-ca"
+ additionalProperties: false
+ description: "Verify-ca SSL mode."
+ required:
+ - "mode"
+ - "ssl_ca_certificate"
+ properties:
+ mode:
+ type: "string"
+ const: "verify-ca"
+ enum:
+ - "verify-ca"
+ default: "verify-ca"
+ order: 0
+ ssl_ca_certificate:
+ type: "string"
+ title: "CA certificate"
+ description: "Specifies the file name of a PEM file that contains\
+ \ Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n\
+ \ See more information - in the docs."
+ airbyte_secret: true
+ multiline: true
+ order: 1
+ - title: "verify-full"
+ additionalProperties: false
+ description: "Verify-full SSL mode."
+ required:
+ - "mode"
+ - "ssl_ca_certificate"
+ properties:
+ mode:
+ type: "string"
+ const: "verify-full"
+ enum:
+ - "verify-full"
+ default: "verify-full"
+ order: 0
+ ssl_ca_certificate:
+ type: "string"
+ title: "CA certificate"
+ description: "Specifies the file name of a PEM file that contains\
+ \ Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n\
+ \ See more information - in the docs."
+ airbyte_secret: true
+ multiline: true
+ order: 1
+ jdbc_url_params:
+ description: "Additional properties to pass to the JDBC URL string when\
+ \ connecting to the database formatted as 'key=value' pairs separated\
+ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+ title: "JDBC URL Params"
+ type: "string"
+ order: 7
+ destinationType:
+ title: "teradata"
+ const: "teradata"
+ enum:
+ - "teradata"
+ order: 0
+ type: "string"
+ destination-teradata-update:
+ title: "Teradata Destination Spec"
+ type: "object"
+ required:
+ - "host"
+ - "username"
+ properties:
+ host:
+ title: "Host"
+ description: "Hostname of the database."
+ type: "string"
+ order: 0
+ username:
+ title: "User"
+ description: "Username to use to access the database."
+ type: "string"
+ order: 1
+ password:
+ title: "Password"
+ description: "Password associated with the username."
+ type: "string"
+ airbyte_secret: true
+ order: 2
+ schema:
+ title: "Default Schema"
+ description: "The default schema tables are written to if the source does\
+ \ not specify a namespace. The usual value for this field is \"public\"\
+ ."
+ type: "string"
+ examples:
+ - "airbyte_td"
+ default: "airbyte_td"
+ order: 3
+ ssl:
+ title: "SSL Connection"
+ description: "Encrypt data using SSL. When activating SSL, please select\
+ \ one of the connection modes."
+ type: "boolean"
+ default: false
+ order: 5
+ ssl_mode:
+ title: "SSL modes"
+ description: "SSL connection modes. \n disable - Chose this mode\
+ \ to disable encryption of communication between Airbyte and destination\
+ \ database\n allow - Chose this mode to enable encryption only\
+ \ when required by the destination database\n prefer - Chose this\
+ \ mode to allow unencrypted connection only if the destination database\
+ \ does not support encryption\n require - Chose this mode to always\
+ \ require encryption. If the destination database server does not support\
+ \ encryption, connection will fail\n verify-ca - Chose this mode\
+ \ to always require encryption and to verify that the destination database\
+ \ server has a valid SSL certificate\n verify-full - This is the\
+ \ most secure mode. Chose this mode to always require encryption and to\
+ \ verify the identity of the destination database server\n See more information\
+ \ - in the docs."
+ type: "object"
+ order: 6
+ oneOf:
+ - title: "disable"
+ additionalProperties: false
+ description: "Disable SSL."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "disable"
+ enum:
+ - "disable"
+ default: "disable"
+ order: 0
+ - title: "allow"
+ additionalProperties: false
+ description: "Allow SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "allow"
+ enum:
+ - "allow"
+ default: "allow"
+ order: 0
+ - title: "prefer"
+ additionalProperties: false
+ description: "Prefer SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "prefer"
+ enum:
+ - "prefer"
+ default: "prefer"
+ order: 0
+ - title: "require"
+ additionalProperties: false
+ description: "Require SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "require"
+ enum:
+ - "require"
+ default: "require"
+ order: 0
+ - title: "verify-ca"
+ additionalProperties: false
+ description: "Verify-ca SSL mode."
+ required:
+ - "mode"
+ - "ssl_ca_certificate"
+ properties:
+ mode:
+ type: "string"
+ const: "verify-ca"
+ enum:
+ - "verify-ca"
+ default: "verify-ca"
+ order: 0
+ ssl_ca_certificate:
+ type: "string"
+ title: "CA certificate"
+ description: "Specifies the file name of a PEM file that contains\
+ \ Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n\
+ \ See more information - in the docs."
+ airbyte_secret: true
+ multiline: true
+ order: 1
+ - title: "verify-full"
+ additionalProperties: false
+ description: "Verify-full SSL mode."
+ required:
+ - "mode"
+ - "ssl_ca_certificate"
+ properties:
+ mode:
+ type: "string"
+ const: "verify-full"
+ enum:
+ - "verify-full"
+ default: "verify-full"
+ order: 0
+ ssl_ca_certificate:
+ type: "string"
+ title: "CA certificate"
+ description: "Specifies the file name of a PEM file that contains\
+ \ Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n\
+ \ See more information - in the docs."
+ airbyte_secret: true
+ multiline: true
+ order: 1
+ jdbc_url_params:
+ description: "Additional properties to pass to the JDBC URL string when\
+ \ connecting to the database formatted as 'key=value' pairs separated\
+ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+ title: "JDBC URL Params"
+ type: "string"
+ order: 7
+ destination-pinecone:
title: "Destination Config"
description: "The configuration model for the Vector DB based destinations.\
\ This model is used to generate the UI for the destination configuration,\n\
@@ -67584,6 +70230,7 @@ components:
title: "OpenAI API key"
airbyte_secret: true
type: "string"
+ x-speakeasy-param-sensitive: true
required:
- "openai_key"
- "mode"
@@ -67603,6 +70250,7 @@ components:
title: "Cohere API key"
airbyte_secret: true
type: "string"
+ x-speakeasy-param-sensitive: true
required:
- "cohere_key"
- "mode"
@@ -67638,6 +70286,7 @@ components:
\ find this in the Azure portal under your Azure OpenAI resource"
airbyte_secret: true
type: "string"
+ x-speakeasy-param-sensitive: true
api_base:
title: "Resource base URL"
description: "The base URL for your Azure OpenAI resource. You can\
@@ -67674,6 +70323,7 @@ components:
default: ""
airbyte_secret: true
type: "string"
+ x-speakeasy-param-sensitive: true
base_url:
title: "Base URL"
description: "The base URL for your OpenAI-compatible service"
@@ -67896,112 +70546,43 @@ components:
title: "Indexing"
type: "object"
properties:
- host:
- title: "Public Endpoint"
- description: "The public endpoint of the Milvus instance. "
- order: 1
- examples:
- - "https://my-instance.zone.zillizcloud.com"
- - "tcp://host.docker.internal:19530"
- - "tcp://my-local-milvus:19530"
- type: "string"
- db:
- title: "Database Name"
- description: "The database to connect to"
- default: ""
- type: "string"
- collection:
- title: "Collection Name"
- description: "The collection to load data into"
- order: 3
+ pinecone_key:
+ title: "Pinecone API key"
+ description: "The Pinecone API key to use matching the environment (copy\
+ \ from Pinecone console)"
+ airbyte_secret: true
type: "string"
- auth:
- title: "Authentication"
- description: "Authentication method"
- type: "object"
- order: 2
- oneOf:
- - title: "API Token"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "token"
- const: "token"
- enum:
- - "token"
- type: "string"
- token:
- title: "API Token"
- description: "API Token for the Milvus instance"
- airbyte_secret: true
- type: "string"
- required:
- - "token"
- - "mode"
- description: "Authenticate using an API token (suitable for Zilliz\
- \ Cloud)"
- - title: "Username/Password"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "username_password"
- const: "username_password"
- enum:
- - "username_password"
- type: "string"
- username:
- title: "Username"
- description: "Username for the Milvus instance"
- order: 1
- type: "string"
- password:
- title: "Password"
- description: "Password for the Milvus instance"
- airbyte_secret: true
- order: 2
- type: "string"
- required:
- - "username"
- - "password"
- - "mode"
- description: "Authenticate using username and password (suitable for\
- \ self-managed Milvus clusters)"
- - title: "No auth"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "no_auth"
- const: "no_auth"
- enum:
- - "no_auth"
- type: "string"
- description: "Do not authenticate (suitable for locally running test\
- \ clusters, do not use for clusters with public IP addresses)"
- required:
- - "mode"
- vector_field:
- title: "Vector Field"
- description: "The field in the entity that contains the vector"
- default: "vector"
+ x-speakeasy-param-sensitive: true
+ pinecone_environment:
+ title: "Pinecone Environment"
+ description: "Pinecone Cloud environment to use"
+ examples:
+ - "us-west1-gcp"
+ - "gcp-starter"
type: "string"
- text_field:
- title: "Text Field"
- description: "The field in the entity that contains the embedded text"
- default: "text"
+ index:
+ title: "Index"
+ description: "Pinecone index in your project to load data into"
type: "string"
required:
- - "host"
- - "collection"
- - "auth"
+ - "pinecone_key"
+ - "pinecone_environment"
+ - "index"
+ description: "Pinecone is a popular vector store that can be used to store\
+ \ and retrieve embeddings."
group: "indexing"
- description: "Indexing configuration"
+ destinationType:
+ title: "pinecone"
+ const: "pinecone"
+ enum:
+ - "pinecone"
+ order: 0
+ type: "string"
required:
- "embedding"
- "processing"
- "indexing"
+ - "destinationType"
groups:
- id: "processing"
title: "Processing"
@@ -68011,94 +70592,7 @@ components:
title: "Indexing"
- id: "advanced"
title: "Advanced"
- destination-google-sheets:
- title: "Destination Google Sheets"
- type: "object"
- required:
- - "spreadsheet_id"
- - "credentials"
- - "destinationType"
- properties:
- spreadsheet_id:
- type: "string"
- title: "Spreadsheet Link"
- description: "The link to your spreadsheet. See this\
- \ guide for more details."
- examples:
- - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG/edit"
- credentials:
- type: "object"
- title: "Authentication via Google (OAuth)"
- description: "Google API Credentials for connecting to Google Sheets and\
- \ Google Drive APIs"
- required:
- - "client_id"
- - "client_secret"
- - "refresh_token"
- properties:
- client_id:
- title: "Client ID"
- type: "string"
- description: "The Client ID of your Google Sheets developer application."
- airbyte_secret: true
- client_secret:
- title: "Client Secret"
- type: "string"
- description: "The Client Secret of your Google Sheets developer application."
- airbyte_secret: true
- refresh_token:
- title: "Refresh Token"
- type: "string"
- description: "The token for obtaining new access token."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- destinationType:
- title: "google-sheets"
- const: "google-sheets"
- enum:
- - "google-sheets"
- order: 0
- type: "string"
- destination-google-sheets-update:
- title: "Destination Google Sheets"
- type: "object"
- required:
- - "spreadsheet_id"
- - "credentials"
- properties:
- spreadsheet_id:
- type: "string"
- title: "Spreadsheet Link"
- description: "The link to your spreadsheet. See this\
- \ guide for more details."
- examples:
- - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG/edit"
- credentials:
- type: "object"
- title: "Authentication via Google (OAuth)"
- description: "Google API Credentials for connecting to Google Sheets and\
- \ Google Drive APIs"
- required:
- - "client_id"
- - "client_secret"
- - "refresh_token"
- properties:
- client_id:
- title: "Client ID"
- type: "string"
- description: "The Client ID of your Google Sheets developer application."
- airbyte_secret: true
- client_secret:
- title: "Client Secret"
- type: "string"
- description: "The Client Secret of your Google Sheets developer application."
- airbyte_secret: true
- refresh_token:
- title: "Refresh Token"
- type: "string"
- description: "The token for obtaining new access token."
- airbyte_secret: true
- destination-astra:
+ destination-pinecone-update:
title: "Destination Config"
description: "The configuration model for the Vector DB based destinations.\
\ This model is used to generate the UI for the destination configuration,\n\
@@ -68130,7 +70624,6 @@ components:
title: "OpenAI API key"
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
required:
- "openai_key"
- "mode"
@@ -68150,7 +70643,6 @@ components:
title: "Cohere API key"
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
required:
- "cohere_key"
- "mode"
@@ -68186,7 +70678,6 @@ components:
\ find this in the Azure portal under your Azure OpenAI resource"
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
api_base:
title: "Resource base URL"
description: "The base URL for your Azure OpenAI resource. You can\
@@ -68223,7 +70714,6 @@ components:
default: ""
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
base_url:
title: "Base URL"
description: "The base URL for your OpenAI-compatible service"
@@ -68446,701 +70936,1030 @@ components:
title: "Indexing"
type: "object"
properties:
- astra_db_app_token:
- title: "Astra DB Application Token"
- description: "The application token authorizes a user to connect to\
- \ a specific Astra DB database. It is created when the user clicks\
- \ the Generate Token button on the Overview tab of the Database page\
- \ in the Astra UI."
+ pinecone_key:
+ title: "Pinecone API key"
+ description: "The Pinecone API key to use matching the environment (copy\
+ \ from Pinecone console)"
airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
- astra_db_endpoint:
- title: "Astra DB Endpoint"
- description: "The endpoint specifies which Astra DB database queries\
- \ are sent to. It can be copied from the Database Details section\
- \ of the Overview tab of the Database page in the Astra UI."
- pattern: "^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\\
- .]*?\\.apps\\.astra\\.datastax\\.com"
+ pinecone_environment:
+ title: "Pinecone Environment"
+ description: "Pinecone Cloud environment to use"
examples:
- - "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com"
- type: "string"
- astra_db_keyspace:
- title: "Astra DB Keyspace"
- description: "Keyspaces (or Namespaces) serve as containers for organizing\
- \ data within a database. You can create a new keyspace uisng the\
- \ Data Explorer tab in the Astra UI. The keyspace default_keyspace\
- \ is created for you when you create a Vector Database in Astra DB."
+ - "us-west1-gcp"
+ - "gcp-starter"
type: "string"
- x-speakeasy-param-sensitive: true
- collection:
- title: "Astra DB collection"
- description: "Collections hold data. They are analagous to tables in\
- \ traditional Cassandra terminology. This tool will create the collection\
- \ with the provided name automatically if it does not already exist.\
- \ Alternatively, you can create one thorugh the Data Explorer tab\
- \ in the Astra UI."
+ index:
+ title: "Index"
+ description: "Pinecone index in your project to load data into"
type: "string"
required:
- - "astra_db_app_token"
- - "astra_db_endpoint"
- - "astra_db_keyspace"
- - "collection"
- description: "Astra DB gives developers the APIs, real-time data and ecosystem\
- \ integrations to put accurate RAG and Gen AI apps with fewer hallucinations\
- \ in production."
+ - "pinecone_key"
+ - "pinecone_environment"
+ - "index"
+ description: "Pinecone is a popular vector store that can be used to store\
+ \ and retrieve embeddings."
group: "indexing"
+ required:
+ - "embedding"
+ - "processing"
+ - "indexing"
+ groups:
+ - id: "processing"
+ title: "Processing"
+ - id: "embedding"
+ title: "Embedding"
+ - id: "indexing"
+ title: "Indexing"
+ - id: "advanced"
+ title: "Advanced"
+ destination-duckdb:
+ title: "Destination Duckdb"
+ type: "object"
+ required:
+ - "destination_path"
+ - "destinationType"
+ properties:
+ motherduck_api_key:
+ title: "MotherDuck API Key"
+ type: "string"
+ description: "API key to use for authentication to a MotherDuck database."
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
+ destination_path:
+ title: "Destination DB"
+ type: "string"
+ description: "Path to the .duckdb file, or the text 'md:' to connect to\
+ \ MotherDuck. The file will be placed inside that local mount. For more\
+ \ information check out our docs"
+ examples:
+ - "/local/destination.duckdb"
+ - "md:"
+ - "motherduck:"
+ schema:
+ title: "Destination Schema"
+ type: "string"
+ description: "Database schema name, default for duckdb is 'main'."
+ example: "main"
+ destinationType:
+ title: "duckdb"
+ const: "duckdb"
+ enum:
+ - "duckdb"
+ order: 0
+ type: "string"
+ destination-duckdb-update:
+ title: "Destination Duckdb"
+ type: "object"
+ required:
+ - "destination_path"
+ properties:
+ motherduck_api_key:
+ title: "MotherDuck API Key"
+ type: "string"
+ description: "API key to use for authentication to a MotherDuck database."
+ airbyte_secret: true
+ destination_path:
+ title: "Destination DB"
+ type: "string"
+ description: "Path to the .duckdb file, or the text 'md:' to connect to\
+ \ MotherDuck. The file will be placed inside that local mount. For more\
+ \ information check out our docs"
+ examples:
+ - "/local/destination.duckdb"
+ - "md:"
+ - "motherduck:"
+ schema:
+ title: "Destination Schema"
+ type: "string"
+ description: "Database schema name, default for duckdb is 'main'."
+ example: "main"
+ destination-sftp-json:
+ title: "Destination SFTP JSON"
+ type: "object"
+ required:
+ - "host"
+ - "username"
+ - "password"
+ - "destination_path"
+ - "destinationType"
+ properties:
+ host:
+ title: "Host"
+ description: "Hostname of the SFTP server."
+ type: "string"
+ order: 0
+ port:
+ title: "Port"
+ description: "Port of the SFTP server."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - 22
+ order: 1
+ username:
+ title: "User"
+ description: "Username to use to access the SFTP server."
+ type: "string"
+ order: 2
+ password:
+ title: "Password"
+ description: "Password associated with the username."
+ type: "string"
+ airbyte_secret: true
+ order: 3
+ x-speakeasy-param-sensitive: true
+ destination_path:
+ title: "Destination path"
+ type: "string"
+ description: "Path to the directory where json files will be written."
+ examples:
+ - "/json_data"
+ order: 4
destinationType:
- title: "astra"
- const: "astra"
+ title: "sftp-json"
+ const: "sftp-json"
enum:
- - "astra"
+ - "sftp-json"
order: 0
type: "string"
+ destination-sftp-json-update:
+ title: "Destination SFTP JSON"
+ type: "object"
required:
- - "embedding"
- - "processing"
- - "indexing"
- - "destinationType"
- groups:
- - id: "processing"
- title: "Processing"
- - id: "embedding"
- title: "Embedding"
- - id: "indexing"
- title: "Indexing"
- - id: "advanced"
- title: "Advanced"
- destination-astra-update:
- title: "Destination Config"
- description: "The configuration model for the Vector DB based destinations.\
- \ This model is used to generate the UI for the destination configuration,\n\
- as well as to provide type safety for the configuration passed to the destination.\n\
- \nThe configuration model is composed of four parts:\n* Processing configuration\n\
- * Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\
- \nProcessing, embedding and advanced configuration are provided by this base\
- \ class, while the indexing configuration is provided by the destination connector\
- \ in the sub class."
+ - "host"
+ - "username"
+ - "password"
+ - "destination_path"
+ properties:
+ host:
+ title: "Host"
+ description: "Hostname of the SFTP server."
+ type: "string"
+ order: 0
+ port:
+ title: "Port"
+ description: "Port of the SFTP server."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - 22
+ order: 1
+ username:
+ title: "User"
+ description: "Username to use to access the SFTP server."
+ type: "string"
+ order: 2
+ password:
+ title: "Password"
+ description: "Password associated with the username."
+ type: "string"
+ airbyte_secret: true
+ order: 3
+ destination_path:
+ title: "Destination path"
+ type: "string"
+ description: "Path to the directory where json files will be written."
+ examples:
+ - "/json_data"
+ order: 4
+ destination-s3:
+ title: "S3 Destination Spec"
type: "object"
+ required:
+ - "s3_bucket_name"
+ - "s3_bucket_path"
+ - "s3_bucket_region"
+ - "format"
+ - "destinationType"
properties:
- embedding:
- title: "Embedding"
- description: "Embedding configuration"
- group: "embedding"
+ access_key_id:
+ type: "string"
+ description: "The access key ID to access the S3 bucket. Airbyte requires\
+ \ Read and Write permissions to the given bucket. Read more here."
+ title: "S3 Key ID"
+ airbyte_secret: true
+ always_show: true
+ examples:
+ - "A012345678910EXAMPLE"
+ order: 0
+ x-speakeasy-param-sensitive: true
+ secret_access_key:
+ type: "string"
+ description: "The corresponding secret to the access key ID. Read more here"
+ title: "S3 Access Key"
+ airbyte_secret: true
+ always_show: true
+ examples:
+ - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
+ order: 1
+ x-speakeasy-param-sensitive: true
+ s3_bucket_name:
+ title: "S3 Bucket Name"
+ type: "string"
+ description: "The name of the S3 bucket. Read more here."
+ examples:
+ - "airbyte_sync"
+ order: 2
+ s3_bucket_path:
+ title: "S3 Bucket Path"
+ description: "Directory under the S3 bucket where data will be written.\
+ \ Read more here"
+ type: "string"
+ examples:
+ - "data_sync/test"
+ order: 3
+ s3_bucket_region:
+ title: "S3 Bucket Region"
+ type: "string"
+ default: ""
+ description: "The region of the S3 bucket. See here for all region codes."
+ enum:
+ - ""
+ - "af-south-1"
+ - "ap-east-1"
+ - "ap-northeast-1"
+ - "ap-northeast-2"
+ - "ap-northeast-3"
+ - "ap-south-1"
+ - "ap-south-2"
+ - "ap-southeast-1"
+ - "ap-southeast-2"
+ - "ap-southeast-3"
+ - "ap-southeast-4"
+ - "ca-central-1"
+ - "ca-west-1"
+ - "cn-north-1"
+ - "cn-northwest-1"
+ - "eu-central-1"
+ - "eu-central-2"
+ - "eu-north-1"
+ - "eu-south-1"
+ - "eu-south-2"
+ - "eu-west-1"
+ - "eu-west-2"
+ - "eu-west-3"
+ - "il-central-1"
+ - "me-central-1"
+ - "me-south-1"
+ - "sa-east-1"
+ - "us-east-1"
+ - "us-east-2"
+ - "us-gov-east-1"
+ - "us-gov-west-1"
+ - "us-west-1"
+ - "us-west-2"
+ order: 4
+ format:
+ title: "Output Format"
type: "object"
+ description: "Format of the data output. See here for more details"
oneOf:
- - title: "OpenAI"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "openai"
- const: "openai"
- enum:
- - "openai"
- type: "string"
- openai_key:
- title: "OpenAI API key"
- airbyte_secret: true
- type: "string"
+ - title: "CSV: Comma-Separated Values"
required:
- - "openai_key"
- - "mode"
- description: "Use the OpenAI API to embed text. This option is using the\
- \ text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "Cohere"
- type: "object"
+ - "format_type"
+ - "flattening"
properties:
- mode:
- title: "Mode"
- default: "cohere"
- const: "cohere"
- enum:
- - "cohere"
+ format_type:
+ title: "Format Type"
type: "string"
- cohere_key:
- title: "Cohere API key"
- airbyte_secret: true
+ enum:
+ - "CSV"
+ default: "CSV"
+ flattening:
type: "string"
+ title: "Flattening"
+ description: "Whether the input json data should be normalized (flattened)\
+ \ in the output CSV. Please refer to docs for details."
+ default: "No flattening"
+ enum:
+ - "No flattening"
+ - "Root level flattening"
+ compression:
+ title: "Compression"
+ type: "object"
+ description: "Whether the output files should be compressed. If compression\
+ \ is selected, the output filename will have an extra extension\
+ \ (GZIP: \".csv.gz\")."
+ oneOf:
+ - title: "No Compression"
+ requires:
+ - "compression_type"
+ properties:
+ compression_type:
+ type: "string"
+ enum:
+ - "No Compression"
+ default: "No Compression"
+ - title: "GZIP"
+ requires:
+ - "compression_type"
+ properties:
+ compression_type:
+ type: "string"
+ enum:
+ - "GZIP"
+ default: "GZIP"
+ - title: "JSON Lines: Newline-delimited JSON"
required:
- - "cohere_key"
- - "mode"
- description: "Use the Cohere API to embed text."
- - title: "Fake"
- type: "object"
+ - "format_type"
properties:
- mode:
- title: "Mode"
- default: "fake"
- const: "fake"
+ format_type:
+ title: "Format Type"
+ type: "string"
enum:
- - "fake"
+ - "JSONL"
+ default: "JSONL"
+ flattening:
type: "string"
- description: "Use a fake embedding made out of random vectors with 1536\
- \ embedding dimensions. This is useful for testing the data pipeline\
- \ without incurring any costs."
+ title: "Flattening"
+ description: "Whether the input json data should be normalized (flattened)\
+ \ in the output JSON Lines. Please refer to docs for details."
+ default: "No flattening"
+ enum:
+ - "No flattening"
+ - "Root level flattening"
+ compression:
+ title: "Compression"
+ type: "object"
+ description: "Whether the output files should be compressed. If compression\
+ \ is selected, the output filename will have an extra extension\
+ \ (GZIP: \".jsonl.gz\")."
+ oneOf:
+ - title: "No Compression"
+ requires: "compression_type"
+ properties:
+ compression_type:
+ type: "string"
+ enum:
+ - "No Compression"
+ default: "No Compression"
+ - title: "GZIP"
+ requires: "compression_type"
+ properties:
+ compression_type:
+ type: "string"
+ enum:
+ - "GZIP"
+ default: "GZIP"
+ - title: "Avro: Apache Avro"
required:
- - "mode"
- - title: "Azure OpenAI"
- type: "object"
+ - "format_type"
+ - "compression_codec"
properties:
- mode:
- title: "Mode"
- default: "azure_openai"
- const: "azure_openai"
- enum:
- - "azure_openai"
- type: "string"
- openai_key:
- title: "Azure OpenAI API key"
- description: "The API key for your Azure OpenAI resource. You can\
- \ find this in the Azure portal under your Azure OpenAI resource"
- airbyte_secret: true
- type: "string"
- api_base:
- title: "Resource base URL"
- description: "The base URL for your Azure OpenAI resource. You can\
- \ find this in the Azure portal under your Azure OpenAI resource"
- examples:
- - "https://your-resource-name.openai.azure.com"
- type: "string"
- deployment:
- title: "Deployment"
- description: "The deployment for your Azure OpenAI resource. You\
- \ can find this in the Azure portal under your Azure OpenAI resource"
- examples:
- - "your-resource-name"
+ format_type:
+ title: "Format Type"
type: "string"
+ enum:
+ - "Avro"
+ default: "Avro"
+ order: 0
+ compression_codec:
+ title: "Compression Codec"
+ description: "The compression algorithm used to compress data. Default\
+ \ to no compression."
+ type: "object"
+ oneOf:
+ - title: "No Compression"
+ required:
+ - "codec"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "no compression"
+ default: "no compression"
+ - title: "Deflate"
+ required:
+ - "codec"
+ - "compression_level"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "Deflate"
+ default: "Deflate"
+ compression_level:
+ title: "Deflate Level"
+ description: "0: no compression & fastest, 9: best compression\
+ \ & slowest."
+ type: "integer"
+ default: 0
+ minimum: 0
+ maximum: 9
+ - title: "bzip2"
+ required:
+ - "codec"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "bzip2"
+ default: "bzip2"
+ - title: "xz"
+ required:
+ - "codec"
+ - "compression_level"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "xz"
+ default: "xz"
+ compression_level:
+ title: "Compression Level"
+ description: "See here for details."
+ type: "integer"
+ default: 6
+ minimum: 0
+ maximum: 9
+ - title: "zstandard"
+ required:
+ - "codec"
+ - "compression_level"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "zstandard"
+ default: "zstandard"
+ compression_level:
+ title: "Compression Level"
+ description: "Negative levels are 'fast' modes akin to lz4 or\
+ \ snappy, levels above 9 are generally for archival purposes,\
+ \ and levels above 18 use a lot of memory."
+ type: "integer"
+ default: 3
+ minimum: -5
+ maximum: 22
+ include_checksum:
+ title: "Include Checksum"
+ description: "If true, include a checksum with each data block."
+ type: "boolean"
+ default: false
+ - title: "snappy"
+ required:
+ - "codec"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "snappy"
+ default: "snappy"
+ order: 1
+ - title: "Parquet: Columnar Storage"
required:
- - "openai_key"
- - "api_base"
- - "deployment"
- - "mode"
- description: "Use the Azure-hosted OpenAI API to embed text. This option\
- \ is using the text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "OpenAI-compatible"
- type: "object"
+ - "format_type"
properties:
- mode:
- title: "Mode"
- default: "openai_compatible"
- const: "openai_compatible"
- enum:
- - "openai_compatible"
- type: "string"
- api_key:
- title: "API key"
- default: ""
- airbyte_secret: true
+ format_type:
+ title: "Format Type"
type: "string"
- base_url:
- title: "Base URL"
- description: "The base URL for your OpenAI-compatible service"
- examples:
- - "https://your-service-name.com"
+ enum:
+ - "Parquet"
+ default: "Parquet"
+ compression_codec:
+ title: "Compression Codec"
+ description: "The compression algorithm used to compress data pages."
type: "string"
- model_name:
- title: "Model name"
- description: "The name of the model to use for embedding"
- default: "text-embedding-ada-002"
+ enum:
+ - "UNCOMPRESSED"
+ - "SNAPPY"
+ - "GZIP"
+ - "LZO"
+ - "BROTLI"
+ - "LZ4"
+ - "ZSTD"
+ default: "UNCOMPRESSED"
+ block_size_mb:
+ title: "Block Size (Row Group Size) (MB)"
+ description: "This is the size of a row group being buffered in memory.\
+ \ It limits the memory usage when writing. Larger values will improve\
+ \ the IO when reading, but consume more memory when writing. Default:\
+ \ 128 MB."
+ type: "integer"
+ default: 128
examples:
- - "text-embedding-ada-002"
- type: "string"
- dimensions:
- title: "Embedding dimensions"
- description: "The number of dimensions the embedding model is generating"
+ - 128
+ max_padding_size_mb:
+ title: "Max Padding Size (MB)"
+ description: "Maximum size allowed as padding to align row groups.\
+ \ This is also the minimum size of a row group. Default: 8 MB."
+ type: "integer"
+ default: 8
examples:
- - 1536
- - 384
+ - 8
+ page_size_kb:
+ title: "Page Size (KB)"
+ description: "The page size is for compression. A block is composed\
+ \ of pages. A page is the smallest unit that must be read fully\
+ \ to access a single record. If this value is too small, the compression\
+ \ will deteriorate. Default: 1024 KB."
type: "integer"
- required:
- - "base_url"
- - "dimensions"
- - "mode"
- description: "Use a service that's compatible with the OpenAI API to embed\
- \ text."
- processing:
- title: "ProcessingConfigModel"
- type: "object"
- properties:
- chunk_size:
- title: "Chunk size"
- description: "Size of chunks in tokens to store in vector store (make\
- \ sure it is not too big for the context if your LLM)"
- maximum: 8191
- minimum: 1
- type: "integer"
- chunk_overlap:
- title: "Chunk overlap"
- description: "Size of overlap between chunks in tokens to store in vector\
- \ store to better capture relevant context"
- default: 0
- type: "integer"
- text_fields:
- title: "Text fields to embed"
- description: "List of fields in the record that should be used to calculate\
- \ the embedding. The field list is applied to all streams in the same\
- \ way and non-existing fields are ignored. If none are defined, all\
- \ fields are considered text fields. When specifying text fields,\
- \ you can access nested fields in the record by using dot notation,\
- \ e.g. `user.name` will access the `name` field in the `user` object.\
- \ It's also possible to use wildcards to access all fields in an object,\
- \ e.g. `users.*.name` will access all `names` fields in all entries\
- \ of the `users` array."
- default: []
- always_show: true
- examples:
- - "text"
- - "user.name"
- - "users.*.name"
- type: "array"
- items:
- type: "string"
- metadata_fields:
- title: "Fields to store as metadata"
- description: "List of fields in the record that should be stored as\
- \ metadata. The field list is applied to all streams in the same way\
- \ and non-existing fields are ignored. If none are defined, all fields\
- \ are considered metadata fields. When specifying text fields, you\
- \ can access nested fields in the record by using dot notation, e.g.\
- \ `user.name` will access the `name` field in the `user` object. It's\
- \ also possible to use wildcards to access all fields in an object,\
- \ e.g. `users.*.name` will access all `names` fields in all entries\
- \ of the `users` array. When specifying nested paths, all matching\
- \ values are flattened into an array set to a field named by the path."
- default: []
- always_show: true
- examples:
- - "age"
- - "user"
- - "user.name"
- type: "array"
- items:
- type: "string"
- text_splitter:
- title: "Text splitter"
- description: "Split text fields into chunks based on the specified method."
- type: "object"
- oneOf:
- - title: "By Separator"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "separator"
- const: "separator"
- enum:
- - "separator"
- type: "string"
- separators:
- title: "Separators"
- description: "List of separator strings to split text fields by.\
- \ The separator itself needs to be wrapped in double quotes,\
- \ e.g. to split by the dot character, use \".\". To split by\
- \ a newline, use \"\\n\"."
- default:
- - "\"\\n\\n\""
- - "\"\\n\""
- - "\" \""
- - "\"\""
- type: "array"
- items:
- type: "string"
- keep_separator:
- title: "Keep separator"
- description: "Whether to keep the separator in the resulting chunks"
- default: false
- type: "boolean"
- description: "Split the text by the list of separators until the chunk\
- \ size is reached, using the earlier mentioned separators where\
- \ possible. This is useful for splitting text fields by paragraphs,\
- \ sentences, words, etc."
- required:
- - "mode"
- - title: "By Markdown header"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "markdown"
- const: "markdown"
- enum:
- - "markdown"
- type: "string"
- split_level:
- title: "Split level"
- description: "Level of markdown headers to split text fields by.\
- \ Headings down to the specified level will be used as split\
- \ points"
- default: 1
- minimum: 1
- maximum: 6
- type: "integer"
- description: "Split the text by Markdown headers down to the specified\
- \ header level. If the chunk size fits multiple sections, they will\
- \ be combined into a single chunk."
- required:
- - "mode"
- - title: "By Programming Language"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "code"
- const: "code"
- enum:
- - "code"
- type: "string"
- language:
- title: "Language"
- description: "Split code in suitable places based on the programming\
- \ language"
- enum:
- - "cpp"
- - "go"
- - "java"
- - "js"
- - "php"
- - "proto"
- - "python"
- - "rst"
- - "ruby"
- - "rust"
- - "scala"
- - "swift"
- - "markdown"
- - "latex"
- - "html"
- - "sol"
- type: "string"
- required:
- - "language"
- - "mode"
- description: "Split the text by suitable delimiters based on the programming\
- \ language. This is useful for splitting code into chunks."
- field_name_mappings:
- title: "Field name mappings"
- description: "List of fields to rename. Not applicable for nested fields,\
- \ but can be used to rename fields already flattened via dot notation."
- default: []
- type: "array"
- items:
- title: "FieldNameMappingConfigModel"
- type: "object"
- properties:
- from_field:
- title: "From field name"
- description: "The field name in the source"
- type: "string"
- to_field:
- title: "To field name"
- description: "The field name to use in the destination"
- type: "string"
- required:
- - "from_field"
- - "to_field"
- required:
- - "chunk_size"
- group: "processing"
- omit_raw_text:
- title: "Do not store raw text"
- description: "Do not store the text that gets embedded along with the vector\
- \ and the metadata in the destination. If set to true, only the vector\
- \ and the metadata will be stored - in this case raw text for LLM use\
- \ cases needs to be retrieved from another source."
- default: false
- group: "advanced"
- type: "boolean"
- indexing:
- title: "Indexing"
- type: "object"
- properties:
- astra_db_app_token:
- title: "Astra DB Application Token"
- description: "The application token authorizes a user to connect to\
- \ a specific Astra DB database. It is created when the user clicks\
- \ the Generate Token button on the Overview tab of the Database page\
- \ in the Astra UI."
- airbyte_secret: true
- type: "string"
- astra_db_endpoint:
- title: "Astra DB Endpoint"
- description: "The endpoint specifies which Astra DB database queries\
- \ are sent to. It can be copied from the Database Details section\
- \ of the Overview tab of the Database page in the Astra UI."
- pattern: "^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\\
- .]*?\\.apps\\.astra\\.datastax\\.com"
- examples:
- - "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com"
- type: "string"
- astra_db_keyspace:
- title: "Astra DB Keyspace"
- description: "Keyspaces (or Namespaces) serve as containers for organizing\
- \ data within a database. You can create a new keyspace uisng the\
- \ Data Explorer tab in the Astra UI. The keyspace default_keyspace\
- \ is created for you when you create a Vector Database in Astra DB."
- type: "string"
- collection:
- title: "Astra DB collection"
- description: "Collections hold data. They are analagous to tables in\
- \ traditional Cassandra terminology. This tool will create the collection\
- \ with the provided name automatically if it does not already exist.\
- \ Alternatively, you can create one thorugh the Data Explorer tab\
- \ in the Astra UI."
- type: "string"
- required:
- - "astra_db_app_token"
- - "astra_db_endpoint"
- - "astra_db_keyspace"
- - "collection"
- description: "Astra DB gives developers the APIs, real-time data and ecosystem\
- \ integrations to put accurate RAG and Gen AI apps with fewer hallucinations\
- \ in production."
- group: "indexing"
- required:
- - "embedding"
- - "processing"
- - "indexing"
- groups:
- - id: "processing"
- title: "Processing"
- - id: "embedding"
- title: "Embedding"
- - id: "indexing"
- title: "Indexing"
- - id: "advanced"
- title: "Advanced"
- destination-teradata:
- title: "Teradata Destination Spec"
+ default: 1024
+ examples:
+ - 1024
+ dictionary_page_size_kb:
+ title: "Dictionary Page Size (KB)"
+ description: "There is one dictionary page per column per row group\
+ \ when dictionary encoding is used. The dictionary page size works\
+ \ like the page size but for dictionary. Default: 1024 KB."
+ type: "integer"
+ default: 1024
+ examples:
+ - 1024
+ dictionary_encoding:
+ title: "Dictionary Encoding"
+ description: "Default: true."
+ type: "boolean"
+ default: true
+ order: 5
+ s3_endpoint:
+ title: "Endpoint"
+ type: "string"
+ default: ""
+ description: "Your S3 endpoint url. Read more here"
+ examples:
+ - "http://localhost:9000"
+ order: 6
+ s3_path_format:
+ title: "S3 Path Format"
+ description: "Format string on how data will be organized inside the S3\
+ \ bucket directory. Read more here"
+ type: "string"
+ examples:
+ - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
+ order: 7
+ file_name_pattern:
+ type: "string"
+ description: "The pattern allows you to set the file-name format for the\
+ \ S3 staging file(s)"
+ title: "S3 Filename pattern"
+ examples:
+ - "{date}"
+ - "{date:yyyy_MM}"
+ - "{timestamp}"
+ - "{part_number}"
+ - "{sync_id}"
+ order: 8
+ destinationType:
+ title: "s3"
+ const: "s3"
+ enum:
+ - "s3"
+ order: 0
+ type: "string"
+ destination-s3-update:
+ title: "S3 Destination Spec"
type: "object"
required:
- - "host"
- - "username"
- - "destinationType"
+ - "s3_bucket_name"
+ - "s3_bucket_path"
+ - "s3_bucket_region"
+ - "format"
properties:
- host:
- title: "Host"
- description: "Hostname of the database."
+ access_key_id:
type: "string"
+ description: "The access key ID to access the S3 bucket. Airbyte requires\
+ \ Read and Write permissions to the given bucket. Read more here."
+ title: "S3 Key ID"
+ airbyte_secret: true
+ always_show: true
+ examples:
+ - "A012345678910EXAMPLE"
order: 0
- username:
- title: "User"
- description: "Username to use to access the database."
+ secret_access_key:
type: "string"
+ description: "The corresponding secret to the access key ID. Read more here"
+ title: "S3 Access Key"
+ airbyte_secret: true
+ always_show: true
+ examples:
+ - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
order: 1
- password:
- title: "Password"
- description: "Password associated with the username."
+ s3_bucket_name:
+ title: "S3 Bucket Name"
type: "string"
- airbyte_secret: true
+ description: "The name of the S3 bucket. Read more here."
+ examples:
+ - "airbyte_sync"
order: 2
- x-speakeasy-param-sensitive: true
- schema:
- title: "Default Schema"
- description: "The default schema tables are written to if the source does\
- \ not specify a namespace. The usual value for this field is \"public\"\
- ."
+ s3_bucket_path:
+ title: "S3 Bucket Path"
+ description: "Directory under the S3 bucket where data will be written.\
+ \ Read more here"
type: "string"
examples:
- - "airbyte_td"
- default: "airbyte_td"
+ - "data_sync/test"
order: 3
- ssl:
- title: "SSL Connection"
- description: "Encrypt data using SSL. When activating SSL, please select\
- \ one of the connection modes."
- type: "boolean"
- default: false
- order: 5
- ssl_mode:
- title: "SSL modes"
- description: "SSL connection modes. \n disable - Chose this mode\
- \ to disable encryption of communication between Airbyte and destination\
- \ database\n allow - Chose this mode to enable encryption only\
- \ when required by the destination database\n prefer - Chose this\
- \ mode to allow unencrypted connection only if the destination database\
- \ does not support encryption\n require - Chose this mode to always\
- \ require encryption. If the destination database server does not support\
- \ encryption, connection will fail\n verify-ca - Chose this mode\
- \ to always require encryption and to verify that the destination database\
- \ server has a valid SSL certificate\n verify-full - This is the\
- \ most secure mode. Chose this mode to always require encryption and to\
- \ verify the identity of the destination database server\n See more information\
- \ - in the docs."
+ s3_bucket_region:
+ title: "S3 Bucket Region"
+ type: "string"
+ default: ""
+ description: "The region of the S3 bucket. See here for all region codes."
+ enum:
+ - ""
+ - "af-south-1"
+ - "ap-east-1"
+ - "ap-northeast-1"
+ - "ap-northeast-2"
+ - "ap-northeast-3"
+ - "ap-south-1"
+ - "ap-south-2"
+ - "ap-southeast-1"
+ - "ap-southeast-2"
+ - "ap-southeast-3"
+ - "ap-southeast-4"
+ - "ca-central-1"
+ - "ca-west-1"
+ - "cn-north-1"
+ - "cn-northwest-1"
+ - "eu-central-1"
+ - "eu-central-2"
+ - "eu-north-1"
+ - "eu-south-1"
+ - "eu-south-2"
+ - "eu-west-1"
+ - "eu-west-2"
+ - "eu-west-3"
+ - "il-central-1"
+ - "me-central-1"
+ - "me-south-1"
+ - "sa-east-1"
+ - "us-east-1"
+ - "us-east-2"
+ - "us-gov-east-1"
+ - "us-gov-west-1"
+ - "us-west-1"
+ - "us-west-2"
+ order: 4
+ format:
+ title: "Output Format"
type: "object"
- order: 6
+ description: "Format of the data output. See here for more details"
oneOf:
- - title: "disable"
- additionalProperties: false
- description: "Disable SSL."
+ - title: "CSV: Comma-Separated Values"
required:
- - "mode"
+ - "format_type"
+ - "flattening"
properties:
- mode:
+ format_type:
+ title: "Format Type"
type: "string"
- const: "disable"
enum:
- - "disable"
- default: "disable"
- order: 0
- - title: "allow"
- additionalProperties: false
- description: "Allow SSL mode."
- required:
- - "mode"
- properties:
- mode:
+ - "CSV"
+ default: "CSV"
+ flattening:
type: "string"
- const: "allow"
+ title: "Flattening"
+ description: "Whether the input json data should be normalized (flattened)\
+ \ in the output CSV. Please refer to docs for details."
+ default: "No flattening"
enum:
- - "allow"
- default: "allow"
- order: 0
- - title: "prefer"
- additionalProperties: false
- description: "Prefer SSL mode."
+ - "No flattening"
+ - "Root level flattening"
+ compression:
+ title: "Compression"
+ type: "object"
+ description: "Whether the output files should be compressed. If compression\
+ \ is selected, the output filename will have an extra extension\
+ \ (GZIP: \".csv.gz\")."
+ oneOf:
+ - title: "No Compression"
+ requires:
+ - "compression_type"
+ properties:
+ compression_type:
+ type: "string"
+ enum:
+ - "No Compression"
+ default: "No Compression"
+ - title: "GZIP"
+ requires:
+ - "compression_type"
+ properties:
+ compression_type:
+ type: "string"
+ enum:
+ - "GZIP"
+ default: "GZIP"
+ - title: "JSON Lines: Newline-delimited JSON"
required:
- - "mode"
+ - "format_type"
properties:
- mode:
+ format_type:
+ title: "Format Type"
type: "string"
- const: "prefer"
enum:
- - "prefer"
- default: "prefer"
- order: 0
- - title: "require"
- additionalProperties: false
- description: "Require SSL mode."
- required:
- - "mode"
- properties:
- mode:
+ - "JSONL"
+ default: "JSONL"
+ flattening:
type: "string"
- const: "require"
+ title: "Flattening"
+ description: "Whether the input json data should be normalized (flattened)\
+ \ in the output JSON Lines. Please refer to docs for details."
+ default: "No flattening"
enum:
- - "require"
- default: "require"
- order: 0
- - title: "verify-ca"
- additionalProperties: false
- description: "Verify-ca SSL mode."
+ - "No flattening"
+ - "Root level flattening"
+ compression:
+ title: "Compression"
+ type: "object"
+ description: "Whether the output files should be compressed. If compression\
+ \ is selected, the output filename will have an extra extension\
+ \ (GZIP: \".jsonl.gz\")."
+ oneOf:
+ - title: "No Compression"
+ requires: "compression_type"
+ properties:
+ compression_type:
+ type: "string"
+ enum:
+ - "No Compression"
+ default: "No Compression"
+ - title: "GZIP"
+ requires: "compression_type"
+ properties:
+ compression_type:
+ type: "string"
+ enum:
+ - "GZIP"
+ default: "GZIP"
+ - title: "Avro: Apache Avro"
required:
- - "mode"
- - "ssl_ca_certificate"
+ - "format_type"
+ - "compression_codec"
properties:
- mode:
+ format_type:
+ title: "Format Type"
type: "string"
- const: "verify-ca"
enum:
- - "verify-ca"
- default: "verify-ca"
+ - "Avro"
+ default: "Avro"
order: 0
- ssl_ca_certificate:
- type: "string"
- title: "CA certificate"
- description: "Specifies the file name of a PEM file that contains\
- \ Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n\
- \ See more information - in the docs."
- airbyte_secret: true
- multiline: true
+ compression_codec:
+ title: "Compression Codec"
+ description: "The compression algorithm used to compress data. Default\
+ \ to no compression."
+ type: "object"
+ oneOf:
+ - title: "No Compression"
+ required:
+ - "codec"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "no compression"
+ default: "no compression"
+ - title: "Deflate"
+ required:
+ - "codec"
+ - "compression_level"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "Deflate"
+ default: "Deflate"
+ compression_level:
+ title: "Deflate Level"
+ description: "0: no compression & fastest, 9: best compression\
+ \ & slowest."
+ type: "integer"
+ default: 0
+ minimum: 0
+ maximum: 9
+ - title: "bzip2"
+ required:
+ - "codec"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "bzip2"
+ default: "bzip2"
+ - title: "xz"
+ required:
+ - "codec"
+ - "compression_level"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "xz"
+ default: "xz"
+ compression_level:
+ title: "Compression Level"
+ description: "See here for details."
+ type: "integer"
+ default: 6
+ minimum: 0
+ maximum: 9
+ - title: "zstandard"
+ required:
+ - "codec"
+ - "compression_level"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "zstandard"
+ default: "zstandard"
+ compression_level:
+ title: "Compression Level"
+ description: "Negative levels are 'fast' modes akin to lz4 or\
+ \ snappy, levels above 9 are generally for archival purposes,\
+ \ and levels above 18 use a lot of memory."
+ type: "integer"
+ default: 3
+ minimum: -5
+ maximum: 22
+ include_checksum:
+ title: "Include Checksum"
+ description: "If true, include a checksum with each data block."
+ type: "boolean"
+ default: false
+ - title: "snappy"
+ required:
+ - "codec"
+ properties:
+ codec:
+ type: "string"
+ enum:
+ - "snappy"
+ default: "snappy"
order: 1
- - title: "verify-full"
- additionalProperties: false
- description: "Verify-full SSL mode."
+ - title: "Parquet: Columnar Storage"
required:
- - "mode"
- - "ssl_ca_certificate"
+ - "format_type"
properties:
- mode:
+ format_type:
+ title: "Format Type"
type: "string"
- const: "verify-full"
enum:
- - "verify-full"
- default: "verify-full"
- order: 0
- ssl_ca_certificate:
+ - "Parquet"
+ default: "Parquet"
+ compression_codec:
+ title: "Compression Codec"
+ description: "The compression algorithm used to compress data pages."
type: "string"
- title: "CA certificate"
- description: "Specifies the file name of a PEM file that contains\
- \ Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n\
- \ See more information - in the docs."
- airbyte_secret: true
- multiline: true
- order: 1
- jdbc_url_params:
- description: "Additional properties to pass to the JDBC URL string when\
- \ connecting to the database formatted as 'key=value' pairs separated\
- \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
- title: "JDBC URL Params"
+ enum:
+ - "UNCOMPRESSED"
+ - "SNAPPY"
+ - "GZIP"
+ - "LZO"
+ - "BROTLI"
+ - "LZ4"
+ - "ZSTD"
+ default: "UNCOMPRESSED"
+ block_size_mb:
+ title: "Block Size (Row Group Size) (MB)"
+ description: "This is the size of a row group being buffered in memory.\
+ \ It limits the memory usage when writing. Larger values will improve\
+ \ the IO when reading, but consume more memory when writing. Default:\
+ \ 128 MB."
+ type: "integer"
+ default: 128
+ examples:
+ - 128
+ max_padding_size_mb:
+ title: "Max Padding Size (MB)"
+ description: "Maximum size allowed as padding to align row groups.\
+ \ This is also the minimum size of a row group. Default: 8 MB."
+ type: "integer"
+ default: 8
+ examples:
+ - 8
+ page_size_kb:
+ title: "Page Size (KB)"
+ description: "The page size is for compression. A block is composed\
+ \ of pages. A page is the smallest unit that must be read fully\
+ \ to access a single record. If this value is too small, the compression\
+ \ will deteriorate. Default: 1024 KB."
+ type: "integer"
+ default: 1024
+ examples:
+ - 1024
+ dictionary_page_size_kb:
+ title: "Dictionary Page Size (KB)"
+ description: "There is one dictionary page per column per row group\
+ \ when dictionary encoding is used. The dictionary page size works\
+ \ like the page size but for dictionary. Default: 1024 KB."
+ type: "integer"
+ default: 1024
+ examples:
+ - 1024
+ dictionary_encoding:
+ title: "Dictionary Encoding"
+ description: "Default: true."
+ type: "boolean"
+ default: true
+ order: 5
+ s3_endpoint:
+ title: "Endpoint"
+ type: "string"
+ default: ""
+ description: "Your S3 endpoint url. Read more here"
+ examples:
+ - "http://localhost:9000"
+ order: 6
+ s3_path_format:
+ title: "S3 Path Format"
+ description: "Format string on how data will be organized inside the S3\
+ \ bucket directory. Read more here"
type: "string"
+ examples:
+ - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
order: 7
- destinationType:
- title: "teradata"
- const: "teradata"
- enum:
- - "teradata"
- order: 0
+ file_name_pattern:
type: "string"
- destination-teradata-update:
- title: "Teradata Destination Spec"
+ description: "The pattern allows you to set the file-name format for the\
+ \ S3 staging file(s)"
+ title: "S3 Filename pattern"
+ examples:
+ - "{date}"
+ - "{date:yyyy_MM}"
+ - "{timestamp}"
+ - "{part_number}"
+ - "{sync_id}"
+ order: 8
+ destination-redis:
+ title: "Redis Destination Spec"
type: "object"
required:
- "host"
- "username"
+ - "port"
+ - "cache_type"
+ - "destinationType"
properties:
host:
title: "Host"
- description: "Hostname of the database."
+ description: "Redis host to connect to."
type: "string"
- order: 0
+ examples:
+ - "localhost,127.0.0.1"
+ order: 1
+ port:
+ title: "Port"
+ description: "Port of Redis."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 6379
+ order: 2
username:
- title: "User"
- description: "Username to use to access the database."
+ title: "Username"
+ description: "Username associated with Redis."
type: "string"
- order: 1
+ order: 3
password:
title: "Password"
- description: "Password associated with the username."
+ description: "Password associated with Redis."
type: "string"
airbyte_secret: true
- order: 2
- schema:
- title: "Default Schema"
- description: "The default schema tables are written to if the source does\
- \ not specify a namespace. The usual value for this field is \"public\"\
- ."
- type: "string"
- examples:
- - "airbyte_td"
- default: "airbyte_td"
- order: 3
+ order: 4
+ x-speakeasy-param-sensitive: true
ssl:
title: "SSL Connection"
- description: "Encrypt data using SSL. When activating SSL, please select\
- \ one of the connection modes."
type: "boolean"
+ description: "Indicates whether SSL encryption protocol will be used to\
+ \ connect to Redis. It is recommended to use SSL connection if possible."
default: false
order: 5
ssl_mode:
- title: "SSL modes"
- description: "SSL connection modes. \n disable - Chose this mode\
- \ to disable encryption of communication between Airbyte and destination\
- \ database\n allow - Chose this mode to enable encryption only\
- \ when required by the destination database\n prefer - Chose this\
- \ mode to allow unencrypted connection only if the destination database\
- \ does not support encryption\n require - Chose this mode to always\
- \ require encryption. If the destination database server does not support\
- \ encryption, connection will fail\n verify-ca - Chose this mode\
- \ to always require encryption and to verify that the destination database\
- \ server has a valid SSL certificate\n verify-full - This is the\
- \ most secure mode. Chose this mode to always require encryption and to\
- \ verify the identity of the destination database server\n See more information\
- \ - in the docs."
+ title: "SSL Modes"
+ description: "SSL connection modes. \n verify-full - This is\
+ \ the most secure mode. Always require encryption and verifies the identity\
+ \ of the source database server"
type: "object"
order: 6
oneOf:
@@ -69157,75 +71976,14 @@ components:
- "disable"
default: "disable"
order: 0
- - title: "allow"
- additionalProperties: false
- description: "Allow SSL mode."
- required:
- - "mode"
- properties:
- mode:
- type: "string"
- const: "allow"
- enum:
- - "allow"
- default: "allow"
- order: 0
- - title: "prefer"
- additionalProperties: false
- description: "Prefer SSL mode."
- required:
- - "mode"
- properties:
- mode:
- type: "string"
- const: "prefer"
- enum:
- - "prefer"
- default: "prefer"
- order: 0
- - title: "require"
- additionalProperties: false
- description: "Require SSL mode."
- required:
- - "mode"
- properties:
- mode:
- type: "string"
- const: "require"
- enum:
- - "require"
- default: "require"
- order: 0
- - title: "verify-ca"
- additionalProperties: false
- description: "Verify-ca SSL mode."
- required:
- - "mode"
- - "ssl_ca_certificate"
- properties:
- mode:
- type: "string"
- const: "verify-ca"
- enum:
- - "verify-ca"
- default: "verify-ca"
- order: 0
- ssl_ca_certificate:
- type: "string"
- title: "CA certificate"
- description: "Specifies the file name of a PEM file that contains\
- \ Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n\
- \ See more information - in the docs."
- airbyte_secret: true
- multiline: true
- order: 1
- title: "verify-full"
additionalProperties: false
description: "Verify-full SSL mode."
required:
- "mode"
- - "ssl_ca_certificate"
+ - "ca_certificate"
+ - "client_certificate"
+ - "client_key"
properties:
mode:
type: "string"
@@ -69234,1446 +71992,1196 @@ components:
- "verify-full"
default: "verify-full"
order: 0
- ssl_ca_certificate:
+ ca_certificate:
type: "string"
- title: "CA certificate"
- description: "Specifies the file name of a PEM file that contains\
- \ Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n\
- \ See more information - in the docs."
+ title: "CA Certificate"
+ description: "CA certificate"
airbyte_secret: true
multiline: true
order: 1
- jdbc_url_params:
- description: "Additional properties to pass to the JDBC URL string when\
- \ connecting to the database formatted as 'key=value' pairs separated\
- \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
- title: "JDBC URL Params"
- type: "string"
- order: 7
- destination-pinecone:
- title: "Destination Config"
- description: "The configuration model for the Vector DB based destinations.\
- \ This model is used to generate the UI for the destination configuration,\n\
- as well as to provide type safety for the configuration passed to the destination.\n\
- \nThe configuration model is composed of four parts:\n* Processing configuration\n\
- * Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\
- \nProcessing, embedding and advanced configuration are provided by this base\
- \ class, while the indexing configuration is provided by the destination connector\
- \ in the sub class."
- type: "object"
- properties:
- embedding:
- title: "Embedding"
- description: "Embedding configuration"
- group: "embedding"
- type: "object"
- oneOf:
- - title: "OpenAI"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "openai"
- const: "openai"
- enum:
- - "openai"
+ client_certificate:
type: "string"
- openai_key:
- title: "OpenAI API key"
+ title: "Client Certificate"
+ description: "Client certificate"
airbyte_secret: true
+ multiline: true
+ order: 2
+ client_key:
type: "string"
+ title: "Client Key"
+ description: "Client key"
+ airbyte_secret: true
+ multiline: true
+ order: 3
x-speakeasy-param-sensitive: true
- required:
- - "openai_key"
- - "mode"
- description: "Use the OpenAI API to embed text. This option is using the\
- \ text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "Cohere"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "cohere"
- const: "cohere"
- enum:
- - "cohere"
+ client_key_password:
type: "string"
- cohere_key:
- title: "Cohere API key"
+ title: "Client key password"
+ description: "Password for keystorage. If you do not add it - the\
+ \ password will be generated automatically."
airbyte_secret: true
- type: "string"
+ order: 4
x-speakeasy-param-sensitive: true
+ cache_type:
+ title: "Cache type"
+ type: "string"
+ default: "hash"
+ description: "Redis cache type to store data in."
+ enum:
+ - "hash"
+ order: 7
+ tunnel_method:
+ type: "object"
+ title: "SSH Tunnel Method"
+ description: "Whether to initiate an SSH tunnel before connecting to the\
+ \ database, and if so, which kind of authentication to use."
+ oneOf:
+ - title: "No Tunnel"
required:
- - "cohere_key"
- - "mode"
- description: "Use the Cohere API to embed text."
- - title: "Fake"
- type: "object"
+ - "tunnel_method"
properties:
- mode:
- title: "Mode"
- default: "fake"
- const: "fake"
- enum:
- - "fake"
+ tunnel_method:
+ description: "No ssh tunnel needed to connect to database"
type: "string"
- description: "Use a fake embedding made out of random vectors with 1536\
- \ embedding dimensions. This is useful for testing the data pipeline\
- \ without incurring any costs."
+ const: "NO_TUNNEL"
+ order: 0
+ enum:
+ - "NO_TUNNEL"
+ - title: "SSH Key Authentication"
required:
- - "mode"
- - title: "Azure OpenAI"
- type: "object"
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "ssh_key"
properties:
- mode:
- title: "Mode"
- default: "azure_openai"
- const: "azure_openai"
- enum:
- - "azure_openai"
- type: "string"
- openai_key:
- title: "Azure OpenAI API key"
- description: "The API key for your Azure OpenAI resource. You can\
- \ find this in the Azure portal under your Azure OpenAI resource"
- airbyte_secret: true
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and ssh key"
type: "string"
- x-speakeasy-param-sensitive: true
- api_base:
- title: "Resource base URL"
- description: "The base URL for your Azure OpenAI resource. You can\
- \ find this in the Azure portal under your Azure OpenAI resource"
- examples:
- - "https://your-resource-name.openai.azure.com"
+ const: "SSH_KEY_AUTH"
+ order: 0
+ enum:
+ - "SSH_KEY_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
type: "string"
- deployment:
- title: "Deployment"
- description: "The deployment for your Azure OpenAI resource. You\
- \ can find this in the Azure portal under your Azure OpenAI resource"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
examples:
- - "your-resource-name"
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host."
type: "string"
- required:
- - "openai_key"
- - "api_base"
- - "deployment"
- - "mode"
- description: "Use the Azure-hosted OpenAI API to embed text. This option\
- \ is using the text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "OpenAI-compatible"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "openai_compatible"
- const: "openai_compatible"
- enum:
- - "openai_compatible"
+ order: 3
+ ssh_key:
+ title: "SSH Private Key"
+ description: "OS-level user account ssh key credentials in RSA PEM\
+ \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
type: "string"
- api_key:
- title: "API key"
- default: ""
airbyte_secret: true
- type: "string"
+ multiline: true
+ order: 4
x-speakeasy-param-sensitive: true
- base_url:
- title: "Base URL"
- description: "The base URL for your OpenAI-compatible service"
- examples:
- - "https://your-service-name.com"
+ - title: "Password Authentication"
+ required:
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "tunnel_user_password"
+ properties:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and password authentication"
type: "string"
- model_name:
- title: "Model name"
- description: "The name of the model to use for embedding"
- default: "text-embedding-ada-002"
- examples:
- - "text-embedding-ada-002"
+ const: "SSH_PASSWORD_AUTH"
+ order: 0
+ enum:
+ - "SSH_PASSWORD_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
type: "string"
- dimensions:
- title: "Embedding dimensions"
- description: "The number of dimensions the embedding model is generating"
- examples:
- - 1536
- - 384
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
type: "integer"
- required:
- - "base_url"
- - "dimensions"
- - "mode"
- description: "Use a service that's compatible with the OpenAI API to embed\
- \ text."
- processing:
- title: "ProcessingConfigModel"
- type: "object"
- properties:
- chunk_size:
- title: "Chunk size"
- description: "Size of chunks in tokens to store in vector store (make\
- \ sure it is not too big for the context if your LLM)"
- maximum: 8191
- minimum: 1
- type: "integer"
- chunk_overlap:
- title: "Chunk overlap"
- description: "Size of overlap between chunks in tokens to store in vector\
- \ store to better capture relevant context"
- default: 0
- type: "integer"
- text_fields:
- title: "Text fields to embed"
- description: "List of fields in the record that should be used to calculate\
- \ the embedding. The field list is applied to all streams in the same\
- \ way and non-existing fields are ignored. If none are defined, all\
- \ fields are considered text fields. When specifying text fields,\
- \ you can access nested fields in the record by using dot notation,\
- \ e.g. `user.name` will access the `name` field in the `user` object.\
- \ It's also possible to use wildcards to access all fields in an object,\
- \ e.g. `users.*.name` will access all `names` fields in all entries\
- \ of the `users` array."
- default: []
- always_show: true
- examples:
- - "text"
- - "user.name"
- - "users.*.name"
- type: "array"
- items:
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host"
type: "string"
- metadata_fields:
- title: "Fields to store as metadata"
- description: "List of fields in the record that should be stored as\
- \ metadata. The field list is applied to all streams in the same way\
- \ and non-existing fields are ignored. If none are defined, all fields\
- \ are considered metadata fields. When specifying text fields, you\
- \ can access nested fields in the record by using dot notation, e.g.\
- \ `user.name` will access the `name` field in the `user` object. It's\
- \ also possible to use wildcards to access all fields in an object,\
- \ e.g. `users.*.name` will access all `names` fields in all entries\
- \ of the `users` array. When specifying nested paths, all matching\
- \ values are flattened into an array set to a field named by the path."
- default: []
- always_show: true
- examples:
- - "age"
- - "user"
- - "user.name"
- type: "array"
- items:
+ order: 3
+ tunnel_user_password:
+ title: "Password"
+ description: "OS-level password for logging into the jump server host"
type: "string"
- text_splitter:
- title: "Text splitter"
- description: "Split text fields into chunks based on the specified method."
- type: "object"
- oneOf:
- - title: "By Separator"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "separator"
- const: "separator"
- enum:
- - "separator"
- type: "string"
- separators:
- title: "Separators"
- description: "List of separator strings to split text fields by.\
- \ The separator itself needs to be wrapped in double quotes,\
- \ e.g. to split by the dot character, use \".\". To split by\
- \ a newline, use \"\\n\"."
- default:
- - "\"\\n\\n\""
- - "\"\\n\""
- - "\" \""
- - "\"\""
- type: "array"
- items:
- type: "string"
- keep_separator:
- title: "Keep separator"
- description: "Whether to keep the separator in the resulting chunks"
- default: false
- type: "boolean"
- description: "Split the text by the list of separators until the chunk\
- \ size is reached, using the earlier mentioned separators where\
- \ possible. This is useful for splitting text fields by paragraphs,\
- \ sentences, words, etc."
- required:
- - "mode"
- - title: "By Markdown header"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "markdown"
- const: "markdown"
- enum:
- - "markdown"
- type: "string"
- split_level:
- title: "Split level"
- description: "Level of markdown headers to split text fields by.\
- \ Headings down to the specified level will be used as split\
- \ points"
- default: 1
- minimum: 1
- maximum: 6
- type: "integer"
- description: "Split the text by Markdown headers down to the specified\
- \ header level. If the chunk size fits multiple sections, they will\
- \ be combined into a single chunk."
- required:
- - "mode"
- - title: "By Programming Language"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "code"
- const: "code"
- enum:
- - "code"
- type: "string"
- language:
- title: "Language"
- description: "Split code in suitable places based on the programming\
- \ language"
- enum:
- - "cpp"
- - "go"
- - "java"
- - "js"
- - "php"
- - "proto"
- - "python"
- - "rst"
- - "ruby"
- - "rust"
- - "scala"
- - "swift"
- - "markdown"
- - "latex"
- - "html"
- - "sol"
- type: "string"
- required:
- - "language"
- - "mode"
- description: "Split the text by suitable delimiters based on the programming\
- \ language. This is useful for splitting code into chunks."
- field_name_mappings:
- title: "Field name mappings"
- description: "List of fields to rename. Not applicable for nested fields,\
- \ but can be used to rename fields already flattened via dot notation."
- default: []
- type: "array"
- items:
- title: "FieldNameMappingConfigModel"
- type: "object"
- properties:
- from_field:
- title: "From field name"
- description: "The field name in the source"
- type: "string"
- to_field:
- title: "To field name"
- description: "The field name to use in the destination"
- type: "string"
- required:
- - "from_field"
- - "to_field"
- required:
- - "chunk_size"
- group: "processing"
- omit_raw_text:
- title: "Do not store raw text"
- description: "Do not store the text that gets embedded along with the vector\
- \ and the metadata in the destination. If set to true, only the vector\
- \ and the metadata will be stored - in this case raw text for LLM use\
- \ cases needs to be retrieved from another source."
- default: false
- group: "advanced"
- type: "boolean"
- indexing:
- title: "Indexing"
- type: "object"
- properties:
- pinecone_key:
- title: "Pinecone API key"
- description: "The Pinecone API key to use matching the environment (copy\
- \ from Pinecone console)"
- airbyte_secret: true
- type: "string"
- x-speakeasy-param-sensitive: true
- pinecone_environment:
- title: "Pinecone Environment"
- description: "Pinecone Cloud environment to use"
- examples:
- - "us-west1-gcp"
- - "gcp-starter"
- type: "string"
- index:
- title: "Index"
- description: "Pinecone index in your project to load data into"
- type: "string"
- required:
- - "pinecone_key"
- - "pinecone_environment"
- - "index"
- description: "Pinecone is a popular vector store that can be used to store\
- \ and retrieve embeddings."
- group: "indexing"
+ airbyte_secret: true
+ order: 4
+ x-speakeasy-param-sensitive: true
destinationType:
- title: "pinecone"
- const: "pinecone"
+ title: "redis"
+ const: "redis"
enum:
- - "pinecone"
+ - "redis"
order: 0
type: "string"
- required:
- - "embedding"
- - "processing"
- - "indexing"
- - "destinationType"
- groups:
- - id: "processing"
- title: "Processing"
- - id: "embedding"
- title: "Embedding"
- - id: "indexing"
- title: "Indexing"
- - id: "advanced"
- title: "Advanced"
- destination-pinecone-update:
- title: "Destination Config"
- description: "The configuration model for the Vector DB based destinations.\
- \ This model is used to generate the UI for the destination configuration,\n\
- as well as to provide type safety for the configuration passed to the destination.\n\
- \nThe configuration model is composed of four parts:\n* Processing configuration\n\
- * Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\
- \nProcessing, embedding and advanced configuration are provided by this base\
- \ class, while the indexing configuration is provided by the destination connector\
- \ in the sub class."
+ destination-redis-update:
+ title: "Redis Destination Spec"
type: "object"
+ required:
+ - "host"
+ - "username"
+ - "port"
+ - "cache_type"
properties:
- embedding:
- title: "Embedding"
- description: "Embedding configuration"
- group: "embedding"
- type: "object"
- oneOf:
- - title: "OpenAI"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "openai"
- const: "openai"
- enum:
- - "openai"
- type: "string"
- openai_key:
- title: "OpenAI API key"
- airbyte_secret: true
- type: "string"
- required:
- - "openai_key"
- - "mode"
- description: "Use the OpenAI API to embed text. This option is using the\
- \ text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "Cohere"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "cohere"
- const: "cohere"
- enum:
- - "cohere"
- type: "string"
- cohere_key:
- title: "Cohere API key"
- airbyte_secret: true
- type: "string"
+ host:
+ title: "Host"
+ description: "Redis host to connect to."
+ type: "string"
+ examples:
+ - "localhost,127.0.0.1"
+ order: 1
+ port:
+ title: "Port"
+ description: "Port of Redis."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 6379
+ order: 2
+ username:
+ title: "Username"
+ description: "Username associated with Redis."
+ type: "string"
+ order: 3
+ password:
+ title: "Password"
+ description: "Password associated with Redis."
+ type: "string"
+ airbyte_secret: true
+ order: 4
+ ssl:
+ title: "SSL Connection"
+ type: "boolean"
+ description: "Indicates whether SSL encryption protocol will be used to\
+ \ connect to Redis. It is recommended to use SSL connection if possible."
+ default: false
+ order: 5
+ ssl_mode:
+ title: "SSL Modes"
+ description: "SSL connection modes. \n verify-full - This is\
+ \ the most secure mode. Always require encryption and verifies the identity\
+ \ of the source database server"
+ type: "object"
+ order: 6
+ oneOf:
+ - title: "disable"
+ additionalProperties: false
+ description: "Disable SSL."
required:
- - "cohere_key"
- "mode"
- description: "Use the Cohere API to embed text."
- - title: "Fake"
- type: "object"
properties:
mode:
- title: "Mode"
- default: "fake"
- const: "fake"
- enum:
- - "fake"
type: "string"
- description: "Use a fake embedding made out of random vectors with 1536\
- \ embedding dimensions. This is useful for testing the data pipeline\
- \ without incurring any costs."
+ const: "disable"
+ enum:
+ - "disable"
+ default: "disable"
+ order: 0
+ - title: "verify-full"
+ additionalProperties: false
+ description: "Verify-full SSL mode."
required:
- "mode"
- - title: "Azure OpenAI"
- type: "object"
+ - "ca_certificate"
+ - "client_certificate"
+ - "client_key"
properties:
mode:
- title: "Mode"
- default: "azure_openai"
- const: "azure_openai"
+ type: "string"
+ const: "verify-full"
enum:
- - "azure_openai"
+ - "verify-full"
+ default: "verify-full"
+ order: 0
+ ca_certificate:
type: "string"
- openai_key:
- title: "Azure OpenAI API key"
- description: "The API key for your Azure OpenAI resource. You can\
- \ find this in the Azure portal under your Azure OpenAI resource"
+ title: "CA Certificate"
+ description: "CA certificate"
airbyte_secret: true
+ multiline: true
+ order: 1
+ client_certificate:
type: "string"
- api_base:
- title: "Resource base URL"
- description: "The base URL for your Azure OpenAI resource. You can\
- \ find this in the Azure portal under your Azure OpenAI resource"
- examples:
- - "https://your-resource-name.openai.azure.com"
+ title: "Client Certificate"
+ description: "Client certificate"
+ airbyte_secret: true
+ multiline: true
+ order: 2
+ client_key:
type: "string"
- deployment:
- title: "Deployment"
- description: "The deployment for your Azure OpenAI resource. You\
- \ can find this in the Azure portal under your Azure OpenAI resource"
- examples:
- - "your-resource-name"
+ title: "Client Key"
+ description: "Client key"
+ airbyte_secret: true
+ multiline: true
+ order: 3
+ client_key_password:
type: "string"
+ title: "Client key password"
+ description: "Password for keystorage. If you do not add it - the\
+ \ password will be generated automatically."
+ airbyte_secret: true
+ order: 4
+ cache_type:
+ title: "Cache type"
+ type: "string"
+ default: "hash"
+ description: "Redis cache type to store data in."
+ enum:
+ - "hash"
+ order: 7
+ tunnel_method:
+ type: "object"
+ title: "SSH Tunnel Method"
+ description: "Whether to initiate an SSH tunnel before connecting to the\
+ \ database, and if so, which kind of authentication to use."
+ oneOf:
+ - title: "No Tunnel"
required:
- - "openai_key"
- - "api_base"
- - "deployment"
- - "mode"
- description: "Use the Azure-hosted OpenAI API to embed text. This option\
- \ is using the text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "OpenAI-compatible"
- type: "object"
+ - "tunnel_method"
properties:
- mode:
- title: "Mode"
- default: "openai_compatible"
- const: "openai_compatible"
+ tunnel_method:
+ description: "No ssh tunnel needed to connect to database"
+ type: "string"
+ const: "NO_TUNNEL"
+ order: 0
enum:
- - "openai_compatible"
+ - "NO_TUNNEL"
+ - title: "SSH Key Authentication"
+ required:
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "ssh_key"
+ properties:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and ssh key"
type: "string"
- api_key:
- title: "API key"
- default: ""
- airbyte_secret: true
+ const: "SSH_KEY_AUTH"
+ order: 0
+ enum:
+ - "SSH_KEY_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
type: "string"
- base_url:
- title: "Base URL"
- description: "The base URL for your OpenAI-compatible service"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
examples:
- - "https://your-service-name.com"
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host."
type: "string"
- model_name:
- title: "Model name"
- description: "The name of the model to use for embedding"
- default: "text-embedding-ada-002"
- examples:
- - "text-embedding-ada-002"
+ order: 3
+ ssh_key:
+ title: "SSH Private Key"
+ description: "OS-level user account ssh key credentials in RSA PEM\
+ \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
type: "string"
- dimensions:
- title: "Embedding dimensions"
- description: "The number of dimensions the embedding model is generating"
- examples:
- - 1536
- - 384
- type: "integer"
+ airbyte_secret: true
+ multiline: true
+ order: 4
+ - title: "Password Authentication"
required:
- - "base_url"
- - "dimensions"
- - "mode"
- description: "Use a service that's compatible with the OpenAI API to embed\
- \ text."
- processing:
- title: "ProcessingConfigModel"
- type: "object"
- properties:
- chunk_size:
- title: "Chunk size"
- description: "Size of chunks in tokens to store in vector store (make\
- \ sure it is not too big for the context if your LLM)"
- maximum: 8191
- minimum: 1
- type: "integer"
- chunk_overlap:
- title: "Chunk overlap"
- description: "Size of overlap between chunks in tokens to store in vector\
- \ store to better capture relevant context"
- default: 0
- type: "integer"
- text_fields:
- title: "Text fields to embed"
- description: "List of fields in the record that should be used to calculate\
- \ the embedding. The field list is applied to all streams in the same\
- \ way and non-existing fields are ignored. If none are defined, all\
- \ fields are considered text fields. When specifying text fields,\
- \ you can access nested fields in the record by using dot notation,\
- \ e.g. `user.name` will access the `name` field in the `user` object.\
- \ It's also possible to use wildcards to access all fields in an object,\
- \ e.g. `users.*.name` will access all `names` fields in all entries\
- \ of the `users` array."
- default: []
- always_show: true
- examples:
- - "text"
- - "user.name"
- - "users.*.name"
- type: "array"
- items:
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "tunnel_user_password"
+ properties:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and password authentication"
type: "string"
- metadata_fields:
- title: "Fields to store as metadata"
- description: "List of fields in the record that should be stored as\
- \ metadata. The field list is applied to all streams in the same way\
- \ and non-existing fields are ignored. If none are defined, all fields\
- \ are considered metadata fields. When specifying text fields, you\
- \ can access nested fields in the record by using dot notation, e.g.\
- \ `user.name` will access the `name` field in the `user` object. It's\
- \ also possible to use wildcards to access all fields in an object,\
- \ e.g. `users.*.name` will access all `names` fields in all entries\
- \ of the `users` array. When specifying nested paths, all matching\
- \ values are flattened into an array set to a field named by the path."
- default: []
- always_show: true
- examples:
- - "age"
- - "user"
- - "user.name"
- type: "array"
- items:
+ const: "SSH_PASSWORD_AUTH"
+ order: 0
+ enum:
+ - "SSH_PASSWORD_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
type: "string"
- text_splitter:
- title: "Text splitter"
- description: "Split text fields into chunks based on the specified method."
- type: "object"
- oneOf:
- - title: "By Separator"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "separator"
- const: "separator"
- enum:
- - "separator"
- type: "string"
- separators:
- title: "Separators"
- description: "List of separator strings to split text fields by.\
- \ The separator itself needs to be wrapped in double quotes,\
- \ e.g. to split by the dot character, use \".\". To split by\
- \ a newline, use \"\\n\"."
- default:
- - "\"\\n\\n\""
- - "\"\\n\""
- - "\" \""
- - "\"\""
- type: "array"
- items:
- type: "string"
- keep_separator:
- title: "Keep separator"
- description: "Whether to keep the separator in the resulting chunks"
- default: false
- type: "boolean"
- description: "Split the text by the list of separators until the chunk\
- \ size is reached, using the earlier mentioned separators where\
- \ possible. This is useful for splitting text fields by paragraphs,\
- \ sentences, words, etc."
- required:
- - "mode"
- - title: "By Markdown header"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "markdown"
- const: "markdown"
- enum:
- - "markdown"
- type: "string"
- split_level:
- title: "Split level"
- description: "Level of markdown headers to split text fields by.\
- \ Headings down to the specified level will be used as split\
- \ points"
- default: 1
- minimum: 1
- maximum: 6
- type: "integer"
- description: "Split the text by Markdown headers down to the specified\
- \ header level. If the chunk size fits multiple sections, they will\
- \ be combined into a single chunk."
- required:
- - "mode"
- - title: "By Programming Language"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "code"
- const: "code"
- enum:
- - "code"
- type: "string"
- language:
- title: "Language"
- description: "Split code in suitable places based on the programming\
- \ language"
- enum:
- - "cpp"
- - "go"
- - "java"
- - "js"
- - "php"
- - "proto"
- - "python"
- - "rst"
- - "ruby"
- - "rust"
- - "scala"
- - "swift"
- - "markdown"
- - "latex"
- - "html"
- - "sol"
- type: "string"
- required:
- - "language"
- - "mode"
- description: "Split the text by suitable delimiters based on the programming\
- \ language. This is useful for splitting code into chunks."
- field_name_mappings:
- title: "Field name mappings"
- description: "List of fields to rename. Not applicable for nested fields,\
- \ but can be used to rename fields already flattened via dot notation."
- default: []
- type: "array"
- items:
- title: "FieldNameMappingConfigModel"
- type: "object"
- properties:
- from_field:
- title: "From field name"
- description: "The field name in the source"
- type: "string"
- to_field:
- title: "To field name"
- description: "The field name to use in the destination"
- type: "string"
- required:
- - "from_field"
- - "to_field"
- required:
- - "chunk_size"
- group: "processing"
- omit_raw_text:
- title: "Do not store raw text"
- description: "Do not store the text that gets embedded along with the vector\
- \ and the metadata in the destination. If set to true, only the vector\
- \ and the metadata will be stored - in this case raw text for LLM use\
- \ cases needs to be retrieved from another source."
- default: false
- group: "advanced"
- type: "boolean"
- indexing:
- title: "Indexing"
- type: "object"
- properties:
- pinecone_key:
- title: "Pinecone API key"
- description: "The Pinecone API key to use matching the environment (copy\
- \ from Pinecone console)"
- airbyte_secret: true
- type: "string"
- pinecone_environment:
- title: "Pinecone Environment"
- description: "Pinecone Cloud environment to use"
- examples:
- - "us-west1-gcp"
- - "gcp-starter"
- type: "string"
- index:
- title: "Index"
- description: "Pinecone index in your project to load data into"
- type: "string"
- required:
- - "pinecone_key"
- - "pinecone_environment"
- - "index"
- description: "Pinecone is a popular vector store that can be used to store\
- \ and retrieve embeddings."
- group: "indexing"
- required:
- - "embedding"
- - "processing"
- - "indexing"
- groups:
- - id: "processing"
- title: "Processing"
- - id: "embedding"
- title: "Embedding"
- - id: "indexing"
- title: "Indexing"
- - id: "advanced"
- title: "Advanced"
- destination-duckdb:
- title: "Destination Duckdb"
- type: "object"
- required:
- - "destination_path"
- - "destinationType"
- properties:
- motherduck_api_key:
- title: "MotherDuck API Key"
- type: "string"
- description: "API key to use for authentication to a MotherDuck database."
- airbyte_secret: true
- x-speakeasy-param-sensitive: true
- destination_path:
- title: "Destination DB"
- type: "string"
- description: "Path to the .duckdb file, or the text 'md:' to connect to\
- \ MotherDuck. The file will be placed inside that local mount. For more\
- \ information check out our docs"
- examples:
- - "/local/destination.duckdb"
- - "md:"
- - "motherduck:"
- schema:
- title: "Destination Schema"
- type: "string"
- description: "Database schema name, default for duckdb is 'main'."
- example: "main"
- destinationType:
- title: "duckdb"
- const: "duckdb"
- enum:
- - "duckdb"
- order: 0
- type: "string"
- destination-duckdb-update:
- title: "Destination Duckdb"
- type: "object"
- required:
- - "destination_path"
- properties:
- motherduck_api_key:
- title: "MotherDuck API Key"
- type: "string"
- description: "API key to use for authentication to a MotherDuck database."
- airbyte_secret: true
- destination_path:
- title: "Destination DB"
- type: "string"
- description: "Path to the .duckdb file, or the text 'md:' to connect to\
- \ MotherDuck. The file will be placed inside that local mount. For more\
- \ information check out our docs"
- examples:
- - "/local/destination.duckdb"
- - "md:"
- - "motherduck:"
- schema:
- title: "Destination Schema"
- type: "string"
- description: "Database schema name, default for duckdb is 'main'."
- example: "main"
- destination-sftp-json:
- title: "Destination SFTP JSON"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host"
+ type: "string"
+ order: 3
+ tunnel_user_password:
+ title: "Password"
+ description: "OS-level password for logging into the jump server host"
+ type: "string"
+ airbyte_secret: true
+ order: 4
+ destination-typesense:
+ title: "Destination Typesense"
type: "object"
required:
+ - "api_key"
- "host"
- - "username"
- - "password"
- - "destination_path"
- "destinationType"
properties:
+ api_key:
+ title: "API Key"
+ type: "string"
+ description: "Typesense API Key"
+ order: 0
+ x-speakeasy-param-sensitive: true
host:
title: "Host"
- description: "Hostname of the SFTP server."
type: "string"
- order: 0
+ description: "Hostname of the Typesense instance without protocol."
+ order: 1
port:
title: "Port"
- description: "Port of the SFTP server."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
- examples:
- - 22
- order: 1
- username:
- title: "User"
- description: "Username to use to access the SFTP server."
type: "string"
+ description: "Port of the Typesense instance. Ex: 8108, 80, 443. Default\
+ \ is 443"
order: 2
- password:
- title: "Password"
- description: "Password associated with the username."
+ protocol:
+ title: "Protocol"
type: "string"
- airbyte_secret: true
+ description: "Protocol of the Typesense instance. Ex: http or https. Default\
+ \ is https"
order: 3
- x-speakeasy-param-sensitive: true
- destination_path:
- title: "Destination path"
- type: "string"
- description: "Path to the directory where json files will be written."
- examples:
- - "/json_data"
+ batch_size:
+ title: "Batch size"
+ type: "integer"
+ description: "How many documents should be imported together. Default 1000"
order: 4
+ path:
+ title: "Path"
+ type: "string"
+ description: "Path of the Typesense instance. Default is none"
+ order: 5
destinationType:
- title: "sftp-json"
- const: "sftp-json"
+ title: "typesense"
+ const: "typesense"
enum:
- - "sftp-json"
+ - "typesense"
order: 0
type: "string"
- destination-sftp-json-update:
- title: "Destination SFTP JSON"
+ destination-typesense-update:
+ title: "Destination Typesense"
type: "object"
required:
+ - "api_key"
- "host"
- - "username"
- - "password"
- - "destination_path"
properties:
+ api_key:
+ title: "API Key"
+ type: "string"
+ description: "Typesense API Key"
+ order: 0
host:
title: "Host"
- description: "Hostname of the SFTP server."
type: "string"
- order: 0
+ description: "Hostname of the Typesense instance without protocol."
+ order: 1
port:
title: "Port"
- description: "Port of the SFTP server."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
- examples:
- - 22
- order: 1
- username:
- title: "User"
- description: "Username to use to access the SFTP server."
type: "string"
+ description: "Port of the Typesense instance. Ex: 8108, 80, 443. Default\
+ \ is 443"
order: 2
- password:
- title: "Password"
- description: "Password associated with the username."
+ protocol:
+ title: "Protocol"
type: "string"
- airbyte_secret: true
+ description: "Protocol of the Typesense instance. Ex: http or https. Default\
+ \ is https"
order: 3
- destination_path:
- title: "Destination path"
- type: "string"
- description: "Path to the directory where json files will be written."
- examples:
- - "/json_data"
+ batch_size:
+ title: "Batch size"
+ type: "integer"
+ description: "How many documents should be imported together. Default 1000"
order: 4
- destination-s3:
- title: "S3 Destination Spec"
+ path:
+ title: "Path"
+ type: "string"
+ description: "Path of the Typesense instance. Default is none"
+ order: 5
+ destination-bigquery:
+ title: "BigQuery Destination Spec"
type: "object"
required:
- - "s3_bucket_name"
- - "s3_bucket_path"
- - "s3_bucket_region"
- - "format"
+ - "project_id"
+ - "dataset_location"
+ - "dataset_id"
- "destinationType"
properties:
- access_key_id:
+ project_id:
type: "string"
- description: "The access key ID to access the S3 bucket. Airbyte requires\
- \ Read and Write permissions to the given bucket. Read more here."
- title: "S3 Key ID"
- airbyte_secret: true
- always_show: true
- examples:
- - "A012345678910EXAMPLE"
+ title: "Project ID"
+ group: "connection"
order: 0
- x-speakeasy-param-sensitive: true
- secret_access_key:
+ dataset_location:
type: "string"
- description: "The corresponding secret to the access key ID. Read more here"
- title: "S3 Access Key"
+ description: "The location of the dataset. Warning: Changes made after creation\
+ \ will not be applied. Read more here."
+ title: "Dataset Location"
+ group: "connection"
+ order: 1
+ enum:
+ - "US"
+ - "EU"
+ - "asia-east1"
+ - "asia-east2"
+ - "asia-northeast1"
+ - "asia-northeast2"
+ - "asia-northeast3"
+ - "asia-south1"
+ - "asia-south2"
+ - "asia-southeast1"
+ - "asia-southeast2"
+ - "australia-southeast1"
+ - "australia-southeast2"
+ - "europe-central1"
+ - "europe-central2"
+ - "europe-north1"
+ - "europe-southwest1"
+ - "europe-west1"
+ - "europe-west2"
+ - "europe-west3"
+ - "europe-west4"
+ - "europe-west6"
+ - "europe-west7"
+ - "europe-west8"
+ - "europe-west9"
+ - "europe-west12"
+ - "me-central1"
+ - "me-central2"
+ - "me-west1"
+ - "northamerica-northeast1"
+ - "northamerica-northeast2"
+ - "southamerica-east1"
+ - "southamerica-west1"
+ - "us-central1"
+ - "us-east1"
+ - "us-east2"
+ - "us-east3"
+ - "us-east4"
+ - "us-east5"
+ - "us-south1"
+ - "us-west1"
+ - "us-west2"
+ - "us-west3"
+ - "us-west4"
+ dataset_id:
+ type: "string"
+ description: "The default BigQuery Dataset ID that tables are replicated\
+ \ to if the source does not specify a namespace. Read more here."
+ title: "Default Dataset ID"
+ group: "connection"
+ order: 2
+ loading_method:
+ type: "object"
+ title: "Loading Method"
+ description: "The way data will be uploaded to BigQuery."
+ display_type: "radio"
+ group: "connection"
+ order: 3
+ oneOf:
+ - title: "GCS Staging"
+ description: "(recommended) Writes large batches of records to\
+ \ a file, uploads the file to GCS, then uses COPY INTO to load your\
+ \ data into BigQuery. Provides best-in-class speed, reliability and\
+ \ scalability. Read more about GCS Staging here."
+ required:
+ - "method"
+ - "gcs_bucket_name"
+ - "gcs_bucket_path"
+ - "credential"
+ properties:
+ method:
+ type: "string"
+ const: "GCS Staging"
+ order: 0
+ enum:
+ - "GCS Staging"
+ credential:
+ title: "Credential"
+ description: "An HMAC key is a type of credential and can be associated\
+ \ with a service account or a user account in Cloud Storage. Read\
+ \ more here."
+ type: "object"
+ order: 1
+ oneOf:
+ - title: "HMAC key"
+ required:
+ - "credential_type"
+ - "hmac_key_access_id"
+ - "hmac_key_secret"
+ properties:
+ credential_type:
+ type: "string"
+ const: "HMAC_KEY"
+ order: 0
+ enum:
+ - "HMAC_KEY"
+ hmac_key_access_id:
+ type: "string"
+ description: "HMAC key access ID. When linked to a service account,\
+ \ this ID is 61 characters long; when linked to a user account,\
+ \ it is 24 characters long."
+ title: "HMAC Key Access ID"
+ airbyte_secret: true
+ examples:
+ - "1234567890abcdefghij1234"
+ order: 1
+ x-speakeasy-param-sensitive: true
+ hmac_key_secret:
+ type: "string"
+ description: "The corresponding secret for the access ID. It\
+ \ is a 40-character base-64 encoded string."
+ title: "HMAC Key Secret"
+ airbyte_secret: true
+ examples:
+ - "1234567890abcdefghij1234567890ABCDEFGHIJ"
+ order: 2
+ x-speakeasy-param-sensitive: true
+ gcs_bucket_name:
+ title: "GCS Bucket Name"
+ type: "string"
+ description: "The name of the GCS bucket. Read more here."
+ examples:
+ - "airbyte_sync"
+ order: 2
+ gcs_bucket_path:
+ title: "GCS Bucket Path"
+ description: "Directory under the GCS bucket where data will be written."
+ type: "string"
+ examples:
+ - "data_sync/test"
+ order: 3
+ keep_files_in_gcs-bucket:
+ type: "string"
+ description: "This upload method is supposed to temporary store records\
+ \ in GCS bucket. By this select you can chose if these records should\
+ \ be removed from GCS when migration has finished. The default \"\
+ Delete all tmp files from GCS\" value is used if not set explicitly."
+ title: "GCS Tmp Files Afterward Processing"
+ default: "Delete all tmp files from GCS"
+ enum:
+ - "Delete all tmp files from GCS"
+ - "Keep all tmp files in GCS"
+ order: 4
+ - title: "Standard Inserts"
+ required:
+ - "method"
+ description: "(not recommended) Direct loading using SQL INSERT\
+ \ statements. This method is extremely inefficient and provided only\
+ \ for quick testing. In all other cases, you should use GCS staging."
+ properties:
+ method:
+ type: "string"
+ const: "Standard"
+ enum:
+ - "Standard"
+ credentials_json:
+ type: "string"
+ description: "The contents of the JSON service account key. Check out the\
+ \ docs if you need help generating this key. Default credentials will\
+ \ be used if this field is left empty."
+ title: "Service Account Key JSON (Required for cloud, optional for open-source)"
airbyte_secret: true
+ group: "connection"
+ order: 4
always_show: true
- examples:
- - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
- order: 1
- x-speakeasy-param-sensitive: true
- s3_bucket_name:
- title: "S3 Bucket Name"
+ transformation_priority:
type: "string"
- description: "The name of the S3 bucket. Read more here. Batch queries are queued and started as soon as idle resources\
+ \ are available in the BigQuery shared resource pool, which usually occurs\
+ \ within a few minutes. Batch queries don’t count towards your concurrent\
+ \ rate limit. Read more about batch queries here. The default \"interactive\" value is used if not set explicitly."
+ title: "Transformation Query Run Type"
+ default: "interactive"
+ enum:
+ - "interactive"
+ - "batch"
+ order: 5
+ group: "advanced"
+ big_query_client_buffer_size_mb:
+ title: "Google BigQuery Client Chunk Size"
+ description: "Google BigQuery client's chunk (buffer) size (MIN=1, MAX =\
+ \ 15) for each table. The size that will be written by a single RPC. Written\
+ \ data will be buffered and only flushed upon reaching this size or closing\
+ \ the channel. The default 15MB value is used if not set explicitly. Read\
+ \ more here."
+ type: "integer"
+ minimum: 1
+ maximum: 15
+ default: 15
examples:
- - "airbyte_sync"
- order: 2
- s3_bucket_path:
- title: "S3 Bucket Path"
- description: "Directory under the S3 bucket where data will be written.\
- \ Read more here"
+ - "15"
+ order: 6
+ group: "advanced"
+ raw_data_dataset:
type: "string"
- examples:
- - "data_sync/test"
- order: 3
- s3_bucket_region:
- title: "S3 Bucket Region"
+ description: "The dataset to write raw tables into (default: airbyte_internal)"
+ title: "Raw Table Dataset Name"
+ order: 7
+ group: "advanced"
+ disable_type_dedupe:
+ type: "boolean"
+ default: false
+ description: "Disable Writing Final Tables. WARNING! The data format in\
+ \ _airbyte_data is likely stable but there are no guarantees that other\
+ \ metadata columns will remain the same in future versions"
+ title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
+ \ table schema might change between versions)"
+ order: 8
+ group: "advanced"
+ destinationType:
+ title: "bigquery"
+ const: "bigquery"
+ enum:
+ - "bigquery"
+ order: 0
type: "string"
- default: ""
- description: "The region of the S3 bucket. See here for all region codes."
+ groups:
+ - id: "connection"
+ title: "Connection"
+ - id: "advanced"
+ title: "Advanced"
+ destination-bigquery-update:
+ title: "BigQuery Destination Spec"
+ type: "object"
+ required:
+ - "project_id"
+ - "dataset_location"
+ - "dataset_id"
+ properties:
+ project_id:
+ type: "string"
+ description: "The GCP project ID for the project containing the target BigQuery\
+ \ dataset. Read more here."
+ title: "Project ID"
+ group: "connection"
+ order: 0
+ dataset_location:
+ type: "string"
+ description: "The location of the dataset. Warning: Changes made after creation\
+ \ will not be applied. Read more here."
+ title: "Dataset Location"
+ group: "connection"
+ order: 1
enum:
- - ""
- - "af-south-1"
- - "ap-east-1"
- - "ap-northeast-1"
- - "ap-northeast-2"
- - "ap-northeast-3"
- - "ap-south-1"
- - "ap-south-2"
- - "ap-southeast-1"
- - "ap-southeast-2"
- - "ap-southeast-3"
- - "ap-southeast-4"
- - "ca-central-1"
- - "ca-west-1"
- - "cn-north-1"
- - "cn-northwest-1"
- - "eu-central-1"
- - "eu-central-2"
- - "eu-north-1"
- - "eu-south-1"
- - "eu-south-2"
- - "eu-west-1"
- - "eu-west-2"
- - "eu-west-3"
- - "il-central-1"
- - "me-central-1"
- - "me-south-1"
- - "sa-east-1"
- - "us-east-1"
- - "us-east-2"
- - "us-gov-east-1"
- - "us-gov-west-1"
- - "us-west-1"
- - "us-west-2"
- order: 4
- format:
- title: "Output Format"
+ - "US"
+ - "EU"
+ - "asia-east1"
+ - "asia-east2"
+ - "asia-northeast1"
+ - "asia-northeast2"
+ - "asia-northeast3"
+ - "asia-south1"
+ - "asia-south2"
+ - "asia-southeast1"
+ - "asia-southeast2"
+ - "australia-southeast1"
+ - "australia-southeast2"
+ - "europe-central1"
+ - "europe-central2"
+ - "europe-north1"
+ - "europe-southwest1"
+ - "europe-west1"
+ - "europe-west2"
+ - "europe-west3"
+ - "europe-west4"
+ - "europe-west6"
+ - "europe-west7"
+ - "europe-west8"
+ - "europe-west9"
+ - "europe-west12"
+ - "me-central1"
+ - "me-central2"
+ - "me-west1"
+ - "northamerica-northeast1"
+ - "northamerica-northeast2"
+ - "southamerica-east1"
+ - "southamerica-west1"
+ - "us-central1"
+ - "us-east1"
+ - "us-east2"
+ - "us-east3"
+ - "us-east4"
+ - "us-east5"
+ - "us-south1"
+ - "us-west1"
+ - "us-west2"
+ - "us-west3"
+ - "us-west4"
+ dataset_id:
+ type: "string"
+ description: "The default BigQuery Dataset ID that tables are replicated\
+ \ to if the source does not specify a namespace. Read more here."
+ title: "Default Dataset ID"
+ group: "connection"
+ order: 2
+ loading_method:
type: "object"
- description: "Format of the data output. See here for more details"
+ title: "Loading Method"
+ description: "The way data will be uploaded to BigQuery."
+ display_type: "radio"
+ group: "connection"
+ order: 3
oneOf:
- - title: "CSV: Comma-Separated Values"
+ - title: "GCS Staging"
+ description: "(recommended) Writes large batches of records to\
+ \ a file, uploads the file to GCS, then uses COPY INTO to load your\
+ \ data into BigQuery. Provides best-in-class speed, reliability and\
+ \ scalability. Read more about GCS Staging here."
required:
- - "format_type"
- - "flattening"
+ - "method"
+ - "gcs_bucket_name"
+ - "gcs_bucket_path"
+ - "credential"
properties:
- format_type:
- title: "Format Type"
- type: "string"
- enum:
- - "CSV"
- default: "CSV"
- flattening:
+ method:
type: "string"
- title: "Flattening"
- description: "Whether the input json data should be normalized (flattened)\
- \ in the output CSV. Please refer to docs for details."
- default: "No flattening"
+ const: "GCS Staging"
+ order: 0
enum:
- - "No flattening"
- - "Root level flattening"
- compression:
- title: "Compression"
+ - "GCS Staging"
+ credential:
+ title: "Credential"
+ description: "An HMAC key is a type of credential and can be associated\
+ \ with a service account or a user account in Cloud Storage. Read\
+ \ more here."
type: "object"
- description: "Whether the output files should be compressed. If compression\
- \ is selected, the output filename will have an extra extension\
- \ (GZIP: \".csv.gz\")."
+ order: 1
oneOf:
- - title: "No Compression"
- requires:
- - "compression_type"
+ - title: "HMAC key"
+ required:
+ - "credential_type"
+ - "hmac_key_access_id"
+ - "hmac_key_secret"
properties:
- compression_type:
+ credential_type:
type: "string"
+ const: "HMAC_KEY"
+ order: 0
enum:
- - "No Compression"
- default: "No Compression"
- - title: "GZIP"
- requires:
- - "compression_type"
- properties:
- compression_type:
+ - "HMAC_KEY"
+ hmac_key_access_id:
type: "string"
- enum:
- - "GZIP"
- default: "GZIP"
- - title: "JSON Lines: Newline-delimited JSON"
- required:
- - "format_type"
- properties:
- format_type:
- title: "Format Type"
+ description: "HMAC key access ID. When linked to a service account,\
+ \ this ID is 61 characters long; when linked to a user account,\
+ \ it is 24 characters long."
+ title: "HMAC Key Access ID"
+ airbyte_secret: true
+ examples:
+ - "1234567890abcdefghij1234"
+ order: 1
+ hmac_key_secret:
+ type: "string"
+ description: "The corresponding secret for the access ID. It\
+ \ is a 40-character base-64 encoded string."
+ title: "HMAC Key Secret"
+ airbyte_secret: true
+ examples:
+ - "1234567890abcdefghij1234567890ABCDEFGHIJ"
+ order: 2
+ gcs_bucket_name:
+ title: "GCS Bucket Name"
type: "string"
- enum:
- - "JSONL"
- default: "JSONL"
- flattening:
+ description: "The name of the GCS bucket. Read more here."
+ examples:
+ - "airbyte_sync"
+ order: 2
+ gcs_bucket_path:
+ title: "GCS Bucket Path"
+ description: "Directory under the GCS bucket where data will be written."
type: "string"
- title: "Flattening"
- description: "Whether the input json data should be normalized (flattened)\
- \ in the output JSON Lines. Please refer to docs for details."
- default: "No flattening"
- enum:
- - "No flattening"
- - "Root level flattening"
- compression:
- title: "Compression"
- type: "object"
- description: "Whether the output files should be compressed. If compression\
- \ is selected, the output filename will have an extra extension\
- \ (GZIP: \".jsonl.gz\")."
- oneOf:
- - title: "No Compression"
- requires: "compression_type"
- properties:
- compression_type:
- type: "string"
- enum:
- - "No Compression"
- default: "No Compression"
- - title: "GZIP"
- requires: "compression_type"
- properties:
- compression_type:
- type: "string"
- enum:
- - "GZIP"
- default: "GZIP"
- - title: "Avro: Apache Avro"
- required:
- - "format_type"
- - "compression_codec"
- properties:
- format_type:
- title: "Format Type"
+ examples:
+ - "data_sync/test"
+ order: 3
+ keep_files_in_gcs-bucket:
type: "string"
+ description: "This upload method is supposed to temporary store records\
+ \ in GCS bucket. By this select you can chose if these records should\
+ \ be removed from GCS when migration has finished. The default \"\
+ Delete all tmp files from GCS\" value is used if not set explicitly."
+ title: "GCS Tmp Files Afterward Processing"
+ default: "Delete all tmp files from GCS"
enum:
- - "Avro"
- default: "Avro"
- order: 0
- compression_codec:
- title: "Compression Codec"
- description: "The compression algorithm used to compress data. Default\
- \ to no compression."
- type: "object"
- oneOf:
- - title: "No Compression"
- required:
- - "codec"
- properties:
- codec:
- type: "string"
- enum:
- - "no compression"
- default: "no compression"
- - title: "Deflate"
- required:
- - "codec"
- - "compression_level"
- properties:
- codec:
- type: "string"
- enum:
- - "Deflate"
- default: "Deflate"
- compression_level:
- title: "Deflate Level"
- description: "0: no compression & fastest, 9: best compression\
- \ & slowest."
- type: "integer"
- default: 0
- minimum: 0
- maximum: 9
- - title: "bzip2"
- required:
- - "codec"
- properties:
- codec:
- type: "string"
- enum:
- - "bzip2"
- default: "bzip2"
- - title: "xz"
- required:
- - "codec"
- - "compression_level"
- properties:
- codec:
- type: "string"
- enum:
- - "xz"
- default: "xz"
- compression_level:
- title: "Compression Level"
- description: "See here for details."
- type: "integer"
- default: 6
- minimum: 0
- maximum: 9
- - title: "zstandard"
- required:
- - "codec"
- - "compression_level"
- properties:
- codec:
- type: "string"
- enum:
- - "zstandard"
- default: "zstandard"
- compression_level:
- title: "Compression Level"
- description: "Negative levels are 'fast' modes akin to lz4 or\
- \ snappy, levels above 9 are generally for archival purposes,\
- \ and levels above 18 use a lot of memory."
- type: "integer"
- default: 3
- minimum: -5
- maximum: 22
- include_checksum:
- title: "Include Checksum"
- description: "If true, include a checksum with each data block."
- type: "boolean"
- default: false
- - title: "snappy"
- required:
- - "codec"
- properties:
- codec:
- type: "string"
- enum:
- - "snappy"
- default: "snappy"
- order: 1
- - title: "Parquet: Columnar Storage"
+ - "Delete all tmp files from GCS"
+ - "Keep all tmp files in GCS"
+ order: 4
+ - title: "Standard Inserts"
required:
- - "format_type"
+ - "method"
+ description: "(not recommended) Direct loading using SQL INSERT\
+ \ statements. This method is extremely inefficient and provided only\
+ \ for quick testing. In all other cases, you should use GCS staging."
properties:
- format_type:
- title: "Format Type"
- type: "string"
- enum:
- - "Parquet"
- default: "Parquet"
- compression_codec:
- title: "Compression Codec"
- description: "The compression algorithm used to compress data pages."
+ method:
type: "string"
+ const: "Standard"
enum:
- - "UNCOMPRESSED"
- - "SNAPPY"
- - "GZIP"
- - "LZO"
- - "BROTLI"
- - "LZ4"
- - "ZSTD"
- default: "UNCOMPRESSED"
- block_size_mb:
- title: "Block Size (Row Group Size) (MB)"
- description: "This is the size of a row group being buffered in memory.\
- \ It limits the memory usage when writing. Larger values will improve\
- \ the IO when reading, but consume more memory when writing. Default:\
- \ 128 MB."
- type: "integer"
- default: 128
- examples:
- - 128
- max_padding_size_mb:
- title: "Max Padding Size (MB)"
- description: "Maximum size allowed as padding to align row groups.\
- \ This is also the minimum size of a row group. Default: 8 MB."
- type: "integer"
- default: 8
- examples:
- - 8
- page_size_kb:
- title: "Page Size (KB)"
- description: "The page size is for compression. A block is composed\
- \ of pages. A page is the smallest unit that must be read fully\
- \ to access a single record. If this value is too small, the compression\
- \ will deteriorate. Default: 1024 KB."
- type: "integer"
- default: 1024
- examples:
- - 1024
- dictionary_page_size_kb:
- title: "Dictionary Page Size (KB)"
- description: "There is one dictionary page per column per row group\
- \ when dictionary encoding is used. The dictionary page size works\
- \ like the page size but for dictionary. Default: 1024 KB."
- type: "integer"
- default: 1024
- examples:
- - 1024
- dictionary_encoding:
- title: "Dictionary Encoding"
- description: "Default: true."
- type: "boolean"
- default: true
- order: 5
- s3_endpoint:
- title: "Endpoint"
+ - "Standard"
+ credentials_json:
type: "string"
- default: ""
- description: "Your S3 endpoint url. Read more here"
+ description: "The contents of the JSON service account key. Check out the\
+ \ docs if you need help generating this key. Default credentials will\
+ \ be used if this field is left empty."
+ title: "Service Account Key JSON (Required for cloud, optional for open-source)"
+ airbyte_secret: true
+ group: "connection"
+ order: 4
+ always_show: true
+ transformation_priority:
+ type: "string"
+ description: "Interactive run type means that the query is executed as soon\
+ \ as possible, and these queries count towards concurrent rate limit and\
+ \ daily limit. Read more about interactive run type here. Batch queries are queued and started as soon as idle resources\
+ \ are available in the BigQuery shared resource pool, which usually occurs\
+ \ within a few minutes. Batch queries don’t count towards your concurrent\
+ \ rate limit. Read more about batch queries here. The default \"interactive\" value is used if not set explicitly."
+ title: "Transformation Query Run Type"
+ default: "interactive"
+ enum:
+ - "interactive"
+ - "batch"
+ order: 5
+ group: "advanced"
+ big_query_client_buffer_size_mb:
+ title: "Google BigQuery Client Chunk Size"
+ description: "Google BigQuery client's chunk (buffer) size (MIN=1, MAX =\
+ \ 15) for each table. The size that will be written by a single RPC. Written\
+ \ data will be buffered and only flushed upon reaching this size or closing\
+ \ the channel. The default 15MB value is used if not set explicitly. Read\
+ \ more here."
+ type: "integer"
+ minimum: 1
+ maximum: 15
+ default: 15
examples:
- - "http://localhost:9000"
+ - "15"
order: 6
- s3_path_format:
- title: "S3 Path Format"
- description: "Format string on how data will be organized inside the S3\
- \ bucket directory. Read more here"
+ group: "advanced"
+ raw_data_dataset:
type: "string"
- examples:
- - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
+ description: "The dataset to write raw tables into (default: airbyte_internal)"
+ title: "Raw Table Dataset Name"
order: 7
- file_name_pattern:
- type: "string"
- description: "The pattern allows you to set the file-name format for the\
- \ S3 staging file(s)"
- title: "S3 Filename pattern"
- examples:
- - "{date}"
- - "{date:yyyy_MM}"
- - "{timestamp}"
- - "{part_number}"
- - "{sync_id}"
+ group: "advanced"
+ disable_type_dedupe:
+ type: "boolean"
+ default: false
+ description: "Disable Writing Final Tables. WARNING! The data format in\
+ \ _airbyte_data is likely stable but there are no guarantees that other\
+ \ metadata columns will remain the same in future versions"
+ title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
+ \ table schema might change between versions)"
order: 8
+ group: "advanced"
+ groups:
+ - id: "connection"
+ title: "Connection"
+ - id: "advanced"
+ title: "Advanced"
+ destination-elasticsearch:
+ title: "Elasticsearch Connection Configuration"
+ type: "object"
+ required:
+ - "endpoint"
+ - "destinationType"
+ properties:
+ endpoint:
+ title: "Server Endpoint"
+ type: "string"
+ description: "The full url of the Elasticsearch server"
+ upsert:
+ type: "boolean"
+ title: "Upsert Records"
+ description: "If a primary key identifier is defined in the source, an upsert\
+ \ will be performed using the primary key value as the elasticsearch doc\
+ \ id. Does not support composite primary keys."
+ default: true
+ ca_certificate:
+ type: "string"
+ title: "CA certificate"
+ description: "CA certificate"
+ airbyte_secret: true
+ multiline: true
+ authenticationMethod:
+ title: "Authentication Method"
+ type: "object"
+ description: "The type of authentication to be used"
+ oneOf:
+ - title: "Api Key/Secret"
+ additionalProperties: false
+ description: "Use a api key and secret combination to authenticate"
+ required:
+ - "method"
+ - "apiKeyId"
+ - "apiKeySecret"
+ properties:
+ method:
+ type: "string"
+ const: "secret"
+ enum:
+ - "secret"
+ apiKeyId:
+ title: "API Key ID"
+ description: "The Key ID to used when accessing an enterprise Elasticsearch\
+ \ instance."
+ type: "string"
+ apiKeySecret:
+ title: "API Key Secret"
+ description: "The secret associated with the API Key ID."
+ type: "string"
+ airbyte_secret: true
+ - title: "Username/Password"
+ additionalProperties: false
+ description: "Basic auth header with a username and password"
+ required:
+ - "method"
+ - "username"
+ - "password"
+ properties:
+ method:
+ type: "string"
+ const: "basic"
+ enum:
+ - "basic"
+ username:
+ title: "Username"
+ description: "Basic auth username to access a secure Elasticsearch\
+ \ server"
+ type: "string"
+ password:
+ title: "Password"
+ description: "Basic auth password to access a secure Elasticsearch\
+ \ server"
+ type: "string"
+ airbyte_secret: true
+ x-speakeasy-param-sensitive: true
destinationType:
- title: "s3"
- const: "s3"
+ title: "elasticsearch"
+ const: "elasticsearch"
enum:
- - "s3"
+ - "elasticsearch"
order: 0
type: "string"
- destination-s3-update:
- title: "S3 Destination Spec"
+ destination-elasticsearch-update:
+ title: "Elasticsearch Connection Configuration"
+ type: "object"
+ required:
+ - "endpoint"
+ properties:
+ endpoint:
+ title: "Server Endpoint"
+ type: "string"
+ description: "The full url of the Elasticsearch server"
+ upsert:
+ type: "boolean"
+ title: "Upsert Records"
+ description: "If a primary key identifier is defined in the source, an upsert\
+ \ will be performed using the primary key value as the elasticsearch doc\
+ \ id. Does not support composite primary keys."
+ default: true
+ ca_certificate:
+ type: "string"
+ title: "CA certificate"
+ description: "CA certificate"
+ airbyte_secret: true
+ multiline: true
+ authenticationMethod:
+ title: "Authentication Method"
+ type: "object"
+ description: "The type of authentication to be used"
+ oneOf:
+ - title: "Api Key/Secret"
+ additionalProperties: false
+ description: "Use a api key and secret combination to authenticate"
+ required:
+ - "method"
+ - "apiKeyId"
+ - "apiKeySecret"
+ properties:
+ method:
+ type: "string"
+ const: "secret"
+ enum:
+ - "secret"
+ apiKeyId:
+ title: "API Key ID"
+ description: "The Key ID to used when accessing an enterprise Elasticsearch\
+ \ instance."
+ type: "string"
+ apiKeySecret:
+ title: "API Key Secret"
+ description: "The secret associated with the API Key ID."
+ type: "string"
+ airbyte_secret: true
+ - title: "Username/Password"
+ additionalProperties: false
+ description: "Basic auth header with a username and password"
+ required:
+ - "method"
+ - "username"
+ - "password"
+ properties:
+ method:
+ type: "string"
+ const: "basic"
+ enum:
+ - "basic"
+ username:
+ title: "Username"
+ description: "Basic auth username to access a secure Elasticsearch\
+ \ server"
+ type: "string"
+ password:
+ title: "Password"
+ description: "Basic auth password to access a secure Elasticsearch\
+ \ server"
+ type: "string"
+ airbyte_secret: true
+ destination-azure-blob-storage:
+ title: "AzureBlobStorage Destination Spec"
type: "object"
required:
- - "s3_bucket_name"
- - "s3_bucket_path"
- - "s3_bucket_region"
+ - "azure_blob_storage_account_name"
+ - "azure_blob_storage_account_key"
- "format"
+ - "destinationType"
properties:
- access_key_id:
+ azure_blob_storage_endpoint_domain_name:
+ title: "Endpoint Domain Name"
type: "string"
- description: "The access key ID to access the S3 bucket. Airbyte requires\
- \ Read and Write permissions to the given bucket. Read more here."
- title: "S3 Key ID"
- airbyte_secret: true
- always_show: true
+ default: "blob.core.windows.net"
+ description: "This is Azure Blob Storage endpoint domain name. Leave default\
+ \ value (or leave it empty if run container from command line) to use\
+ \ Microsoft native from example."
examples:
- - "A012345678910EXAMPLE"
- order: 0
- secret_access_key:
+ - "blob.core.windows.net"
+ azure_blob_storage_container_name:
+ title: "Azure blob storage container (Bucket) Name"
type: "string"
- description: "The corresponding secret to the access key ID. Read more here"
- title: "S3 Access Key"
- airbyte_secret: true
- always_show: true
+ description: "The name of the Azure blob storage container. If not exists\
+ \ - will be created automatically. May be empty, then will be created\
+ \ automatically airbytecontainer+timestamp"
examples:
- - "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
- order: 1
- s3_bucket_name:
- title: "S3 Bucket Name"
+ - "airbytetescontainername"
+ azure_blob_storage_account_name:
+ title: "Azure Blob Storage account name"
type: "string"
- description: "The name of the S3 bucket. Read more here."
+ description: "The account's name of the Azure Blob Storage."
examples:
- - "airbyte_sync"
- order: 2
- s3_bucket_path:
- title: "S3 Bucket Path"
- description: "Directory under the S3 bucket where data will be written.\
- \ Read more here"
+ - "airbyte5storage"
+ azure_blob_storage_account_key:
+ title: "Azure Blob Storage account key"
+ description: "The Azure blob storage account key."
+ airbyte_secret: true
type: "string"
examples:
- - "data_sync/test"
- order: 3
- s3_bucket_region:
- title: "S3 Bucket Region"
- type: "string"
- default: ""
- description: "The region of the S3 bucket. See here for all region codes."
- enum:
- - ""
- - "af-south-1"
- - "ap-east-1"
- - "ap-northeast-1"
- - "ap-northeast-2"
- - "ap-northeast-3"
- - "ap-south-1"
- - "ap-south-2"
- - "ap-southeast-1"
- - "ap-southeast-2"
- - "ap-southeast-3"
- - "ap-southeast-4"
- - "ca-central-1"
- - "ca-west-1"
- - "cn-north-1"
- - "cn-northwest-1"
- - "eu-central-1"
- - "eu-central-2"
- - "eu-north-1"
- - "eu-south-1"
- - "eu-south-2"
- - "eu-west-1"
- - "eu-west-2"
- - "eu-west-3"
- - "il-central-1"
- - "me-central-1"
- - "me-south-1"
- - "sa-east-1"
- - "us-east-1"
- - "us-east-2"
- - "us-gov-east-1"
- - "us-gov-west-1"
- - "us-west-1"
- - "us-west-2"
- order: 4
+ - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
+ x-speakeasy-param-sensitive: true
+ azure_blob_storage_output_buffer_size:
+ title: "Azure Blob Storage output buffer size (Megabytes)"
+ type: "integer"
+ description: "The amount of megabytes to buffer for the output stream to\
+ \ Azure. This will impact memory footprint on workers, but may need adjustment\
+ \ for performance and appropriate block size in Azure."
+ minimum: 1
+ maximum: 2047
+ default: 5
+ examples:
+ - 5
+ azure_blob_storage_spill_size:
+ title: "Azure Blob Storage file spill size"
+ type: "integer"
+ description: "The amount of megabytes after which the connector should spill\
+ \ the records in a new blob object. Make sure to configure size greater\
+ \ than individual records. Enter 0 if not applicable"
+ default: 500
+ examples:
+ - 500
format:
title: "Output Format"
type: "object"
- description: "Format of the data output. See here for more details"
+ description: "Output data format"
oneOf:
- title: "CSV: Comma-Separated Values"
required:
@@ -70681,573 +73189,570 @@ components:
- "flattening"
properties:
format_type:
- title: "Format Type"
type: "string"
+ const: "CSV"
enum:
- "CSV"
- default: "CSV"
flattening:
type: "string"
- title: "Flattening"
+ title: "Normalization (Flattening)"
description: "Whether the input json data should be normalized (flattened)\
\ in the output CSV. Please refer to docs for details."
default: "No flattening"
enum:
- "No flattening"
- "Root level flattening"
- compression:
- title: "Compression"
- type: "object"
- description: "Whether the output files should be compressed. If compression\
- \ is selected, the output filename will have an extra extension\
- \ (GZIP: \".csv.gz\")."
- oneOf:
- - title: "No Compression"
- requires:
- - "compression_type"
- properties:
- compression_type:
- type: "string"
- enum:
- - "No Compression"
- default: "No Compression"
- - title: "GZIP"
- requires:
- - "compression_type"
- properties:
- compression_type:
- type: "string"
- enum:
- - "GZIP"
- default: "GZIP"
- - title: "JSON Lines: Newline-delimited JSON"
+ - title: "JSON Lines: newline-delimited JSON"
required:
- "format_type"
properties:
format_type:
- title: "Format Type"
type: "string"
+ const: "JSONL"
enum:
- "JSONL"
- default: "JSONL"
- flattening:
- type: "string"
- title: "Flattening"
- description: "Whether the input json data should be normalized (flattened)\
- \ in the output JSON Lines. Please refer to docs for details."
- default: "No flattening"
- enum:
- - "No flattening"
- - "Root level flattening"
- compression:
- title: "Compression"
- type: "object"
- description: "Whether the output files should be compressed. If compression\
- \ is selected, the output filename will have an extra extension\
- \ (GZIP: \".jsonl.gz\")."
- oneOf:
- - title: "No Compression"
- requires: "compression_type"
- properties:
- compression_type:
- type: "string"
- enum:
- - "No Compression"
- default: "No Compression"
- - title: "GZIP"
- requires: "compression_type"
- properties:
- compression_type:
- type: "string"
- enum:
- - "GZIP"
- default: "GZIP"
- - title: "Avro: Apache Avro"
- required:
- - "format_type"
- - "compression_codec"
- properties:
- format_type:
- title: "Format Type"
- type: "string"
- enum:
- - "Avro"
- default: "Avro"
- order: 0
- compression_codec:
- title: "Compression Codec"
- description: "The compression algorithm used to compress data. Default\
- \ to no compression."
- type: "object"
- oneOf:
- - title: "No Compression"
- required:
- - "codec"
- properties:
- codec:
- type: "string"
- enum:
- - "no compression"
- default: "no compression"
- - title: "Deflate"
- required:
- - "codec"
- - "compression_level"
- properties:
- codec:
- type: "string"
- enum:
- - "Deflate"
- default: "Deflate"
- compression_level:
- title: "Deflate Level"
- description: "0: no compression & fastest, 9: best compression\
- \ & slowest."
- type: "integer"
- default: 0
- minimum: 0
- maximum: 9
- - title: "bzip2"
- required:
- - "codec"
- properties:
- codec:
- type: "string"
- enum:
- - "bzip2"
- default: "bzip2"
- - title: "xz"
- required:
- - "codec"
- - "compression_level"
- properties:
- codec:
- type: "string"
- enum:
- - "xz"
- default: "xz"
- compression_level:
- title: "Compression Level"
- description: "See here for details."
- type: "integer"
- default: 6
- minimum: 0
- maximum: 9
- - title: "zstandard"
- required:
- - "codec"
- - "compression_level"
- properties:
- codec:
- type: "string"
- enum:
- - "zstandard"
- default: "zstandard"
- compression_level:
- title: "Compression Level"
- description: "Negative levels are 'fast' modes akin to lz4 or\
- \ snappy, levels above 9 are generally for archival purposes,\
- \ and levels above 18 use a lot of memory."
- type: "integer"
- default: 3
- minimum: -5
- maximum: 22
- include_checksum:
- title: "Include Checksum"
- description: "If true, include a checksum with each data block."
- type: "boolean"
- default: false
- - title: "snappy"
- required:
- - "codec"
- properties:
- codec:
- type: "string"
- enum:
- - "snappy"
- default: "snappy"
- order: 1
- - title: "Parquet: Columnar Storage"
- required:
- - "format_type"
- properties:
- format_type:
- title: "Format Type"
- type: "string"
- enum:
- - "Parquet"
- default: "Parquet"
- compression_codec:
- title: "Compression Codec"
- description: "The compression algorithm used to compress data pages."
- type: "string"
- enum:
- - "UNCOMPRESSED"
- - "SNAPPY"
- - "GZIP"
- - "LZO"
- - "BROTLI"
- - "LZ4"
- - "ZSTD"
- default: "UNCOMPRESSED"
- block_size_mb:
- title: "Block Size (Row Group Size) (MB)"
- description: "This is the size of a row group being buffered in memory.\
- \ It limits the memory usage when writing. Larger values will improve\
- \ the IO when reading, but consume more memory when writing. Default:\
- \ 128 MB."
- type: "integer"
- default: 128
- examples:
- - 128
- max_padding_size_mb:
- title: "Max Padding Size (MB)"
- description: "Maximum size allowed as padding to align row groups.\
- \ This is also the minimum size of a row group. Default: 8 MB."
- type: "integer"
- default: 8
- examples:
- - 8
- page_size_kb:
- title: "Page Size (KB)"
- description: "The page size is for compression. A block is composed\
- \ of pages. A page is the smallest unit that must be read fully\
- \ to access a single record. If this value is too small, the compression\
- \ will deteriorate. Default: 1024 KB."
- type: "integer"
- default: 1024
- examples:
- - 1024
- dictionary_page_size_kb:
- title: "Dictionary Page Size (KB)"
- description: "There is one dictionary page per column per row group\
- \ when dictionary encoding is used. The dictionary page size works\
- \ like the page size but for dictionary. Default: 1024 KB."
- type: "integer"
- default: 1024
- examples:
- - 1024
- dictionary_encoding:
- title: "Dictionary Encoding"
- description: "Default: true."
- type: "boolean"
- default: true
- order: 5
- s3_endpoint:
- title: "Endpoint"
- type: "string"
- default: ""
- description: "Your S3 endpoint url. Read more here"
- examples:
- - "http://localhost:9000"
- order: 6
- s3_path_format:
- title: "S3 Path Format"
- description: "Format string on how data will be organized inside the S3\
- \ bucket directory. Read more here"
- type: "string"
- examples:
- - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
- order: 7
- file_name_pattern:
+ destinationType:
+ title: "azure-blob-storage"
+ const: "azure-blob-storage"
+ enum:
+ - "azure-blob-storage"
+ order: 0
type: "string"
- description: "The pattern allows you to set the file-name format for the\
- \ S3 staging file(s)"
- title: "S3 Filename pattern"
- examples:
- - "{date}"
- - "{date:yyyy_MM}"
- - "{timestamp}"
- - "{part_number}"
- - "{sync_id}"
- order: 8
- destination-redis:
- title: "Redis Destination Spec"
+ destination-azure-blob-storage-update:
+ title: "AzureBlobStorage Destination Spec"
type: "object"
required:
- - "host"
- - "username"
- - "port"
- - "cache_type"
- - "destinationType"
+ - "azure_blob_storage_account_name"
+ - "azure_blob_storage_account_key"
+ - "format"
properties:
- host:
- title: "Host"
- description: "Redis host to connect to."
+ azure_blob_storage_endpoint_domain_name:
+ title: "Endpoint Domain Name"
type: "string"
+ default: "blob.core.windows.net"
+ description: "This is Azure Blob Storage endpoint domain name. Leave default\
+ \ value (or leave it empty if run container from command line) to use\
+ \ Microsoft native from example."
examples:
- - "localhost,127.0.0.1"
- order: 1
- port:
- title: "Port"
- description: "Port of Redis."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 6379
- order: 2
- username:
- title: "Username"
- description: "Username associated with Redis."
+ - "blob.core.windows.net"
+ azure_blob_storage_container_name:
+ title: "Azure blob storage container (Bucket) Name"
type: "string"
- order: 3
- password:
- title: "Password"
- description: "Password associated with Redis."
+ description: "The name of the Azure blob storage container. If not exists\
+ \ - will be created automatically. May be empty, then will be created\
+ \ automatically airbytecontainer+timestamp"
+ examples:
+ - "airbytetescontainername"
+ azure_blob_storage_account_name:
+ title: "Azure Blob Storage account name"
type: "string"
+ description: "The account's name of the Azure Blob Storage."
+ examples:
+ - "airbyte5storage"
+ azure_blob_storage_account_key:
+ title: "Azure Blob Storage account key"
+ description: "The Azure blob storage account key."
airbyte_secret: true
- order: 4
- x-speakeasy-param-sensitive: true
- ssl:
- title: "SSL Connection"
- type: "boolean"
- description: "Indicates whether SSL encryption protocol will be used to\
- \ connect to Redis. It is recommended to use SSL connection if possible."
- default: false
- order: 5
- ssl_mode:
- title: "SSL Modes"
- description: "SSL connection modes. \n verify-full - This is\
- \ the most secure mode. Always require encryption and verifies the identity\
- \ of the source database server"
+ type: "string"
+ examples:
+ - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
+ azure_blob_storage_output_buffer_size:
+ title: "Azure Blob Storage output buffer size (Megabytes)"
+ type: "integer"
+ description: "The amount of megabytes to buffer for the output stream to\
+ \ Azure. This will impact memory footprint on workers, but may need adjustment\
+ \ for performance and appropriate block size in Azure."
+ minimum: 1
+ maximum: 2047
+ default: 5
+ examples:
+ - 5
+ azure_blob_storage_spill_size:
+ title: "Azure Blob Storage file spill size"
+ type: "integer"
+ description: "The amount of megabytes after which the connector should spill\
+ \ the records in a new blob object. Make sure to configure size greater\
+ \ than individual records. Enter 0 if not applicable"
+ default: 500
+ examples:
+ - 500
+ format:
+ title: "Output Format"
type: "object"
- order: 6
+ description: "Output data format"
oneOf:
- - title: "disable"
- additionalProperties: false
- description: "Disable SSL."
+ - title: "CSV: Comma-Separated Values"
required:
- - "mode"
+ - "format_type"
+ - "flattening"
properties:
- mode:
+ format_type:
type: "string"
- const: "disable"
+ const: "CSV"
enum:
- - "disable"
- default: "disable"
- order: 0
- - title: "verify-full"
- additionalProperties: false
- description: "Verify-full SSL mode."
+ - "CSV"
+ flattening:
+ type: "string"
+ title: "Normalization (Flattening)"
+ description: "Whether the input json data should be normalized (flattened)\
+ \ in the output CSV. Please refer to docs for details."
+ default: "No flattening"
+ enum:
+ - "No flattening"
+ - "Root level flattening"
+ - title: "JSON Lines: newline-delimited JSON"
required:
- - "mode"
- - "ca_certificate"
- - "client_certificate"
- - "client_key"
+ - "format_type"
properties:
- mode:
+ format_type:
type: "string"
- const: "verify-full"
+ const: "JSONL"
enum:
- - "verify-full"
- default: "verify-full"
- order: 0
- ca_certificate:
+ - "JSONL"
+ destination-langchain:
+ title: "Langchain Destination Config"
+ type: "object"
+ properties:
+ processing:
+ title: "ProcessingConfigModel"
+ type: "object"
+ properties:
+ chunk_size:
+ title: "Chunk size"
+ description: "Size of chunks in tokens to store in vector store (make\
+ \ sure it is not too big for the context if your LLM)"
+ maximum: 8191
+ type: "integer"
+ chunk_overlap:
+ title: "Chunk overlap"
+ description: "Size of overlap between chunks in tokens to store in vector\
+ \ store to better capture relevant context"
+ default: 0
+ type: "integer"
+ text_fields:
+ title: "Text fields to embed"
+ description: "List of fields in the record that should be used to calculate\
+ \ the embedding. All other fields are passed along as meta fields.\
+ \ The field list is applied to all streams in the same way and non-existing\
+ \ fields are ignored. If none are defined, all fields are considered\
+ \ text fields. When specifying text fields, you can access nested\
+ \ fields in the record by using dot notation, e.g. `user.name` will\
+ \ access the `name` field in the `user` object. It's also possible\
+ \ to use wildcards to access all fields in an object, e.g. `users.*.name`\
+ \ will access all `names` fields in all entries of the `users` array."
+ always_show: true
+ examples:
+ - "text"
+ - "user.name"
+ - "users.*.name"
+ type: "array"
+ items:
type: "string"
- title: "CA Certificate"
- description: "CA certificate"
- airbyte_secret: true
- multiline: true
- order: 1
- client_certificate:
+ required:
+ - "chunk_size"
+ - "text_fields"
+ group: "processing"
+ embedding:
+ title: "Embedding"
+ description: "Embedding configuration"
+ group: "embedding"
+ type: "object"
+ oneOf:
+ - title: "OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "openai"
+ const: "openai"
+ enum:
+ - "openai"
type: "string"
- title: "Client Certificate"
- description: "Client certificate"
+ openai_key:
+ title: "OpenAI API key"
airbyte_secret: true
- multiline: true
- order: 2
- client_key:
type: "string"
- title: "Client Key"
- description: "Client key"
- airbyte_secret: true
- multiline: true
- order: 3
x-speakeasy-param-sensitive: true
- client_key_password:
+ required:
+ - "openai_key"
+ description: "Use the OpenAI API to embed text. This option is using the\
+ \ text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "Fake"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "fake"
+ const: "fake"
+ enum:
+ - "fake"
type: "string"
- title: "Client key password"
- description: "Password for keystorage. If you do not add it - the\
- \ password will be generated automatically."
- airbyte_secret: true
- order: 4
- x-speakeasy-param-sensitive: true
- cache_type:
- title: "Cache type"
- type: "string"
- default: "hash"
- description: "Redis cache type to store data in."
- enum:
- - "hash"
- order: 7
- tunnel_method:
+ description: "Use a fake embedding made out of random vectors with 1536\
+ \ embedding dimensions. This is useful for testing the data pipeline\
+ \ without incurring any costs."
+ indexing:
+ title: "Indexing"
+ description: "Indexing configuration"
+ group: "indexing"
type: "object"
- title: "SSH Tunnel Method"
- description: "Whether to initiate an SSH tunnel before connecting to the\
- \ database, and if so, which kind of authentication to use."
oneOf:
- - title: "No Tunnel"
- required:
- - "tunnel_method"
+ - title: "Pinecone"
+ type: "object"
properties:
- tunnel_method:
- description: "No ssh tunnel needed to connect to database"
- type: "string"
- const: "NO_TUNNEL"
- order: 0
+ mode:
+ title: "Mode"
+ default: "pinecone"
+ const: "pinecone"
enum:
- - "NO_TUNNEL"
- - title: "SSH Key Authentication"
+ - "pinecone"
+ type: "string"
+ pinecone_key:
+ title: "Pinecone API key"
+ airbyte_secret: true
+ type: "string"
+ x-speakeasy-param-sensitive: true
+ pinecone_environment:
+ title: "Pinecone environment"
+ description: "Pinecone environment to use"
+ type: "string"
+ index:
+ title: "Index"
+ description: "Pinecone index to use"
+ type: "string"
required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "ssh_key"
+ - "pinecone_key"
+ - "pinecone_environment"
+ - "index"
+ description: "Pinecone is a popular vector store that can be used to store\
+ \ and retrieve embeddings. It is a managed service and can also be queried\
+ \ from outside of langchain."
+ - title: "DocArrayHnswSearch"
+ type: "object"
properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and ssh key"
+ mode:
+ title: "Mode"
+ default: "DocArrayHnswSearch"
+ const: "DocArrayHnswSearch"
+ enum:
+ - "DocArrayHnswSearch"
type: "string"
- const: "SSH_KEY_AUTH"
- order: 0
+ destination_path:
+ title: "Destination Path"
+ description: "Path to the directory where hnswlib and meta data files\
+ \ will be written. The files will be placed inside that local mount.\
+ \ All files in the specified destination directory will be deleted\
+ \ on each run."
+ examples:
+ - "/local/my_hnswlib_index"
+ type: "string"
+ required:
+ - "destination_path"
+ description: "DocArrayHnswSearch is a lightweight Document Index implementation\
+ \ provided by Docarray that runs fully locally and is best suited for\
+ \ small- to medium-sized datasets. It stores vectors on disk in hnswlib,\
+ \ and stores all other data in SQLite."
+ - title: "Chroma (local persistance)"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "chroma_local"
+ const: "chroma_local"
enum:
- - "SSH_KEY_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
+ - "chroma_local"
type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
+ destination_path:
+ title: "Destination Path"
+ description: "Path to the directory where chroma files will be written.\
+ \ The files will be placed inside that local mount."
examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host."
+ - "/local/my_chroma_db"
type: "string"
- order: 3
- ssh_key:
- title: "SSH Private Key"
- description: "OS-level user account ssh key credentials in RSA PEM\
- \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
+ collection_name:
+ title: "Collection Name"
+ description: "Name of the collection to use."
+ default: "langchain"
+ type: "string"
+ required:
+ - "destination_path"
+ description: "Chroma is a popular vector store that can be used to store\
+ \ and retrieve embeddings. It will build its index in memory and persist\
+ \ it to disk by the end of the sync."
+ destinationType:
+ title: "langchain"
+ const: "langchain"
+ enum:
+ - "langchain"
+ order: 0
+ type: "string"
+ required:
+ - "processing"
+ - "embedding"
+ - "indexing"
+ - "destinationType"
+ groups:
+ - id: "processing"
+ title: "Processing"
+ - id: "embedding"
+ title: "Embedding"
+ - id: "indexing"
+ title: "Indexing"
+ destination-langchain-update:
+ title: "Langchain Destination Config"
+ type: "object"
+ properties:
+ processing:
+ title: "ProcessingConfigModel"
+ type: "object"
+ properties:
+ chunk_size:
+ title: "Chunk size"
+ description: "Size of chunks in tokens to store in vector store (make\
+ \ sure it is not too big for the context if your LLM)"
+ maximum: 8191
+ type: "integer"
+ chunk_overlap:
+ title: "Chunk overlap"
+ description: "Size of overlap between chunks in tokens to store in vector\
+ \ store to better capture relevant context"
+ default: 0
+ type: "integer"
+ text_fields:
+ title: "Text fields to embed"
+ description: "List of fields in the record that should be used to calculate\
+ \ the embedding. All other fields are passed along as meta fields.\
+ \ The field list is applied to all streams in the same way and non-existing\
+ \ fields are ignored. If none are defined, all fields are considered\
+ \ text fields. When specifying text fields, you can access nested\
+ \ fields in the record by using dot notation, e.g. `user.name` will\
+ \ access the `name` field in the `user` object. It's also possible\
+ \ to use wildcards to access all fields in an object, e.g. `users.*.name`\
+ \ will access all `names` fields in all entries of the `users` array."
+ always_show: true
+ examples:
+ - "text"
+ - "user.name"
+ - "users.*.name"
+ type: "array"
+ items:
+ type: "string"
+ required:
+ - "chunk_size"
+ - "text_fields"
+ group: "processing"
+ embedding:
+ title: "Embedding"
+ description: "Embedding configuration"
+ group: "embedding"
+ type: "object"
+ oneOf:
+ - title: "OpenAI"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "openai"
+ const: "openai"
+ enum:
+ - "openai"
type: "string"
+ openai_key:
+ title: "OpenAI API key"
airbyte_secret: true
- multiline: true
- order: 4
- x-speakeasy-param-sensitive: true
- - title: "Password Authentication"
+ type: "string"
required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "tunnel_user_password"
+ - "openai_key"
+ description: "Use the OpenAI API to embed text. This option is using the\
+ \ text-embedding-ada-002 model with 1536 embedding dimensions."
+ - title: "Fake"
+ type: "object"
properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and password authentication"
+ mode:
+ title: "Mode"
+ default: "fake"
+ const: "fake"
+ enum:
+ - "fake"
type: "string"
- const: "SSH_PASSWORD_AUTH"
- order: 0
+ description: "Use a fake embedding made out of random vectors with 1536\
+ \ embedding dimensions. This is useful for testing the data pipeline\
+ \ without incurring any costs."
+ indexing:
+ title: "Indexing"
+ description: "Indexing configuration"
+ group: "indexing"
+ type: "object"
+ oneOf:
+ - title: "Pinecone"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "pinecone"
+ const: "pinecone"
enum:
- - "SSH_PASSWORD_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
+ - "pinecone"
type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
+ pinecone_key:
+ title: "Pinecone API key"
+ airbyte_secret: true
+ type: "string"
+ pinecone_environment:
+ title: "Pinecone environment"
+ description: "Pinecone environment to use"
+ type: "string"
+ index:
+ title: "Index"
+ description: "Pinecone index to use"
+ type: "string"
+ required:
+ - "pinecone_key"
+ - "pinecone_environment"
+ - "index"
+ description: "Pinecone is a popular vector store that can be used to store\
+ \ and retrieve embeddings. It is a managed service and can also be queried\
+ \ from outside of langchain."
+ - title: "DocArrayHnswSearch"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "DocArrayHnswSearch"
+ const: "DocArrayHnswSearch"
+ enum:
+ - "DocArrayHnswSearch"
+ type: "string"
+ destination_path:
+ title: "Destination Path"
+ description: "Path to the directory where hnswlib and meta data files\
+ \ will be written. The files will be placed inside that local mount.\
+ \ All files in the specified destination directory will be deleted\
+ \ on each run."
+ examples:
+ - "/local/my_hnswlib_index"
+ type: "string"
+ required:
+ - "destination_path"
+ description: "DocArrayHnswSearch is a lightweight Document Index implementation\
+ \ provided by Docarray that runs fully locally and is best suited for\
+ \ small- to medium-sized datasets. It stores vectors on disk in hnswlib,\
+ \ and stores all other data in SQLite."
+ - title: "Chroma (local persistance)"
+ type: "object"
+ properties:
+ mode:
+ title: "Mode"
+ default: "chroma_local"
+ const: "chroma_local"
+ enum:
+ - "chroma_local"
+ type: "string"
+ destination_path:
+ title: "Destination Path"
+ description: "Path to the directory where chroma files will be written.\
+ \ The files will be placed inside that local mount."
examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host"
+ - "/local/my_chroma_db"
type: "string"
- order: 3
- tunnel_user_password:
- title: "Password"
- description: "OS-level password for logging into the jump server host"
+ collection_name:
+ title: "Collection Name"
+ description: "Name of the collection to use."
+ default: "langchain"
type: "string"
- airbyte_secret: true
- order: 4
- x-speakeasy-param-sensitive: true
- destinationType:
- title: "redis"
- const: "redis"
- enum:
- - "redis"
- order: 0
- type: "string"
- destination-redis-update:
- title: "Redis Destination Spec"
+ required:
+ - "destination_path"
+ description: "Chroma is a popular vector store that can be used to store\
+ \ and retrieve embeddings. It will build its index in memory and persist\
+ \ it to disk by the end of the sync."
+ required:
+ - "processing"
+ - "embedding"
+ - "indexing"
+ groups:
+ - id: "processing"
+ title: "Processing"
+ - id: "embedding"
+ title: "Embedding"
+ - id: "indexing"
+ title: "Indexing"
+ destination-yellowbrick:
+ title: "Yellowbrick Destination Spec"
type: "object"
required:
- "host"
- - "username"
- "port"
- - "cache_type"
+ - "username"
+ - "database"
+ - "schema"
+ - "destinationType"
properties:
host:
title: "Host"
- description: "Redis host to connect to."
+ description: "Hostname of the database."
type: "string"
- examples:
- - "localhost,127.0.0.1"
- order: 1
+ order: 0
port:
title: "Port"
- description: "Port of Redis."
+ description: "Port of the database."
type: "integer"
minimum: 0
maximum: 65536
- default: 6379
+ default: 5432
+ examples:
+ - "5432"
+ order: 1
+ database:
+ title: "DB Name"
+ description: "Name of the database."
+ type: "string"
order: 2
- username:
- title: "Username"
- description: "Username associated with Redis."
+ schema:
+ title: "Default Schema"
+ description: "The default schema tables are written to if the source does\
+ \ not specify a namespace. The usual value for this field is \"public\"\
+ ."
type: "string"
+ examples:
+ - "public"
+ default: "public"
order: 3
+ username:
+ title: "User"
+ description: "Username to use to access the database."
+ type: "string"
+ order: 4
password:
title: "Password"
- description: "Password associated with Redis."
+ description: "Password associated with the username."
type: "string"
airbyte_secret: true
- order: 4
+ order: 5
+ x-speakeasy-param-sensitive: true
ssl:
title: "SSL Connection"
+ description: "Encrypt data using SSL. When activating SSL, please select\
+ \ one of the connection modes."
type: "boolean"
- description: "Indicates whether SSL encryption protocol will be used to\
- \ connect to Redis. It is recommended to use SSL connection if possible."
default: false
- order: 5
+ order: 6
ssl_mode:
- title: "SSL Modes"
- description: "SSL connection modes. \n verify-full - This is\
- \ the most secure mode. Always require encryption and verifies the identity\
- \ of the source database server"
+ title: "SSL modes"
+ description: "SSL connection modes. \n disable - Chose this mode\
+ \ to disable encryption of communication between Airbyte and destination\
+ \ database\n allow - Chose this mode to enable encryption only\
+ \ when required by the source database\n prefer - Chose this mode\
+ \ to allow unencrypted connection only if the source database does not\
+ \ support encryption\n require - Chose this mode to always require\
+ \ encryption. If the source database server does not support encryption,\
+ \ connection will fail\n verify-ca - Chose this mode to always\
+ \ require encryption and to verify that the source database server has\
+ \ a valid SSL certificate\n verify-full - This is the most secure\
+ \ mode. Chose this mode to always require encryption and to verify the\
+ \ identity of the source database server\n See more information - in the\
+ \ docs."
type: "object"
- order: 6
+ order: 7
oneOf:
- title: "disable"
additionalProperties: false
@@ -71262,6 +73767,74 @@ components:
- "disable"
default: "disable"
order: 0
+ - title: "allow"
+ additionalProperties: false
+ description: "Allow SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "allow"
+ enum:
+ - "allow"
+ default: "allow"
+ order: 0
+ - title: "prefer"
+ additionalProperties: false
+ description: "Prefer SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "prefer"
+ enum:
+ - "prefer"
+ default: "prefer"
+ order: 0
+ - title: "require"
+ additionalProperties: false
+ description: "Require SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "require"
+ enum:
+ - "require"
+ default: "require"
+ order: 0
+ - title: "verify-ca"
+ additionalProperties: false
+ description: "Verify-ca SSL mode."
+ required:
+ - "mode"
+ - "ca_certificate"
+ properties:
+ mode:
+ type: "string"
+ const: "verify-ca"
+ enum:
+ - "verify-ca"
+ default: "verify-ca"
+ order: 0
+ ca_certificate:
+ type: "string"
+ title: "CA certificate"
+ description: "CA certificate"
+ airbyte_secret: true
+ multiline: true
+ order: 1
+ client_key_password:
+ type: "string"
+ title: "Client key password"
+ description: "Password for keystorage. This field is optional. If\
+ \ you do not add it - the password will be generated automatically."
+ airbyte_secret: true
+ order: 4
+ x-speakeasy-param-sensitive: true
- title: "verify-full"
additionalProperties: false
description: "Verify-full SSL mode."
@@ -71280,40 +73853,41 @@ components:
order: 0
ca_certificate:
type: "string"
- title: "CA Certificate"
+ title: "CA certificate"
description: "CA certificate"
airbyte_secret: true
multiline: true
order: 1
client_certificate:
type: "string"
- title: "Client Certificate"
+ title: "Client certificate"
description: "Client certificate"
airbyte_secret: true
multiline: true
order: 2
client_key:
type: "string"
- title: "Client Key"
+ title: "Client key"
description: "Client key"
airbyte_secret: true
multiline: true
order: 3
+ x-speakeasy-param-sensitive: true
client_key_password:
type: "string"
title: "Client key password"
- description: "Password for keystorage. If you do not add it - the\
- \ password will be generated automatically."
+ description: "Password for keystorage. This field is optional. If\
+ \ you do not add it - the password will be generated automatically."
airbyte_secret: true
order: 4
- cache_type:
- title: "Cache type"
+ x-speakeasy-param-sensitive: true
+ jdbc_url_params:
+ description: "Additional properties to pass to the JDBC URL string when\
+ \ connecting to the database formatted as 'key=value' pairs separated\
+ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+ title: "JDBC URL Params"
type: "string"
- default: "hash"
- description: "Redis cache type to store data in."
- enum:
- - "hash"
- order: 7
+ order: 8
tunnel_method:
type: "object"
title: "SSH Tunnel Method"
@@ -71362,1373 +73936,734 @@ components:
maximum: 65536
default: 22
examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host."
- type: "string"
- order: 3
- ssh_key:
- title: "SSH Private Key"
- description: "OS-level user account ssh key credentials in RSA PEM\
- \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
- type: "string"
- airbyte_secret: true
- multiline: true
- order: 4
- - title: "Password Authentication"
- required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "tunnel_user_password"
- properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and password authentication"
- type: "string"
- const: "SSH_PASSWORD_AUTH"
- order: 0
- enum:
- - "SSH_PASSWORD_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
- type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
- examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host"
- type: "string"
- order: 3
- tunnel_user_password:
- title: "Password"
- description: "OS-level password for logging into the jump server host"
- type: "string"
- airbyte_secret: true
- order: 4
- destination-typesense:
- title: "Destination Typesense"
- type: "object"
- required:
- - "api_key"
- - "host"
- - "destinationType"
- properties:
- api_key:
- title: "API Key"
- type: "string"
- description: "Typesense API Key"
- order: 0
- x-speakeasy-param-sensitive: true
- host:
- title: "Host"
- type: "string"
- description: "Hostname of the Typesense instance without protocol."
- order: 1
- port:
- title: "Port"
- type: "string"
- description: "Port of the Typesense instance. Ex: 8108, 80, 443. Default\
- \ is 443"
- order: 2
- protocol:
- title: "Protocol"
- type: "string"
- description: "Protocol of the Typesense instance. Ex: http or https. Default\
- \ is https"
- order: 3
- batch_size:
- title: "Batch size"
- type: "integer"
- description: "How many documents should be imported together. Default 1000"
- order: 4
- path:
- title: "Path"
- type: "string"
- description: "Path of the Typesense instance. Default is none"
- order: 5
- destinationType:
- title: "typesense"
- const: "typesense"
- enum:
- - "typesense"
- order: 0
- type: "string"
- destination-typesense-update:
- title: "Destination Typesense"
- type: "object"
- required:
- - "api_key"
- - "host"
- properties:
- api_key:
- title: "API Key"
- type: "string"
- description: "Typesense API Key"
- order: 0
- host:
- title: "Host"
- type: "string"
- description: "Hostname of the Typesense instance without protocol."
- order: 1
- port:
- title: "Port"
- type: "string"
- description: "Port of the Typesense instance. Ex: 8108, 80, 443. Default\
- \ is 443"
- order: 2
- protocol:
- title: "Protocol"
- type: "string"
- description: "Protocol of the Typesense instance. Ex: http or https. Default\
- \ is https"
- order: 3
- batch_size:
- title: "Batch size"
- type: "integer"
- description: "How many documents should be imported together. Default 1000"
- order: 4
- path:
- title: "Path"
- type: "string"
- description: "Path of the Typesense instance. Default is none"
- order: 5
- destination-bigquery:
- title: "BigQuery Destination Spec"
- type: "object"
- required:
- - "project_id"
- - "dataset_location"
- - "dataset_id"
- - "destinationType"
- properties:
- project_id:
- type: "string"
- description: "The GCP project ID for the project containing the target BigQuery\
- \ dataset. Read more here."
- title: "Project ID"
- group: "connection"
- order: 0
- dataset_location:
- type: "string"
- description: "The location of the dataset. Warning: Changes made after creation\
- \ will not be applied. Read more here."
- title: "Dataset Location"
- group: "connection"
- order: 1
- enum:
- - "US"
- - "EU"
- - "asia-east1"
- - "asia-east2"
- - "asia-northeast1"
- - "asia-northeast2"
- - "asia-northeast3"
- - "asia-south1"
- - "asia-south2"
- - "asia-southeast1"
- - "asia-southeast2"
- - "australia-southeast1"
- - "australia-southeast2"
- - "europe-central1"
- - "europe-central2"
- - "europe-north1"
- - "europe-southwest1"
- - "europe-west1"
- - "europe-west2"
- - "europe-west3"
- - "europe-west4"
- - "europe-west6"
- - "europe-west7"
- - "europe-west8"
- - "europe-west9"
- - "europe-west12"
- - "me-central1"
- - "me-central2"
- - "me-west1"
- - "northamerica-northeast1"
- - "northamerica-northeast2"
- - "southamerica-east1"
- - "southamerica-west1"
- - "us-central1"
- - "us-east1"
- - "us-east2"
- - "us-east3"
- - "us-east4"
- - "us-east5"
- - "us-south1"
- - "us-west1"
- - "us-west2"
- - "us-west3"
- - "us-west4"
- dataset_id:
- type: "string"
- description: "The default BigQuery Dataset ID that tables are replicated\
- \ to if the source does not specify a namespace. Read more here."
- title: "Default Dataset ID"
- group: "connection"
- order: 2
- loading_method:
- type: "object"
- title: "Loading Method"
- description: "The way data will be uploaded to BigQuery."
- display_type: "radio"
- group: "connection"
- order: 3
- oneOf:
- - title: "GCS Staging"
- description: "(recommended) Writes large batches of records to\
- \ a file, uploads the file to GCS, then uses COPY INTO to load your\
- \ data into BigQuery. Provides best-in-class speed, reliability and\
- \ scalability. Read more about GCS Staging here."
- required:
- - "method"
- - "gcs_bucket_name"
- - "gcs_bucket_path"
- - "credential"
- properties:
- method:
- type: "string"
- const: "GCS Staging"
- order: 0
- enum:
- - "GCS Staging"
- credential:
- title: "Credential"
- description: "An HMAC key is a type of credential and can be associated\
- \ with a service account or a user account in Cloud Storage. Read\
- \ more here."
- type: "object"
- order: 1
- oneOf:
- - title: "HMAC key"
- required:
- - "credential_type"
- - "hmac_key_access_id"
- - "hmac_key_secret"
- properties:
- credential_type:
- type: "string"
- const: "HMAC_KEY"
- order: 0
- enum:
- - "HMAC_KEY"
- hmac_key_access_id:
- type: "string"
- description: "HMAC key access ID. When linked to a service account,\
- \ this ID is 61 characters long; when linked to a user account,\
- \ it is 24 characters long."
- title: "HMAC Key Access ID"
- airbyte_secret: true
- examples:
- - "1234567890abcdefghij1234"
- order: 1
- x-speakeasy-param-sensitive: true
- hmac_key_secret:
- type: "string"
- description: "The corresponding secret for the access ID. It\
- \ is a 40-character base-64 encoded string."
- title: "HMAC Key Secret"
- airbyte_secret: true
- examples:
- - "1234567890abcdefghij1234567890ABCDEFGHIJ"
- order: 2
- x-speakeasy-param-sensitive: true
- gcs_bucket_name:
- title: "GCS Bucket Name"
- type: "string"
- description: "The name of the GCS bucket. Read more here."
- examples:
- - "airbyte_sync"
+ - "22"
order: 2
- gcs_bucket_path:
- title: "GCS Bucket Path"
- description: "Directory under the GCS bucket where data will be written."
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host."
type: "string"
- examples:
- - "data_sync/test"
order: 3
- keep_files_in_gcs-bucket:
+ ssh_key:
+ title: "SSH Private Key"
+ description: "OS-level user account ssh key credentials in RSA PEM\
+ \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
type: "string"
- description: "This upload method is supposed to temporary store records\
- \ in GCS bucket. By this select you can chose if these records should\
- \ be removed from GCS when migration has finished. The default \"\
- Delete all tmp files from GCS\" value is used if not set explicitly."
- title: "GCS Tmp Files Afterward Processing"
- default: "Delete all tmp files from GCS"
- enum:
- - "Delete all tmp files from GCS"
- - "Keep all tmp files in GCS"
+ airbyte_secret: true
+ multiline: true
order: 4
- - title: "Standard Inserts"
+ x-speakeasy-param-sensitive: true
+ - title: "Password Authentication"
required:
- - "method"
- description: "(not recommended) Direct loading using SQL INSERT\
- \ statements. This method is extremely inefficient and provided only\
- \ for quick testing. In all other cases, you should use GCS staging."
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "tunnel_user_password"
properties:
- method:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and password authentication"
type: "string"
- const: "Standard"
+ const: "SSH_PASSWORD_AUTH"
+ order: 0
enum:
- - "Standard"
- credentials_json:
- type: "string"
- description: "The contents of the JSON service account key. Check out the\
- \ docs if you need help generating this key. Default credentials will\
- \ be used if this field is left empty."
- title: "Service Account Key JSON (Required for cloud, optional for open-source)"
- airbyte_secret: true
- group: "connection"
- order: 4
- always_show: true
- transformation_priority:
- type: "string"
- description: "Interactive run type means that the query is executed as soon\
- \ as possible, and these queries count towards concurrent rate limit and\
- \ daily limit. Read more about interactive run type here. Batch queries are queued and started as soon as idle resources\
- \ are available in the BigQuery shared resource pool, which usually occurs\
- \ within a few minutes. Batch queries don’t count towards your concurrent\
- \ rate limit. Read more about batch queries here. The default \"interactive\" value is used if not set explicitly."
- title: "Transformation Query Run Type"
- default: "interactive"
- enum:
- - "interactive"
- - "batch"
- order: 5
- group: "advanced"
- big_query_client_buffer_size_mb:
- title: "Google BigQuery Client Chunk Size"
- description: "Google BigQuery client's chunk (buffer) size (MIN=1, MAX =\
- \ 15) for each table. The size that will be written by a single RPC. Written\
- \ data will be buffered and only flushed upon reaching this size or closing\
- \ the channel. The default 15MB value is used if not set explicitly. Read\
- \ more here."
- type: "integer"
- minimum: 1
- maximum: 15
- default: 15
- examples:
- - "15"
- order: 6
- group: "advanced"
- raw_data_dataset:
- type: "string"
- description: "The dataset to write raw tables into (default: airbyte_internal)"
- title: "Raw Table Dataset Name"
- order: 7
- group: "advanced"
- disable_type_dedupe:
- type: "boolean"
- default: false
- description: "Disable Writing Final Tables. WARNING! The data format in\
- \ _airbyte_data is likely stable but there are no guarantees that other\
- \ metadata columns will remain the same in future versions"
- title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
- \ table schema might change between versions)"
- order: 8
- group: "advanced"
+ - "SSH_PASSWORD_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
+ type: "string"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host"
+ type: "string"
+ order: 3
+ tunnel_user_password:
+ title: "Password"
+ description: "OS-level password for logging into the jump server host"
+ type: "string"
+ airbyte_secret: true
+ order: 4
+ x-speakeasy-param-sensitive: true
destinationType:
- title: "bigquery"
- const: "bigquery"
+ title: "yellowbrick"
+ const: "yellowbrick"
enum:
- - "bigquery"
+ - "yellowbrick"
order: 0
type: "string"
- groups:
- - id: "connection"
- title: "Connection"
- - id: "advanced"
- title: "Advanced"
- destination-bigquery-update:
- title: "BigQuery Destination Spec"
+ destination-yellowbrick-update:
+ title: "Yellowbrick Destination Spec"
type: "object"
required:
- - "project_id"
- - "dataset_location"
- - "dataset_id"
+ - "host"
+ - "port"
+ - "username"
+ - "database"
+ - "schema"
properties:
- project_id:
+ host:
+ title: "Host"
+ description: "Hostname of the database."
type: "string"
- description: "The GCP project ID for the project containing the target BigQuery\
- \ dataset. Read more here."
- title: "Project ID"
- group: "connection"
order: 0
- dataset_location:
- type: "string"
- description: "The location of the dataset. Warning: Changes made after creation\
- \ will not be applied. Read more here."
- title: "Dataset Location"
- group: "connection"
+ port:
+ title: "Port"
+ description: "Port of the database."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 5432
+ examples:
+ - "5432"
order: 1
- enum:
- - "US"
- - "EU"
- - "asia-east1"
- - "asia-east2"
- - "asia-northeast1"
- - "asia-northeast2"
- - "asia-northeast3"
- - "asia-south1"
- - "asia-south2"
- - "asia-southeast1"
- - "asia-southeast2"
- - "australia-southeast1"
- - "australia-southeast2"
- - "europe-central1"
- - "europe-central2"
- - "europe-north1"
- - "europe-southwest1"
- - "europe-west1"
- - "europe-west2"
- - "europe-west3"
- - "europe-west4"
- - "europe-west6"
- - "europe-west7"
- - "europe-west8"
- - "europe-west9"
- - "europe-west12"
- - "me-central1"
- - "me-central2"
- - "me-west1"
- - "northamerica-northeast1"
- - "northamerica-northeast2"
- - "southamerica-east1"
- - "southamerica-west1"
- - "us-central1"
- - "us-east1"
- - "us-east2"
- - "us-east3"
- - "us-east4"
- - "us-east5"
- - "us-south1"
- - "us-west1"
- - "us-west2"
- - "us-west3"
- - "us-west4"
- dataset_id:
+ database:
+ title: "DB Name"
+ description: "Name of the database."
type: "string"
- description: "The default BigQuery Dataset ID that tables are replicated\
- \ to if the source does not specify a namespace. Read more here."
- title: "Default Dataset ID"
- group: "connection"
order: 2
- loading_method:
- type: "object"
- title: "Loading Method"
- description: "The way data will be uploaded to BigQuery."
- display_type: "radio"
- group: "connection"
+ schema:
+ title: "Default Schema"
+ description: "The default schema tables are written to if the source does\
+ \ not specify a namespace. The usual value for this field is \"public\"\
+ ."
+ type: "string"
+ examples:
+ - "public"
+ default: "public"
order: 3
+ username:
+ title: "User"
+ description: "Username to use to access the database."
+ type: "string"
+ order: 4
+ password:
+ title: "Password"
+ description: "Password associated with the username."
+ type: "string"
+ airbyte_secret: true
+ order: 5
+ ssl:
+ title: "SSL Connection"
+ description: "Encrypt data using SSL. When activating SSL, please select\
+ \ one of the connection modes."
+ type: "boolean"
+ default: false
+ order: 6
+ ssl_mode:
+ title: "SSL modes"
+ description: "SSL connection modes. \n disable - Chose this mode\
+ \ to disable encryption of communication between Airbyte and destination\
+ \ database\n allow - Chose this mode to enable encryption only\
+ \ when required by the source database\n prefer - Chose this mode\
+ \ to allow unencrypted connection only if the source database does not\
+ \ support encryption\n require - Chose this mode to always require\
+ \ encryption. If the source database server does not support encryption,\
+ \ connection will fail\n verify-ca - Chose this mode to always\
+ \ require encryption and to verify that the source database server has\
+ \ a valid SSL certificate\n verify-full - This is the most secure\
+ \ mode. Chose this mode to always require encryption and to verify the\
+ \ identity of the source database server\n See more information - in the\
+ \ docs."
+ type: "object"
+ order: 7
oneOf:
- - title: "GCS Staging"
- description: "(recommended) Writes large batches of records to\
- \ a file, uploads the file to GCS, then uses COPY INTO to load your\
- \ data into BigQuery. Provides best-in-class speed, reliability and\
- \ scalability. Read more about GCS Staging here."
+ - title: "disable"
+ additionalProperties: false
+ description: "Disable SSL."
required:
- - "method"
- - "gcs_bucket_name"
- - "gcs_bucket_path"
- - "credential"
+ - "mode"
properties:
- method:
+ mode:
type: "string"
- const: "GCS Staging"
- order: 0
+ const: "disable"
enum:
- - "GCS Staging"
- credential:
- title: "Credential"
- description: "An HMAC key is a type of credential and can be associated\
- \ with a service account or a user account in Cloud Storage. Read\
- \ more here."
- type: "object"
- order: 1
- oneOf:
- - title: "HMAC key"
- required:
- - "credential_type"
- - "hmac_key_access_id"
- - "hmac_key_secret"
- properties:
- credential_type:
- type: "string"
- const: "HMAC_KEY"
- order: 0
- enum:
- - "HMAC_KEY"
- hmac_key_access_id:
- type: "string"
- description: "HMAC key access ID. When linked to a service account,\
- \ this ID is 61 characters long; when linked to a user account,\
- \ it is 24 characters long."
- title: "HMAC Key Access ID"
- airbyte_secret: true
- examples:
- - "1234567890abcdefghij1234"
- order: 1
- hmac_key_secret:
- type: "string"
- description: "The corresponding secret for the access ID. It\
- \ is a 40-character base-64 encoded string."
- title: "HMAC Key Secret"
- airbyte_secret: true
- examples:
- - "1234567890abcdefghij1234567890ABCDEFGHIJ"
- order: 2
- gcs_bucket_name:
- title: "GCS Bucket Name"
- type: "string"
- description: "The name of the GCS bucket. Read more here."
- examples:
- - "airbyte_sync"
- order: 2
- gcs_bucket_path:
- title: "GCS Bucket Path"
- description: "Directory under the GCS bucket where data will be written."
- type: "string"
- examples:
- - "data_sync/test"
- order: 3
- keep_files_in_gcs-bucket:
+ - "disable"
+ default: "disable"
+ order: 0
+ - title: "allow"
+ additionalProperties: false
+ description: "Allow SSL mode."
+ required:
+ - "mode"
+ properties:
+ mode:
type: "string"
- description: "This upload method is supposed to temporary store records\
- \ in GCS bucket. By this select you can chose if these records should\
- \ be removed from GCS when migration has finished. The default \"\
- Delete all tmp files from GCS\" value is used if not set explicitly."
- title: "GCS Tmp Files Afterward Processing"
- default: "Delete all tmp files from GCS"
+ const: "allow"
enum:
- - "Delete all tmp files from GCS"
- - "Keep all tmp files in GCS"
- order: 4
- - title: "Standard Inserts"
+ - "allow"
+ default: "allow"
+ order: 0
+ - title: "prefer"
+ additionalProperties: false
+ description: "Prefer SSL mode."
required:
- - "method"
- description: "(not recommended) Direct loading using SQL INSERT\
- \ statements. This method is extremely inefficient and provided only\
- \ for quick testing. In all other cases, you should use GCS staging."
+ - "mode"
properties:
- method:
+ mode:
type: "string"
- const: "Standard"
+ const: "prefer"
enum:
- - "Standard"
- credentials_json:
- type: "string"
- description: "The contents of the JSON service account key. Check out the\
- \ docs if you need help generating this key. Default credentials will\
- \ be used if this field is left empty."
- title: "Service Account Key JSON (Required for cloud, optional for open-source)"
- airbyte_secret: true
- group: "connection"
- order: 4
- always_show: true
- transformation_priority:
- type: "string"
- description: "Interactive run type means that the query is executed as soon\
- \ as possible, and these queries count towards concurrent rate limit and\
- \ daily limit. Read more about interactive run type here. Batch queries are queued and started as soon as idle resources\
- \ are available in the BigQuery shared resource pool, which usually occurs\
- \ within a few minutes. Batch queries don’t count towards your concurrent\
- \ rate limit. Read more about batch queries here. The default \"interactive\" value is used if not set explicitly."
- title: "Transformation Query Run Type"
- default: "interactive"
- enum:
- - "interactive"
- - "batch"
- order: 5
- group: "advanced"
- big_query_client_buffer_size_mb:
- title: "Google BigQuery Client Chunk Size"
- description: "Google BigQuery client's chunk (buffer) size (MIN=1, MAX =\
- \ 15) for each table. The size that will be written by a single RPC. Written\
- \ data will be buffered and only flushed upon reaching this size or closing\
- \ the channel. The default 15MB value is used if not set explicitly. Read\
- \ more here."
- type: "integer"
- minimum: 1
- maximum: 15
- default: 15
- examples:
- - "15"
- order: 6
- group: "advanced"
- raw_data_dataset:
- type: "string"
- description: "The dataset to write raw tables into (default: airbyte_internal)"
- title: "Raw Table Dataset Name"
- order: 7
- group: "advanced"
- disable_type_dedupe:
- type: "boolean"
- default: false
- description: "Disable Writing Final Tables. WARNING! The data format in\
- \ _airbyte_data is likely stable but there are no guarantees that other\
- \ metadata columns will remain the same in future versions"
- title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
- \ table schema might change between versions)"
- order: 8
- group: "advanced"
- groups:
- - id: "connection"
- title: "Connection"
- - id: "advanced"
- title: "Advanced"
- destination-elasticsearch:
- title: "Elasticsearch Connection Configuration"
- type: "object"
- required:
- - "endpoint"
- - "destinationType"
- properties:
- endpoint:
- title: "Server Endpoint"
- type: "string"
- description: "The full url of the Elasticsearch server"
- upsert:
- type: "boolean"
- title: "Upsert Records"
- description: "If a primary key identifier is defined in the source, an upsert\
- \ will be performed using the primary key value as the elasticsearch doc\
- \ id. Does not support composite primary keys."
- default: true
- ca_certificate:
- type: "string"
- title: "CA certificate"
- description: "CA certificate"
- airbyte_secret: true
- multiline: true
- authenticationMethod:
- title: "Authentication Method"
- type: "object"
- description: "The type of authentication to be used"
- oneOf:
- - title: "Api Key/Secret"
+ - "prefer"
+ default: "prefer"
+ order: 0
+ - title: "require"
additionalProperties: false
- description: "Use a api key and secret combination to authenticate"
+ description: "Require SSL mode."
required:
- - "method"
- - "apiKeyId"
- - "apiKeySecret"
+ - "mode"
+ properties:
+ mode:
+ type: "string"
+ const: "require"
+ enum:
+ - "require"
+ default: "require"
+ order: 0
+ - title: "verify-ca"
+ additionalProperties: false
+ description: "Verify-ca SSL mode."
+ required:
+ - "mode"
+ - "ca_certificate"
properties:
- method:
+ mode:
type: "string"
- const: "secret"
+ const: "verify-ca"
enum:
- - "secret"
- apiKeyId:
- title: "API Key ID"
- description: "The Key ID to used when accessing an enterprise Elasticsearch\
- \ instance."
+ - "verify-ca"
+ default: "verify-ca"
+ order: 0
+ ca_certificate:
type: "string"
- apiKeySecret:
- title: "API Key Secret"
- description: "The secret associated with the API Key ID."
+ title: "CA certificate"
+ description: "CA certificate"
+ airbyte_secret: true
+ multiline: true
+ order: 1
+ client_key_password:
type: "string"
+ title: "Client key password"
+ description: "Password for keystorage. This field is optional. If\
+ \ you do not add it - the password will be generated automatically."
airbyte_secret: true
- - title: "Username/Password"
+ order: 4
+ - title: "verify-full"
additionalProperties: false
- description: "Basic auth header with a username and password"
+ description: "Verify-full SSL mode."
required:
- - "method"
- - "username"
- - "password"
+ - "mode"
+ - "ca_certificate"
+ - "client_certificate"
+ - "client_key"
properties:
- method:
+ mode:
type: "string"
- const: "basic"
+ const: "verify-full"
enum:
- - "basic"
- username:
- title: "Username"
- description: "Basic auth username to access a secure Elasticsearch\
- \ server"
+ - "verify-full"
+ default: "verify-full"
+ order: 0
+ ca_certificate:
type: "string"
- password:
- title: "Password"
- description: "Basic auth password to access a secure Elasticsearch\
- \ server"
+ title: "CA certificate"
+ description: "CA certificate"
+ airbyte_secret: true
+ multiline: true
+ order: 1
+ client_certificate:
type: "string"
+ title: "Client certificate"
+ description: "Client certificate"
airbyte_secret: true
- x-speakeasy-param-sensitive: true
- destinationType:
- title: "elasticsearch"
- const: "elasticsearch"
- enum:
- - "elasticsearch"
- order: 0
- type: "string"
- destination-elasticsearch-update:
- title: "Elasticsearch Connection Configuration"
- type: "object"
- required:
- - "endpoint"
- properties:
- endpoint:
- title: "Server Endpoint"
- type: "string"
- description: "The full url of the Elasticsearch server"
- upsert:
- type: "boolean"
- title: "Upsert Records"
- description: "If a primary key identifier is defined in the source, an upsert\
- \ will be performed using the primary key value as the elasticsearch doc\
- \ id. Does not support composite primary keys."
- default: true
- ca_certificate:
+ multiline: true
+ order: 2
+ client_key:
+ type: "string"
+ title: "Client key"
+ description: "Client key"
+ airbyte_secret: true
+ multiline: true
+ order: 3
+ client_key_password:
+ type: "string"
+ title: "Client key password"
+ description: "Password for keystorage. This field is optional. If\
+ \ you do not add it - the password will be generated automatically."
+ airbyte_secret: true
+ order: 4
+ jdbc_url_params:
+ description: "Additional properties to pass to the JDBC URL string when\
+ \ connecting to the database formatted as 'key=value' pairs separated\
+ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+ title: "JDBC URL Params"
type: "string"
- title: "CA certificate"
- description: "CA certificate"
- airbyte_secret: true
- multiline: true
- authenticationMethod:
- title: "Authentication Method"
+ order: 8
+ tunnel_method:
type: "object"
- description: "The type of authentication to be used"
+ title: "SSH Tunnel Method"
+ description: "Whether to initiate an SSH tunnel before connecting to the\
+ \ database, and if so, which kind of authentication to use."
oneOf:
- - title: "Api Key/Secret"
- additionalProperties: false
- description: "Use a api key and secret combination to authenticate"
+ - title: "No Tunnel"
required:
- - "method"
- - "apiKeyId"
- - "apiKeySecret"
+ - "tunnel_method"
properties:
- method:
+ tunnel_method:
+ description: "No ssh tunnel needed to connect to database"
type: "string"
- const: "secret"
+ const: "NO_TUNNEL"
+ order: 0
enum:
- - "secret"
- apiKeyId:
- title: "API Key ID"
- description: "The Key ID to used when accessing an enterprise Elasticsearch\
- \ instance."
+ - "NO_TUNNEL"
+ - title: "SSH Key Authentication"
+ required:
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "ssh_key"
+ properties:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and ssh key"
type: "string"
- apiKeySecret:
- title: "API Key Secret"
- description: "The secret associated with the API Key ID."
+ const: "SSH_KEY_AUTH"
+ order: 0
+ enum:
+ - "SSH_KEY_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
+ type: "string"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host."
+ type: "string"
+ order: 3
+ ssh_key:
+ title: "SSH Private Key"
+ description: "OS-level user account ssh key credentials in RSA PEM\
+ \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
type: "string"
airbyte_secret: true
- - title: "Username/Password"
- additionalProperties: false
- description: "Basic auth header with a username and password"
+ multiline: true
+ order: 4
+ - title: "Password Authentication"
required:
- - "method"
- - "username"
- - "password"
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "tunnel_user_password"
properties:
- method:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and password authentication"
type: "string"
- const: "basic"
+ const: "SSH_PASSWORD_AUTH"
+ order: 0
enum:
- - "basic"
- username:
- title: "Username"
- description: "Basic auth username to access a secure Elasticsearch\
- \ server"
+ - "SSH_PASSWORD_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
type: "string"
- password:
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
+ examples:
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host"
+ type: "string"
+ order: 3
+ tunnel_user_password:
title: "Password"
- description: "Basic auth password to access a secure Elasticsearch\
- \ server"
+ description: "OS-level password for logging into the jump server host"
type: "string"
airbyte_secret: true
- destination-azure-blob-storage:
- title: "AzureBlobStorage Destination Spec"
+ order: 4
+ destination-postgres:
+ title: "Postgres Destination Spec"
type: "object"
required:
- - "azure_blob_storage_account_name"
- - "azure_blob_storage_account_key"
- - "format"
+ - "host"
+ - "port"
+ - "username"
+ - "database"
+ - "schema"
- "destinationType"
properties:
- azure_blob_storage_endpoint_domain_name:
- title: "Endpoint Domain Name"
- type: "string"
- default: "blob.core.windows.net"
- description: "This is Azure Blob Storage endpoint domain name. Leave default\
- \ value (or leave it empty if run container from command line) to use\
- \ Microsoft native from example."
- examples:
- - "blob.core.windows.net"
- azure_blob_storage_container_name:
- title: "Azure blob storage container (Bucket) Name"
- type: "string"
- description: "The name of the Azure blob storage container. If not exists\
- \ - will be created automatically. May be empty, then will be created\
- \ automatically airbytecontainer+timestamp"
- examples:
- - "airbytetescontainername"
- azure_blob_storage_account_name:
- title: "Azure Blob Storage account name"
- type: "string"
- description: "The account's name of the Azure Blob Storage."
- examples:
- - "airbyte5storage"
- azure_blob_storage_account_key:
- title: "Azure Blob Storage account key"
- description: "The Azure blob storage account key."
- airbyte_secret: true
+ host:
+ title: "Host"
+ description: "Hostname of the database."
type: "string"
- examples:
- - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
- x-speakeasy-param-sensitive: true
- azure_blob_storage_output_buffer_size:
- title: "Azure Blob Storage output buffer size (Megabytes)"
- type: "integer"
- description: "The amount of megabytes to buffer for the output stream to\
- \ Azure. This will impact memory footprint on workers, but may need adjustment\
- \ for performance and appropriate block size in Azure."
- minimum: 1
- maximum: 2047
- default: 5
- examples:
- - 5
- azure_blob_storage_spill_size:
- title: "Azure Blob Storage file spill size"
+ order: 0
+ port:
+ title: "Port"
+ description: "Port of the database."
type: "integer"
- description: "The amount of megabytes after which the connector should spill\
- \ the records in a new blob object. Make sure to configure size greater\
- \ than individual records. Enter 0 if not applicable"
- default: 500
+ minimum: 0
+ maximum: 65536
+ default: 5432
examples:
- - 500
- format:
- title: "Output Format"
- type: "object"
- description: "Output data format"
- oneOf:
- - title: "CSV: Comma-Separated Values"
- required:
- - "format_type"
- - "flattening"
- properties:
- format_type:
- type: "string"
- const: "CSV"
- enum:
- - "CSV"
- flattening:
- type: "string"
- title: "Normalization (Flattening)"
- description: "Whether the input json data should be normalized (flattened)\
- \ in the output CSV. Please refer to docs for details."
- default: "No flattening"
- enum:
- - "No flattening"
- - "Root level flattening"
- - title: "JSON Lines: newline-delimited JSON"
- required:
- - "format_type"
- properties:
- format_type:
- type: "string"
- const: "JSONL"
- enum:
- - "JSONL"
- destinationType:
- title: "azure-blob-storage"
- const: "azure-blob-storage"
- enum:
- - "azure-blob-storage"
- order: 0
+ - "5432"
+ order: 1
+ database:
+ title: "DB Name"
+ description: "Name of the database."
type: "string"
- destination-azure-blob-storage-update:
- title: "AzureBlobStorage Destination Spec"
- type: "object"
- required:
- - "azure_blob_storage_account_name"
- - "azure_blob_storage_account_key"
- - "format"
- properties:
- azure_blob_storage_endpoint_domain_name:
- title: "Endpoint Domain Name"
+ order: 2
+ schema:
+ title: "Default Schema"
+ description: "The default schema tables are written to if the source does\
+ \ not specify a namespace. The usual value for this field is \"public\"\
+ ."
type: "string"
- default: "blob.core.windows.net"
- description: "This is Azure Blob Storage endpoint domain name. Leave default\
- \ value (or leave it empty if run container from command line) to use\
- \ Microsoft native from example."
examples:
- - "blob.core.windows.net"
- azure_blob_storage_container_name:
- title: "Azure blob storage container (Bucket) Name"
+ - "public"
+ default: "public"
+ order: 3
+ username:
+ title: "User"
+ description: "Username to use to access the database."
type: "string"
- description: "The name of the Azure blob storage container. If not exists\
- \ - will be created automatically. May be empty, then will be created\
- \ automatically airbytecontainer+timestamp"
- examples:
- - "airbytetescontainername"
- azure_blob_storage_account_name:
- title: "Azure Blob Storage account name"
+ order: 4
+ password:
+ title: "Password"
+ description: "Password associated with the username."
type: "string"
- description: "The account's name of the Azure Blob Storage."
- examples:
- - "airbyte5storage"
- azure_blob_storage_account_key:
- title: "Azure Blob Storage account key"
- description: "The Azure blob storage account key."
airbyte_secret: true
- type: "string"
- examples:
- - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
- azure_blob_storage_output_buffer_size:
- title: "Azure Blob Storage output buffer size (Megabytes)"
- type: "integer"
- description: "The amount of megabytes to buffer for the output stream to\
- \ Azure. This will impact memory footprint on workers, but may need adjustment\
- \ for performance and appropriate block size in Azure."
- minimum: 1
- maximum: 2047
- default: 5
- examples:
- - 5
- azure_blob_storage_spill_size:
- title: "Azure Blob Storage file spill size"
- type: "integer"
- description: "The amount of megabytes after which the connector should spill\
- \ the records in a new blob object. Make sure to configure size greater\
- \ than individual records. Enter 0 if not applicable"
- default: 500
- examples:
- - 500
- format:
- title: "Output Format"
- type: "object"
- description: "Output data format"
- oneOf:
- - title: "CSV: Comma-Separated Values"
- required:
- - "format_type"
- - "flattening"
- properties:
- format_type:
- type: "string"
- const: "CSV"
- enum:
- - "CSV"
- flattening:
- type: "string"
- title: "Normalization (Flattening)"
- description: "Whether the input json data should be normalized (flattened)\
- \ in the output CSV. Please refer to docs for details."
- default: "No flattening"
- enum:
- - "No flattening"
- - "Root level flattening"
- - title: "JSON Lines: newline-delimited JSON"
- required:
- - "format_type"
- properties:
- format_type:
- type: "string"
- const: "JSONL"
- enum:
- - "JSONL"
- destination-langchain:
- title: "Langchain Destination Config"
- type: "object"
- properties:
- processing:
- title: "ProcessingConfigModel"
- type: "object"
- properties:
- chunk_size:
- title: "Chunk size"
- description: "Size of chunks in tokens to store in vector store (make\
- \ sure it is not too big for the context if your LLM)"
- maximum: 8191
- type: "integer"
- chunk_overlap:
- title: "Chunk overlap"
- description: "Size of overlap between chunks in tokens to store in vector\
- \ store to better capture relevant context"
- default: 0
- type: "integer"
- text_fields:
- title: "Text fields to embed"
- description: "List of fields in the record that should be used to calculate\
- \ the embedding. All other fields are passed along as meta fields.\
- \ The field list is applied to all streams in the same way and non-existing\
- \ fields are ignored. If none are defined, all fields are considered\
- \ text fields. When specifying text fields, you can access nested\
- \ fields in the record by using dot notation, e.g. `user.name` will\
- \ access the `name` field in the `user` object. It's also possible\
- \ to use wildcards to access all fields in an object, e.g. `users.*.name`\
- \ will access all `names` fields in all entries of the `users` array."
- always_show: true
- examples:
- - "text"
- - "user.name"
- - "users.*.name"
- type: "array"
- items:
- type: "string"
- required:
- - "chunk_size"
- - "text_fields"
- group: "processing"
- embedding:
- title: "Embedding"
- description: "Embedding configuration"
- group: "embedding"
+ order: 5
+ x-speakeasy-param-sensitive: true
+ ssl_mode:
+ title: "SSL modes"
+ description: "SSL connection modes. \n disable - Chose this mode\
+ \ to disable encryption of communication between Airbyte and destination\
+ \ database\n allow - Chose this mode to enable encryption only\
+ \ when required by the source database\n prefer - Chose this mode\
+ \ to allow unencrypted connection only if the source database does not\
+ \ support encryption\n require - Chose this mode to always require\
+ \ encryption. If the source database server does not support encryption,\
+ \ connection will fail\n verify-ca - Chose this mode to always\
+ \ require encryption and to verify that the source database server has\
+ \ a valid SSL certificate\n verify-full - This is the most secure\
+ \ mode. Chose this mode to always require encryption and to verify the\
+ \ identity of the source database server\n See more information - in the\
+ \ docs."
type: "object"
+ order: 7
oneOf:
- - title: "OpenAI"
- type: "object"
+ - title: "disable"
+ additionalProperties: false
+ description: "Disable SSL."
+ required:
+ - "mode"
properties:
mode:
- title: "Mode"
- default: "openai"
- const: "openai"
- enum:
- - "openai"
- type: "string"
- openai_key:
- title: "OpenAI API key"
- airbyte_secret: true
type: "string"
- x-speakeasy-param-sensitive: true
+ const: "disable"
+ enum:
+ - "disable"
+ default: "disable"
+ order: 0
+ - title: "allow"
+ additionalProperties: false
+ description: "Allow SSL mode."
required:
- - "openai_key"
- description: "Use the OpenAI API to embed text. This option is using the\
- \ text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "Fake"
- type: "object"
+ - "mode"
properties:
mode:
- title: "Mode"
- default: "fake"
- const: "fake"
- enum:
- - "fake"
type: "string"
- description: "Use a fake embedding made out of random vectors with 1536\
- \ embedding dimensions. This is useful for testing the data pipeline\
- \ without incurring any costs."
- indexing:
- title: "Indexing"
- description: "Indexing configuration"
- group: "indexing"
- type: "object"
- oneOf:
- - title: "Pinecone"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "pinecone"
- const: "pinecone"
+ const: "allow"
enum:
- - "pinecone"
- type: "string"
- pinecone_key:
- title: "Pinecone API key"
- airbyte_secret: true
- type: "string"
- x-speakeasy-param-sensitive: true
- pinecone_environment:
- title: "Pinecone environment"
- description: "Pinecone environment to use"
- type: "string"
- index:
- title: "Index"
- description: "Pinecone index to use"
- type: "string"
+ - "allow"
+ default: "allow"
+ order: 0
+ - title: "prefer"
+ additionalProperties: false
+ description: "Prefer SSL mode."
required:
- - "pinecone_key"
- - "pinecone_environment"
- - "index"
- description: "Pinecone is a popular vector store that can be used to store\
- \ and retrieve embeddings. It is a managed service and can also be queried\
- \ from outside of langchain."
- - title: "DocArrayHnswSearch"
- type: "object"
+ - "mode"
properties:
mode:
- title: "Mode"
- default: "DocArrayHnswSearch"
- const: "DocArrayHnswSearch"
- enum:
- - "DocArrayHnswSearch"
- type: "string"
- destination_path:
- title: "Destination Path"
- description: "Path to the directory where hnswlib and meta data files\
- \ will be written. The files will be placed inside that local mount.\
- \ All files in the specified destination directory will be deleted\
- \ on each run."
- examples:
- - "/local/my_hnswlib_index"
type: "string"
+ const: "prefer"
+ enum:
+ - "prefer"
+ default: "prefer"
+ order: 0
+ - title: "require"
+ additionalProperties: false
+ description: "Require SSL mode."
required:
- - "destination_path"
- description: "DocArrayHnswSearch is a lightweight Document Index implementation\
- \ provided by Docarray that runs fully locally and is best suited for\
- \ small- to medium-sized datasets. It stores vectors on disk in hnswlib,\
- \ and stores all other data in SQLite."
- - title: "Chroma (local persistance)"
- type: "object"
+ - "mode"
properties:
mode:
- title: "Mode"
- default: "chroma_local"
- const: "chroma_local"
- enum:
- - "chroma_local"
- type: "string"
- destination_path:
- title: "Destination Path"
- description: "Path to the directory where chroma files will be written.\
- \ The files will be placed inside that local mount."
- examples:
- - "/local/my_chroma_db"
- type: "string"
- collection_name:
- title: "Collection Name"
- description: "Name of the collection to use."
- default: "langchain"
type: "string"
+ const: "require"
+ enum:
+ - "require"
+ default: "require"
+ order: 0
+ - title: "verify-ca"
+ additionalProperties: false
+ description: "Verify-ca SSL mode."
required:
- - "destination_path"
- description: "Chroma is a popular vector store that can be used to store\
- \ and retrieve embeddings. It will build its index in memory and persist\
- \ it to disk by the end of the sync."
- destinationType:
- title: "langchain"
- const: "langchain"
- enum:
- - "langchain"
- order: 0
- type: "string"
- required:
- - "processing"
- - "embedding"
- - "indexing"
- - "destinationType"
- groups:
- - id: "processing"
- title: "Processing"
- - id: "embedding"
- title: "Embedding"
- - id: "indexing"
- title: "Indexing"
- destination-langchain-update:
- title: "Langchain Destination Config"
- type: "object"
- properties:
- processing:
- title: "ProcessingConfigModel"
- type: "object"
- properties:
- chunk_size:
- title: "Chunk size"
- description: "Size of chunks in tokens to store in vector store (make\
- \ sure it is not too big for the context if your LLM)"
- maximum: 8191
- type: "integer"
- chunk_overlap:
- title: "Chunk overlap"
- description: "Size of overlap between chunks in tokens to store in vector\
- \ store to better capture relevant context"
- default: 0
- type: "integer"
- text_fields:
- title: "Text fields to embed"
- description: "List of fields in the record that should be used to calculate\
- \ the embedding. All other fields are passed along as meta fields.\
- \ The field list is applied to all streams in the same way and non-existing\
- \ fields are ignored. If none are defined, all fields are considered\
- \ text fields. When specifying text fields, you can access nested\
- \ fields in the record by using dot notation, e.g. `user.name` will\
- \ access the `name` field in the `user` object. It's also possible\
- \ to use wildcards to access all fields in an object, e.g. `users.*.name`\
- \ will access all `names` fields in all entries of the `users` array."
- always_show: true
- examples:
- - "text"
- - "user.name"
- - "users.*.name"
- type: "array"
- items:
- type: "string"
- required:
- - "chunk_size"
- - "text_fields"
- group: "processing"
- embedding:
- title: "Embedding"
- description: "Embedding configuration"
- group: "embedding"
- type: "object"
- oneOf:
- - title: "OpenAI"
- type: "object"
+ - "mode"
+ - "ca_certificate"
properties:
mode:
- title: "Mode"
- default: "openai"
- const: "openai"
+ type: "string"
+ const: "verify-ca"
enum:
- - "openai"
+ - "verify-ca"
+ default: "verify-ca"
+ order: 0
+ ca_certificate:
type: "string"
- openai_key:
- title: "OpenAI API key"
+ title: "CA certificate"
+ description: "CA certificate"
airbyte_secret: true
+ multiline: true
+ order: 1
+ client_key_password:
type: "string"
+ title: "Client key password"
+ description: "Password for keystorage. This field is optional. If\
+ \ you do not add it - the password will be generated automatically."
+ airbyte_secret: true
+ order: 4
+ x-speakeasy-param-sensitive: true
+ - title: "verify-full"
+ additionalProperties: false
+ description: "Verify-full SSL mode."
required:
- - "openai_key"
- description: "Use the OpenAI API to embed text. This option is using the\
- \ text-embedding-ada-002 model with 1536 embedding dimensions."
- - title: "Fake"
- type: "object"
+ - "mode"
+ - "ca_certificate"
+ - "client_certificate"
+ - "client_key"
properties:
mode:
- title: "Mode"
- default: "fake"
- const: "fake"
- enum:
- - "fake"
type: "string"
- description: "Use a fake embedding made out of random vectors with 1536\
- \ embedding dimensions. This is useful for testing the data pipeline\
- \ without incurring any costs."
- indexing:
- title: "Indexing"
- description: "Indexing configuration"
- group: "indexing"
- type: "object"
- oneOf:
- - title: "Pinecone"
- type: "object"
- properties:
- mode:
- title: "Mode"
- default: "pinecone"
- const: "pinecone"
+ const: "verify-full"
enum:
- - "pinecone"
+ - "verify-full"
+ default: "verify-full"
+ order: 0
+ ca_certificate:
type: "string"
- pinecone_key:
- title: "Pinecone API key"
+ title: "CA certificate"
+ description: "CA certificate"
airbyte_secret: true
+ multiline: true
+ order: 1
+ client_certificate:
type: "string"
- pinecone_environment:
- title: "Pinecone environment"
- description: "Pinecone environment to use"
+ title: "Client certificate"
+ description: "Client certificate"
+ airbyte_secret: true
+ multiline: true
+ order: 2
+ client_key:
type: "string"
- index:
- title: "Index"
- description: "Pinecone index to use"
+ title: "Client key"
+ description: "Client key"
+ airbyte_secret: true
+ multiline: true
+ order: 3
+ x-speakeasy-param-sensitive: true
+ client_key_password:
type: "string"
+ title: "Client key password"
+ description: "Password for keystorage. This field is optional. If\
+ \ you do not add it - the password will be generated automatically."
+ airbyte_secret: true
+ order: 4
+ x-speakeasy-param-sensitive: true
+ jdbc_url_params:
+ description: "Additional properties to pass to the JDBC URL string when\
+ \ connecting to the database formatted as 'key=value' pairs separated\
+ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+ title: "JDBC URL Params"
+ type: "string"
+ order: 8
+ raw_data_schema:
+ type: "string"
+ description: "The schema to write raw tables into"
+ title: "Raw table schema (defaults to airbyte_internal)"
+ order: 9
+ disable_type_dedupe:
+ type: "boolean"
+ default: false
+ description: "Disable Writing Final Tables. WARNING! The data format in\
+ \ _airbyte_data is likely stable but there are no guarantees that other\
+ \ metadata columns will remain the same in future versions"
+ title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
+ \ table schema might change between versions)"
+ order: 10
+ drop_cascade:
+ type: "boolean"
+ default: false
+ description: "Drop tables with CASCADE. WARNING! This will delete all data\
+ \ in all dependent objects (views, etc.). Use with caution. This option\
+ \ is intended for usecases which can easily rebuild the dependent objects."
+ title: "Drop tables with CASCADE. (WARNING! Risk of unrecoverable data loss)"
+ order: 11
+ tunnel_method:
+ type: "object"
+ title: "SSH Tunnel Method"
+ description: "Whether to initiate an SSH tunnel before connecting to the\
+ \ database, and if so, which kind of authentication to use."
+ oneOf:
+ - title: "No Tunnel"
required:
- - "pinecone_key"
- - "pinecone_environment"
- - "index"
- description: "Pinecone is a popular vector store that can be used to store\
- \ and retrieve embeddings. It is a managed service and can also be queried\
- \ from outside of langchain."
- - title: "DocArrayHnswSearch"
- type: "object"
+ - "tunnel_method"
properties:
- mode:
- title: "Mode"
- default: "DocArrayHnswSearch"
- const: "DocArrayHnswSearch"
+ tunnel_method:
+ description: "No ssh tunnel needed to connect to database"
+ type: "string"
+ const: "NO_TUNNEL"
+ order: 0
enum:
- - "DocArrayHnswSearch"
+ - "NO_TUNNEL"
+ - title: "SSH Key Authentication"
+ required:
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "ssh_key"
+ properties:
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and ssh key"
type: "string"
- destination_path:
- title: "Destination Path"
- description: "Path to the directory where hnswlib and meta data files\
- \ will be written. The files will be placed inside that local mount.\
- \ All files in the specified destination directory will be deleted\
- \ on each run."
+ const: "SSH_KEY_AUTH"
+ order: 0
+ enum:
+ - "SSH_KEY_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
+ type: "string"
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
examples:
- - "/local/my_hnswlib_index"
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host."
+ type: "string"
+ order: 3
+ ssh_key:
+ title: "SSH Private Key"
+ description: "OS-level user account ssh key credentials in RSA PEM\
+ \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
type: "string"
+ airbyte_secret: true
+ multiline: true
+ order: 4
+ x-speakeasy-param-sensitive: true
+ - title: "Password Authentication"
required:
- - "destination_path"
- description: "DocArrayHnswSearch is a lightweight Document Index implementation\
- \ provided by Docarray that runs fully locally and is best suited for\
- \ small- to medium-sized datasets. It stores vectors on disk in hnswlib,\
- \ and stores all other data in SQLite."
- - title: "Chroma (local persistance)"
- type: "object"
+ - "tunnel_method"
+ - "tunnel_host"
+ - "tunnel_port"
+ - "tunnel_user"
+ - "tunnel_user_password"
properties:
- mode:
- title: "Mode"
- default: "chroma_local"
- const: "chroma_local"
+ tunnel_method:
+ description: "Connect through a jump server tunnel host using username\
+ \ and password authentication"
+ type: "string"
+ const: "SSH_PASSWORD_AUTH"
+ order: 0
enum:
- - "chroma_local"
+ - "SSH_PASSWORD_AUTH"
+ tunnel_host:
+ title: "SSH Tunnel Jump Server Host"
+ description: "Hostname of the jump server host that allows inbound\
+ \ ssh tunnel."
type: "string"
- destination_path:
- title: "Destination Path"
- description: "Path to the directory where chroma files will be written.\
- \ The files will be placed inside that local mount."
+ order: 1
+ tunnel_port:
+ title: "SSH Connection Port"
+ description: "Port on the proxy/jump server that accepts inbound ssh\
+ \ connections."
+ type: "integer"
+ minimum: 0
+ maximum: 65536
+ default: 22
examples:
- - "/local/my_chroma_db"
+ - "22"
+ order: 2
+ tunnel_user:
+ title: "SSH Login Username"
+ description: "OS-level username for logging into the jump server host"
type: "string"
- collection_name:
- title: "Collection Name"
- description: "Name of the collection to use."
- default: "langchain"
+ order: 3
+ tunnel_user_password:
+ title: "Password"
+ description: "OS-level password for logging into the jump server host"
type: "string"
- required:
- - "destination_path"
- description: "Chroma is a popular vector store that can be used to store\
- \ and retrieve embeddings. It will build its index in memory and persist\
- \ it to disk by the end of the sync."
- required:
- - "processing"
- - "embedding"
- - "indexing"
- groups:
- - id: "processing"
- title: "Processing"
- - id: "embedding"
- title: "Embedding"
- - id: "indexing"
- title: "Indexing"
- destination-postgres:
+ airbyte_secret: true
+ order: 4
+ x-speakeasy-param-sensitive: true
+ destinationType:
+ title: "postgres"
+ const: "postgres"
+ enum:
+ - "postgres"
+ order: 0
+ type: "string"
+ destination-postgres-update:
title: "Postgres Destination Spec"
type: "object"
required:
@@ -72737,7 +74672,6 @@ components:
- "username"
- "database"
- "schema"
- - "destinationType"
properties:
host:
title: "Host"
@@ -72780,7 +74714,6 @@ components:
type: "string"
airbyte_secret: true
order: 5
- x-speakeasy-param-sensitive: true
ssl_mode:
title: "SSL modes"
description: "SSL connection modes. \n disable - Chose this mode\
@@ -72880,7 +74813,6 @@ components:
\ you do not add it - the password will be generated automatically."
airbyte_secret: true
order: 4
- x-speakeasy-param-sensitive: true
- title: "verify-full"
additionalProperties: false
description: "Verify-full SSL mode."
@@ -72918,7 +74850,6 @@ components:
airbyte_secret: true
multiline: true
order: 3
- x-speakeasy-param-sensitive: true
client_key_password:
type: "string"
title: "Client key password"
@@ -72926,7 +74857,6 @@ components:
\ you do not add it - the password will be generated automatically."
airbyte_secret: true
order: 4
- x-speakeasy-param-sensitive: true
jdbc_url_params:
description: "Additional properties to pass to the JDBC URL string when\
\ connecting to the database formatted as 'key=value' pairs separated\
@@ -72948,337 +74878,14 @@ components:
title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
\ table schema might change between versions)"
order: 10
- tunnel_method:
- type: "object"
- title: "SSH Tunnel Method"
- description: "Whether to initiate an SSH tunnel before connecting to the\
- \ database, and if so, which kind of authentication to use."
- oneOf:
- - title: "No Tunnel"
- required:
- - "tunnel_method"
- properties:
- tunnel_method:
- description: "No ssh tunnel needed to connect to database"
- type: "string"
- const: "NO_TUNNEL"
- order: 0
- enum:
- - "NO_TUNNEL"
- - title: "SSH Key Authentication"
- required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "ssh_key"
- properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and ssh key"
- type: "string"
- const: "SSH_KEY_AUTH"
- order: 0
- enum:
- - "SSH_KEY_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
- type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
- examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host."
- type: "string"
- order: 3
- ssh_key:
- title: "SSH Private Key"
- description: "OS-level user account ssh key credentials in RSA PEM\
- \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
- type: "string"
- airbyte_secret: true
- multiline: true
- order: 4
- x-speakeasy-param-sensitive: true
- - title: "Password Authentication"
- required:
- - "tunnel_method"
- - "tunnel_host"
- - "tunnel_port"
- - "tunnel_user"
- - "tunnel_user_password"
- properties:
- tunnel_method:
- description: "Connect through a jump server tunnel host using username\
- \ and password authentication"
- type: "string"
- const: "SSH_PASSWORD_AUTH"
- order: 0
- enum:
- - "SSH_PASSWORD_AUTH"
- tunnel_host:
- title: "SSH Tunnel Jump Server Host"
- description: "Hostname of the jump server host that allows inbound\
- \ ssh tunnel."
- type: "string"
- order: 1
- tunnel_port:
- title: "SSH Connection Port"
- description: "Port on the proxy/jump server that accepts inbound ssh\
- \ connections."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 22
- examples:
- - "22"
- order: 2
- tunnel_user:
- title: "SSH Login Username"
- description: "OS-level username for logging into the jump server host"
- type: "string"
- order: 3
- tunnel_user_password:
- title: "Password"
- description: "OS-level password for logging into the jump server host"
- type: "string"
- airbyte_secret: true
- order: 4
- x-speakeasy-param-sensitive: true
- destinationType:
- title: "postgres"
- const: "postgres"
- enum:
- - "postgres"
- order: 0
- type: "string"
- destination-postgres-update:
- title: "Postgres Destination Spec"
- type: "object"
- required:
- - "host"
- - "port"
- - "username"
- - "database"
- - "schema"
- properties:
- host:
- title: "Host"
- description: "Hostname of the database."
- type: "string"
- order: 0
- port:
- title: "Port"
- description: "Port of the database."
- type: "integer"
- minimum: 0
- maximum: 65536
- default: 5432
- examples:
- - "5432"
- order: 1
- database:
- title: "DB Name"
- description: "Name of the database."
- type: "string"
- order: 2
- schema:
- title: "Default Schema"
- description: "The default schema tables are written to if the source does\
- \ not specify a namespace. The usual value for this field is \"public\"\
- ."
- type: "string"
- examples:
- - "public"
- default: "public"
- order: 3
- username:
- title: "User"
- description: "Username to use to access the database."
- type: "string"
- order: 4
- password:
- title: "Password"
- description: "Password associated with the username."
- type: "string"
- airbyte_secret: true
- order: 5
- ssl_mode:
- title: "SSL modes"
- description: "SSL connection modes. \n disable - Chose this mode\
- \ to disable encryption of communication between Airbyte and destination\
- \ database\n allow - Chose this mode to enable encryption only\
- \ when required by the source database\n prefer - Chose this mode\
- \ to allow unencrypted connection only if the source database does not\
- \ support encryption\n require - Chose this mode to always require\
- \ encryption. If the source database server does not support encryption,\
- \ connection will fail\n verify-ca - Chose this mode to always\
- \ require encryption and to verify that the source database server has\
- \ a valid SSL certificate\n verify-full - This is the most secure\
- \ mode. Chose this mode to always require encryption and to verify the\
- \ identity of the source database server\n See more information - in the\
- \ docs."
- type: "object"
- order: 7
- oneOf:
- - title: "disable"
- additionalProperties: false
- description: "Disable SSL."
- required:
- - "mode"
- properties:
- mode:
- type: "string"
- const: "disable"
- enum:
- - "disable"
- default: "disable"
- order: 0
- - title: "allow"
- additionalProperties: false
- description: "Allow SSL mode."
- required:
- - "mode"
- properties:
- mode:
- type: "string"
- const: "allow"
- enum:
- - "allow"
- default: "allow"
- order: 0
- - title: "prefer"
- additionalProperties: false
- description: "Prefer SSL mode."
- required:
- - "mode"
- properties:
- mode:
- type: "string"
- const: "prefer"
- enum:
- - "prefer"
- default: "prefer"
- order: 0
- - title: "require"
- additionalProperties: false
- description: "Require SSL mode."
- required:
- - "mode"
- properties:
- mode:
- type: "string"
- const: "require"
- enum:
- - "require"
- default: "require"
- order: 0
- - title: "verify-ca"
- additionalProperties: false
- description: "Verify-ca SSL mode."
- required:
- - "mode"
- - "ca_certificate"
- properties:
- mode:
- type: "string"
- const: "verify-ca"
- enum:
- - "verify-ca"
- default: "verify-ca"
- order: 0
- ca_certificate:
- type: "string"
- title: "CA certificate"
- description: "CA certificate"
- airbyte_secret: true
- multiline: true
- order: 1
- client_key_password:
- type: "string"
- title: "Client key password"
- description: "Password for keystorage. This field is optional. If\
- \ you do not add it - the password will be generated automatically."
- airbyte_secret: true
- order: 4
- - title: "verify-full"
- additionalProperties: false
- description: "Verify-full SSL mode."
- required:
- - "mode"
- - "ca_certificate"
- - "client_certificate"
- - "client_key"
- properties:
- mode:
- type: "string"
- const: "verify-full"
- enum:
- - "verify-full"
- default: "verify-full"
- order: 0
- ca_certificate:
- type: "string"
- title: "CA certificate"
- description: "CA certificate"
- airbyte_secret: true
- multiline: true
- order: 1
- client_certificate:
- type: "string"
- title: "Client certificate"
- description: "Client certificate"
- airbyte_secret: true
- multiline: true
- order: 2
- client_key:
- type: "string"
- title: "Client key"
- description: "Client key"
- airbyte_secret: true
- multiline: true
- order: 3
- client_key_password:
- type: "string"
- title: "Client key password"
- description: "Password for keystorage. This field is optional. If\
- \ you do not add it - the password will be generated automatically."
- airbyte_secret: true
- order: 4
- jdbc_url_params:
- description: "Additional properties to pass to the JDBC URL string when\
- \ connecting to the database formatted as 'key=value' pairs separated\
- \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
- title: "JDBC URL Params"
- type: "string"
- order: 8
- raw_data_schema:
- type: "string"
- description: "The schema to write raw tables into"
- title: "Raw table schema (defaults to airbyte_internal)"
- order: 9
- disable_type_dedupe:
+ drop_cascade:
type: "boolean"
default: false
- description: "Disable Writing Final Tables. WARNING! The data format in\
- \ _airbyte_data is likely stable but there are no guarantees that other\
- \ metadata columns will remain the same in future versions"
- title: "Disable Final Tables. (WARNING! Unstable option; Columns in raw\
- \ table schema might change between versions)"
- order: 10
+ description: "Drop tables with CASCADE. WARNING! This will delete all data\
+ \ in all dependent objects (views, etc.). Use with caution. This option\
+ \ is intended for usecases which can easily rebuild the dependent objects."
+ title: "Drop tables with CASCADE. (WARNING! Risk of unrecoverable data loss)"
+ order: 11
tunnel_method:
type: "object"
title: "SSH Tunnel Method"
@@ -76766,32 +78373,6 @@ components:
type: "string"
x-speakeasy-entity: Source_Qualaroo
x-speakeasy-param-suppress-computed-diff: true
- SourceQuickbooksCreateRequest:
- required:
- - name
- - workspaceId
- - configuration
- type: object
- properties:
- name:
- description: Name of the source e.g. dev-mysql-instance.
- type: string
- definitionId:
- description: The UUID of the connector definition. One of configuration.sourceType or definitionId must be provided.
- format: uuid
- type: string
- workspaceId:
- format: uuid
- type: string
- configuration:
- $ref: "#/components/schemas/source-quickbooks"
- secretId:
- description:
- "Optional secretID obtained through the public API OAuth redirect\
- \ flow."
- type: "string"
- x-speakeasy-entity: Source_Quickbooks
- x-speakeasy-param-suppress-computed-diff: true
SourceRailzCreateRequest:
required:
- name
@@ -78430,32 +80011,6 @@ components:
type: "string"
x-speakeasy-entity: Source_Zoom
x-speakeasy-param-suppress-computed-diff: true
- SourceCustomCreateRequest:
- required:
- - name
- - workspaceId
- - configuration
- type: object
- properties:
- name:
- description: Name of the source e.g. dev-mysql-instance.
- type: string
- definitionId:
- description: The UUID of the connector definition. One of configuration.sourceType or definitionId must be provided.
- format: uuid
- type: string
- workspaceId:
- format: uuid
- type: string
- configuration:
- $ref: "#/components/schemas/source-custom"
- secretId:
- description:
- "Optional secretID obtained through the public API OAuth redirect\
- \ flow."
- type: "string"
- x-speakeasy-entity: Source_Custom
- x-speakeasy-param-suppress-computed-diff: true
DestinationAstraCreateRequest:
required:
- name
@@ -79170,7 +80725,7 @@ components:
$ref: "#/components/schemas/destination-weaviate"
x-speakeasy-entity: Destination_Weaviate
x-speakeasy-param-suppress-computed-diff: true
- DestinationCustomCreateRequest:
+ DestinationYellowbrickCreateRequest:
required:
- name
- workspaceId
@@ -79188,8 +80743,8 @@ components:
format: uuid
type: string
configuration:
- $ref: "#/components/schemas/destination-custom"
- x-speakeasy-entity: Destination_Custom
+ $ref: "#/components/schemas/destination-yellowbrick"
+ x-speakeasy-entity: Destination_Yellowbrick
x-speakeasy-param-suppress-computed-diff: true
SourceAhaPutRequest:
required:
@@ -81271,22 +82826,6 @@ components:
$ref: "#/components/schemas/source-qualaroo-update"
x-speakeasy-entity: Source_Qualaroo
x-speakeasy-param-suppress-computed-diff: true
- SourceQuickbooksPutRequest:
- required:
- - "name"
- - "workspaceId"
- - "configuration"
- type: "object"
- properties:
- name:
- type: "string"
- workspaceId:
- format: "uuid"
- type: "string"
- configuration:
- $ref: "#/components/schemas/source-quickbooks-update"
- x-speakeasy-entity: Source_Quickbooks
- x-speakeasy-param-suppress-computed-diff: true
SourceRailzPutRequest:
required:
- "name"
@@ -82295,22 +83834,6 @@ components:
$ref: "#/components/schemas/source-zoom-update"
x-speakeasy-entity: Source_Zoom
x-speakeasy-param-suppress-computed-diff: true
- SourceCustomPutRequest:
- required:
- - "name"
- - "workspaceId"
- - "configuration"
- type: "object"
- properties:
- name:
- type: "string"
- workspaceId:
- format: "uuid"
- type: "string"
- configuration:
- $ref: "#/components/schemas/source-custom-update"
- x-speakeasy-entity: Source_Custom
- x-speakeasy-param-suppress-computed-diff: true
DestinationAstraPutRequest:
required:
- "name"
@@ -82855,7 +84378,7 @@ components:
$ref: "#/components/schemas/destination-weaviate-update"
x-speakeasy-entity: Destination_Weaviate
x-speakeasy-param-suppress-computed-diff: true
- DestinationCustomPutRequest:
+ DestinationYellowbrickPutRequest:
required:
- "name"
- "workspaceId"
@@ -82868,19 +84391,9 @@ components:
format: "uuid"
type: "string"
configuration:
- $ref: "#/components/schemas/destination-custom-update"
- x-speakeasy-entity: Destination_Custom
+ $ref: "#/components/schemas/destination-yellowbrick-update"
+ x-speakeasy-entity: Destination_Yellowbrick
x-speakeasy-param-suppress-computed-diff: true
- source-custom:
- description: The values required to configure the source.
- example: { user: "charles" }
- destination-custom:
- description: The values required to configure the destination.
- example: { user: "charles" }
- source-custom-update:
- title: "Custom Spec"
- destination-custom-update:
- title: "Custom Spec"
SourceConfiguration:
description: The values required to configure the source.
example: { user: "charles" }
@@ -82895,6 +84408,13 @@ components:
basicAuth:
type: http
scheme: basic
+ clientCredentials:
+ type: oauth2
+ flows:
+ clientCredentials:
+ tokenUrl: https://cloud.airbyte.io/api/v1/applications/token
+ scopes: {}
security:
- bearerAuth: []
- basicAuth: []
+ - clientCredentials: []