diff --git a/.devcontainer/destination-duckdb/devcontainer.json b/.devcontainer/destination-duckdb/devcontainer.json index 587b51126ff6..cd04dcafb77d 100644 --- a/.devcontainer/destination-duckdb/devcontainer.json +++ b/.devcontainer/destination-duckdb/devcontainer.json @@ -23,21 +23,21 @@ "customizations": { "vscode": { "extensions": [ - // Python extensions: - "charliermarsh.ruff", - "matangover.mypy", - "ms-python.black-formatter", - "ms-python.python", - "ms-python.vscode-pylance", + // Python extensions: + "charliermarsh.ruff", + "matangover.mypy", + "ms-python.black-formatter", + "ms-python.python", + "ms-python.vscode-pylance", - // Toml support - "tamasfe.even-better-toml", - - // Yaml and JSON Schema support: - "redhat.vscode-yaml", + // Toml support + "tamasfe.even-better-toml", - // Contributing: - "GitHub.vscode-pull-request-github" + // Yaml and JSON Schema support: + "redhat.vscode-yaml", + + // Contributing: + "GitHub.vscode-pull-request-github" ], "settings": { "extensions.ignoreRecommendations": true, @@ -53,8 +53,8 @@ "." ], "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" - } + "editor.defaultFormatter": "ms-python.black-formatter" + } } } }, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index c790291e9fc0..24e3a907b115 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -11,12 +11,12 @@ class AddedFieldDefinition(BaseModel): - type: Literal['AddedFieldDefinition'] + type: Literal["AddedFieldDefinition"] path: List[str] = Field( ..., - description='List of strings defining the path where to add the value on the record.', - examples=[['segment_id'], ['metadata', 'segment_id']], - title='Path', + description="List of strings defining the path where to add the value on the record.", + examples=[["segment_id"], ["metadata", "segment_id"]], + title="Path", ) value: str = Field( ..., @@ -26,607 +26,601 @@ class AddedFieldDefinition(BaseModel): "{{ record['MetaData']['LastUpdatedTime'] }}", "{{ stream_partition['segment_id'] }}", ], - title='Value', + title="Value", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AddFields(BaseModel): - type: Literal['AddFields'] + type: Literal["AddFields"] fields: List[AddedFieldDefinition] = Field( ..., - description='List of transformations (path and corresponding value) that will be added to the record.', - title='Fields', + description="List of transformations (path and corresponding value) that will be added to the record.", + title="Fields", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AuthFlowType(Enum): - oauth2_0 = 'oauth2.0' - oauth1_0 = 'oauth1.0' + oauth2_0 = "oauth2.0" + oauth1_0 = "oauth1.0" class BasicHttpAuthenticator(BaseModel): - type: Literal['BasicHttpAuthenticator'] + type: Literal["BasicHttpAuthenticator"] username: str = Field( ..., - description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.', + description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], - title='Username', + title="Username", ) password: Optional[str] = Field( - '', - description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.', - examples=["{{ config['password'] }}", ''], - title='Password', + "", + description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", + examples=["{{ config['password'] }}", ""], + title="Password", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class BearerAuthenticator(BaseModel): - type: Literal['BearerAuthenticator'] + type: Literal["BearerAuthenticator"] api_token: str = Field( ..., - description='Token to inject as request header for authenticating with the API.', + description="Token to inject as request header for authenticating with the API.", examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], - title='Bearer Token', + title="Bearer Token", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CheckStream(BaseModel): - type: Literal['CheckStream'] + type: Literal["CheckStream"] stream_names: List[str] = Field( ..., - description='Names of the streams to try reading from when running a check operation.', - examples=[['users'], ['users', 'contacts']], - title='Stream Names', + description="Names of the streams to try reading from when running a check operation.", + examples=[["users"], ["users", "contacts"]], + title="Stream Names", ) class ConstantBackoffStrategy(BaseModel): - type: Literal['ConstantBackoffStrategy'] + type: Literal["ConstantBackoffStrategy"] backoff_time_in_seconds: Union[float, str] = Field( ..., - description='Backoff time in seconds.', + description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], - title='Backoff Time', + title="Backoff Time", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomAuthenticator(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomAuthenticator'] + type: Literal["CustomAuthenticator"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.', - examples=['source_railz.components.ShortLivedTokenAuthenticator'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.", + examples=["source_railz.components.ShortLivedTokenAuthenticator"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomBackoffStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomBackoffStrategy'] + type: Literal["CustomBackoffStrategy"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomBackoffStrategy'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomBackoffStrategy"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomErrorHandler(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomErrorHandler'] + type: Literal["CustomErrorHandler"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.', - examples=['source_railz.components.MyCustomErrorHandler'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.", + examples=["source_railz.components.MyCustomErrorHandler"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomIncrementalSync(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomIncrementalSync'] + type: Literal["CustomIncrementalSync"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.', - examples=['source_railz.components.MyCustomIncrementalSync'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.", + examples=["source_railz.components.MyCustomIncrementalSync"], + title="Class Name", ) cursor_field: str = Field( ..., - description='The location of the value on a record that will be used as a bookmark during sync.', + description="The location of the value on a record that will be used as a bookmark during sync.", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomPaginationStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomPaginationStrategy'] + type: Literal["CustomPaginationStrategy"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomPaginationStrategy'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomPaginationStrategy"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRecordExtractor(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRecordExtractor'] + type: Literal["CustomRecordExtractor"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRecordExtractor'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRecordExtractor"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRequester(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRequester'] + type: Literal["CustomRequester"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRecordExtractor'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRecordExtractor"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRetriever(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRetriever'] + type: Literal["CustomRetriever"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRetriever'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRetriever"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomPartitionRouter(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomPartitionRouter'] + type: Literal["CustomPartitionRouter"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.', - examples=['source_railz.components.MyCustomPartitionRouter'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.", + examples=["source_railz.components.MyCustomPartitionRouter"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomTransformation(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomTransformation'] + type: Literal["CustomTransformation"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.', - examples=['source_railz.components.MyCustomTransformation'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.", + examples=["source_railz.components.MyCustomTransformation"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RefreshTokenUpdater(BaseModel): refresh_token_name: Optional[str] = Field( - 'refresh_token', - description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.', - examples=['refresh_token'], - title='Refresh Token Property Name', + "refresh_token", + description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", + examples=["refresh_token"], + title="Refresh Token Property Name", ) access_token_config_path: Optional[List[str]] = Field( - ['credentials', 'access_token'], - description='Config path to the access token. Make sure the field actually exists in the config.', - examples=[['credentials', 'access_token'], ['access_token']], - title='Config Path To Access Token', + ["credentials", "access_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "access_token"], ["access_token"]], + title="Config Path To Access Token", ) refresh_token_config_path: Optional[List[str]] = Field( - ['credentials', 'refresh_token'], - description='Config path to the access token. Make sure the field actually exists in the config.', - examples=[['credentials', 'refresh_token'], ['refresh_token']], - title='Config Path To Refresh Token', + ["credentials", "refresh_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "refresh_token"], ["refresh_token"]], + title="Config Path To Refresh Token", ) token_expiry_date_config_path: Optional[List[str]] = Field( - ['credentials', 'token_expiry_date'], - description='Config path to the expiry date. Make sure actually exists in the config.', - examples=[['credentials', 'token_expiry_date']], - title='Config Path To Expiry Date', + ["credentials", "token_expiry_date"], + description="Config path to the expiry date. Make sure actually exists in the config.", + examples=[["credentials", "token_expiry_date"]], + title="Config Path To Expiry Date", ) class OAuthAuthenticator(BaseModel): - type: Literal['OAuthAuthenticator'] + type: Literal["OAuthAuthenticator"] client_id: str = Field( ..., - description='The OAuth client ID. Fill it in the user inputs.', + description="The OAuth client ID. Fill it in the user inputs.", examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], - title='Client ID', + title="Client ID", ) client_secret: str = Field( ..., - description='The OAuth client secret. Fill it in the user inputs.', + description="The OAuth client secret. Fill it in the user inputs.", examples=[ "{{ config['client_secret }}", "{{ config['credentials']['client_secret }}", ], - title='Client Secret', + title="Client Secret", ) refresh_token: Optional[str] = Field( None, - description='Credential artifact used to get a new access token.', + description="Credential artifact used to get a new access token.", examples=[ "{{ config['refresh_token'] }}", "{{ config['credentials]['refresh_token'] }}", ], - title='Refresh Token', + title="Refresh Token", ) token_refresh_endpoint: str = Field( ..., - description='The full URL to call to obtain a new access token.', - examples=['https://connect.squareup.com/oauth2/token'], - title='Token Refresh Endpoint', + description="The full URL to call to obtain a new access token.", + examples=["https://connect.squareup.com/oauth2/token"], + title="Token Refresh Endpoint", ) access_token_name: Optional[str] = Field( - 'access_token', - description='The name of the property which contains the access token in the response from the token refresh endpoint.', - examples=['access_token'], - title='Access Token Property Name', + "access_token", + description="The name of the property which contains the access token in the response from the token refresh endpoint.", + examples=["access_token"], + title="Access Token Property Name", ) expires_in_name: Optional[str] = Field( - 'expires_in', - description='The name of the property which contains the expiry date in the response from the token refresh endpoint.', - examples=['expires_in'], - title='Token Expiry Property Name', + "expires_in", + description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", + examples=["expires_in"], + title="Token Expiry Property Name", ) grant_type: Optional[str] = Field( - 'refresh_token', - description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.', - examples=['refresh_token', 'client_credentials'], - title='Grant Type', + "refresh_token", + description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", + examples=["refresh_token", "client_credentials"], + title="Grant Type", ) refresh_request_body: Optional[Dict[str, Any]] = Field( None, - description='Body of the request sent to get a new access token.', + description="Body of the request sent to get a new access token.", examples=[ { - 'applicationId': "{{ config['application_id'] }}", - 'applicationSecret': "{{ config['application_secret'] }}", - 'token': "{{ config['token'] }}", + "applicationId": "{{ config['application_id'] }}", + "applicationSecret": "{{ config['application_secret'] }}", + "token": "{{ config['token'] }}", } ], - title='Refresh Request Body', + title="Refresh Request Body", ) scopes: Optional[List[str]] = Field( None, - description='List of scopes that should be granted to the access token.', - examples=[ - ['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read'] - ], - title='Scopes', + description="List of scopes that should be granted to the access token.", + examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + title="Scopes", ) token_expiry_date: Optional[str] = Field( None, - description='The access token expiry date.', - examples=['2023-04-06T07:12:10.421833+00:00', 1680842386], - title='Token Expiry Date', + description="The access token expiry date.", + examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], + title="Token Expiry Date", ) token_expiry_date_format: Optional[str] = Field( None, - description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.', - examples=['%Y-%m-%d %H:%M:%S.%f+00:00'], - title='Token Expiry Date Format', + description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", + examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], + title="Token Expiry Date Format", ) refresh_token_updater: Optional[RefreshTokenUpdater] = Field( None, - description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.', - title='Token Updater', + description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", + title="Token Updater", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ExponentialBackoffStrategy(BaseModel): - type: Literal['ExponentialBackoffStrategy'] + type: Literal["ExponentialBackoffStrategy"] factor: Optional[Union[float, str]] = Field( 5, - description='Multiplicative constant applied on each retry.', - examples=[5, 5.5, '10'], - title='Factor', + description="Multiplicative constant applied on each retry.", + examples=[5, 5.5, "10"], + title="Factor", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenRequestBearerAuthenticator(BaseModel): - type: Literal['Bearer'] + type: Literal["Bearer"] class HttpMethodEnum(Enum): - GET = 'GET' - POST = 'POST' + GET = "GET" + POST = "POST" class Action(Enum): - SUCCESS = 'SUCCESS' - FAIL = 'FAIL' - RETRY = 'RETRY' - IGNORE = 'IGNORE' + SUCCESS = "SUCCESS" + FAIL = "FAIL" + RETRY = "RETRY" + IGNORE = "IGNORE" class HttpResponseFilter(BaseModel): - type: Literal['HttpResponseFilter'] + type: Literal["HttpResponseFilter"] action: Action = Field( ..., - description='Action to execute if a response matches the filter.', - examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE'], - title='Action', + description="Action to execute if a response matches the filter.", + examples=["SUCCESS", "FAIL", "RETRY", "IGNORE"], + title="Action", ) error_message: Optional[str] = Field( None, - description='Error Message to display if the response matches the filter.', - title='Error Message', + description="Error Message to display if the response matches the filter.", + title="Error Message", ) error_message_contains: Optional[str] = Field( None, - description='Match the response if its error message contains the substring.', - example=['This API operation is not enabled for this site'], - title='Error Message Substring', + description="Match the response if its error message contains the substring.", + example=["This API operation is not enabled for this site"], + title="Error Message Substring", ) http_codes: Optional[List[int]] = Field( None, - description='Match the response if its HTTP code is included in this list.', + description="Match the response if its HTTP code is included in this list.", examples=[[420, 429], [500]], - title='HTTP Codes', + title="HTTP Codes", ) predicate: Optional[str] = Field( None, - description='Match the response if the predicate evaluates to true.', + description="Match the response if the predicate evaluates to true.", examples=[ "{{ 'Too much requests' in response }}", "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", ], - title='Predicate', + title="Predicate", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class InlineSchemaLoader(BaseModel): - type: Literal['InlineSchemaLoader'] + type: Literal["InlineSchemaLoader"] schema_: Optional[Dict[str, Any]] = Field( None, - alias='schema', + alias="schema", description='Describes a streams\' schema. Refer to the Data Types documentation for more details on which types are valid.', - title='Schema', + title="Schema", ) class JsonFileSchemaLoader(BaseModel): - type: Literal['JsonFileSchemaLoader'] + type: Literal["JsonFileSchemaLoader"] file_path: Optional[str] = Field( None, description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", - example=['./schemas/users.json'], - title='File Path', + example=["./schemas/users.json"], + title="File Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class JsonDecoder(BaseModel): - type: Literal['JsonDecoder'] + type: Literal["JsonDecoder"] class MinMaxDatetime(BaseModel): - type: Literal['MinMaxDatetime'] + type: Literal["MinMaxDatetime"] datetime: str = Field( ..., - description='Datetime value.', - examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"], - title='Datetime', + description="Datetime value.", + examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], + title="Datetime", ) datetime_format: Optional[str] = Field( - '', + "", description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'], - title='Datetime Format', + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], + title="Datetime Format", ) max_datetime: Optional[str] = Field( None, - description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.', - examples=['2021-01-01T00:00:00Z', '2021-01-01'], - title='Max Datetime', + description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", + examples=["2021-01-01T00:00:00Z", "2021-01-01"], + title="Max Datetime", ) min_datetime: Optional[str] = Field( None, - description='Floor applied on the datetime value. Must be formatted with the datetime_format field.', - examples=['2010-01-01T00:00:00Z', '2010-01-01'], - title='Min Datetime', + description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", + examples=["2010-01-01T00:00:00Z", "2010-01-01"], + title="Min Datetime", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class NoAuth(BaseModel): - type: Literal['NoAuth'] - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + type: Literal["NoAuth"] + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class NoPagination(BaseModel): - type: Literal['NoPagination'] + type: Literal["NoPagination"] class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[ - Dict[str, Any] - ] = Field( + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", examples=[ - {'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}}, + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, { - 'app_id': { - 'type': 'string', - 'path_in_connector_config': ['info', 'app_id'], + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], } }, ], - title='OAuth user input', + title="OAuth user input", ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", examples=[ { - 'refresh_token': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'refresh_token'], + "refresh_token": { + "type": "string,", + "path_in_connector_config": ["credentials", "refresh_token"], } } ], - title='OAuth output specification', + title="OAuth output specification", ) complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, - description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }', - examples=[ - {'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}} - ], - title='OAuth input specification', + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", + examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", examples=[ { - 'client_id': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'client_id'], + "client_id": { + "type": "string,", + "path_in_connector_config": ["credentials", "client_id"], }, - 'client_secret': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'client_secret'], + "client_secret": { + "type": "string,", + "path_in_connector_config": ["credentials", "client_secret"], }, } ], - title='OAuth server output specification', + title="OAuth server output specification", ) class OffsetIncrement(BaseModel): - type: Literal['OffsetIncrement'] + type: Literal["OffsetIncrement"] page_size: Optional[Union[int, str]] = Field( None, - description='The number of records to include in each pages.', + description="The number of records to include in each pages.", examples=[100, "{{ config['page_size'] }}"], - title='Limit', + title="Limit", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class PageIncrement(BaseModel): - type: Literal['PageIncrement'] + type: Literal["PageIncrement"] page_size: Optional[int] = Field( None, - description='The number of records to include in each pages.', - examples=[100, '100'], - title='Page Size', + description="The number of records to include in each pages.", + examples=[100, "100"], + title="Page Size", ) start_from_page: Optional[int] = Field( 0, - description='Index of the first page to request.', + description="Index of the first page to request.", examples=[0, 1], - title='Start From Page', + title="Start From Page", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class PrimaryKey(BaseModel): __root__: Union[str, List[str], List[List[str]]] = Field( ..., - description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.', - examples=['id', ['code', 'type']], - title='Primary Key', + description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", + examples=["id", ["code", "type"]], + title="Primary Key", ) class RecordFilter(BaseModel): - type: Literal['RecordFilter'] + type: Literal["RecordFilter"] condition: Optional[str] = Field( - '', - description='The predicate to filter a record. Records will be removed if evaluated to False.', + "", + description="The predicate to filter a record. Records will be removed if evaluated to False.", examples=[ "{{ record['created_at'] >= stream_interval['start_time'] }}", "{{ record.status in ['active', 'expired'] }}", ], ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RemoveFields(BaseModel): - type: Literal['RemoveFields'] + type: Literal["RemoveFields"] field_pointers: List[List[str]] = Field( ..., - description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.', - examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]], - title='Field Paths', + description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", + examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], + title="Field Paths", ) class RequestPath(BaseModel): - type: Literal['RequestPath'] + type: Literal["RequestPath"] class InjectInto(Enum): - request_parameter = 'request_parameter' - header = 'header' - body_data = 'body_data' - body_json = 'body_json' + request_parameter = "request_parameter" + header = "header" + body_data = "body_data" + body_json = "body_json" class RequestOption(BaseModel): - type: Literal['RequestOption'] + type: Literal["RequestOption"] field_name: str = Field( ..., - description='Configures which key should be used in the location that the descriptor is being injected into', - examples=['segment_id'], - title='Request Option', + description="Configures which key should be used in the location that the descriptor is being injected into", + examples=["segment_id"], + title="Request Option", ) inject_into: InjectInto = Field( ..., - description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.', - examples=['request_parameter', 'header', 'body_data', 'body_json'], - title='Inject Into', + description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", + examples=["request_parameter", "header", "body_data", "body_json"], + title="Inject Into", ) @@ -638,253 +632,251 @@ class Config: class LegacySessionTokenAuthenticator(BaseModel): - type: Literal['LegacySessionTokenAuthenticator'] + type: Literal["LegacySessionTokenAuthenticator"] header: str = Field( ..., - description='The name of the session token header that will be injected in the request', - examples=['X-Session'], - title='Session Request Header', + description="The name of the session token header that will be injected in the request", + examples=["X-Session"], + title="Session Request Header", ) login_url: str = Field( ..., - description='Path of the login URL (do not include the base URL)', - examples=['session'], - title='Login Path', + description="Path of the login URL (do not include the base URL)", + examples=["session"], + title="Login Path", ) session_token: Optional[str] = Field( None, - description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair', + description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", example=["{{ config['session_token'] }}"], - title='Session Token', + title="Session Token", ) session_token_response_key: str = Field( ..., - description='Name of the key of the session token to be extracted from the response', - examples=['id'], - title='Response Token Response Key', + description="Name of the key of the session token to be extracted from the response", + examples=["id"], + title="Response Token Response Key", ) username: Optional[str] = Field( None, - description='Username used to authenticate and obtain a session token', + description="Username used to authenticate and obtain a session token", examples=[" {{ config['username'] }}"], - title='Username', + title="Username", ) password: Optional[str] = Field( - '', - description='Password used to authenticate and obtain a session token', - examples=["{{ config['password'] }}", ''], - title='Password', + "", + description="Password used to authenticate and obtain a session token", + examples=["{{ config['password'] }}", ""], + title="Password", ) validate_session_url: str = Field( ..., - description='Path of the URL to use to validate that the session token is valid (do not include the base URL)', - examples=['user/current'], - title='Validate Session Path', + description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", + examples=["user/current"], + title="Validate Session Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class WaitTimeFromHeader(BaseModel): - type: Literal['WaitTimeFromHeader'] + type: Literal["WaitTimeFromHeader"] header: str = Field( ..., - description='The name of the response header defining how long to wait before retrying.', - examples=['Retry-After'], - title='Response Header Name', + description="The name of the response header defining how long to wait before retrying.", + examples=["Retry-After"], + title="Response Header Name", ) regex: Optional[str] = Field( None, - description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', - examples=['([-+]?\\d+)'], - title='Extraction Regex', + description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", + examples=["([-+]?\\d+)"], + title="Extraction Regex", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class WaitUntilTimeFromHeader(BaseModel): - type: Literal['WaitUntilTimeFromHeader'] + type: Literal["WaitUntilTimeFromHeader"] header: str = Field( ..., - description='The name of the response header defining how long to wait before retrying.', - examples=['wait_time'], - title='Response Header', + description="The name of the response header defining how long to wait before retrying.", + examples=["wait_time"], + title="Response Header", ) min_wait: Optional[Union[float, str]] = Field( None, - description='Minimum time to wait before retrying.', - examples=[10, '60'], - title='Minimum Wait Time', + description="Minimum time to wait before retrying.", + examples=[10, "60"], + title="Minimum Wait Time", ) regex: Optional[str] = Field( None, - description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', - examples=['([-+]?\\d+)'], - title='Extraction Regex', + description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", + examples=["([-+]?\\d+)"], + title="Extraction Regex", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ApiKeyAuthenticator(BaseModel): - type: Literal['ApiKeyAuthenticator'] + type: Literal["ApiKeyAuthenticator"] api_token: Optional[str] = Field( None, - description='The API key to inject in the request. Fill it in the user inputs.', + description="The API key to inject in the request. Fill it in the user inputs.", examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], - title='API Key', + title="API Key", ) header: Optional[str] = Field( None, - description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.', - examples=['Authorization', 'Api-Token', 'X-Auth-Token'], - title='Header Name', + description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", + examples=["Authorization", "Api-Token", "X-Auth-Token"], + title="Header Name", ) inject_into: Optional[RequestOption] = Field( None, - description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.', + description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", examples=[ - {'inject_into': 'header', 'field_name': 'Authorization'}, - {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, ], - title='Inject API Key Into Outgoing HTTP Request', + title="Inject API Key Into Outgoing HTTP Request", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AuthFlow(BaseModel): - auth_flow_type: Optional[AuthFlowType] = Field( - None, description='The type of auth to use', title='Auth flow type' - ) + auth_flow_type: Optional[AuthFlowType] = Field(None, description="The type of auth to use", title="Auth flow type") predicate_key: Optional[List[str]] = Field( None, - description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.', - examples=[['credentials', 'auth_type']], - title='Predicate key', + description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", + examples=[["credentials", "auth_type"]], + title="Predicate key", ) predicate_value: Optional[str] = Field( None, - description='Value of the predicate_key fields for the advanced auth to be applicable.', - examples=['Oauth'], - title='Predicate value', + description="Value of the predicate_key fields for the advanced auth to be applicable.", + examples=["Oauth"], + title="Predicate value", ) oauth_config_specification: Optional[OAuthConfigSpecification] = None class CursorPagination(BaseModel): - type: Literal['CursorPagination'] + type: Literal["CursorPagination"] cursor_value: str = Field( ..., - description='Value of the cursor defining the next page to fetch.', + description="Value of the cursor defining the next page to fetch.", examples=[ - '{{ headers.link.next.cursor }}', + "{{ headers.link.next.cursor }}", "{{ last_records[-1]['key'] }}", "{{ response['nextPage'] }}", ], - title='Cursor Value', + title="Cursor Value", ) page_size: Optional[int] = Field( None, - description='The number of records to include in each pages.', + description="The number of records to include in each pages.", examples=[100], - title='Page Size', + title="Page Size", ) stop_condition: Optional[str] = Field( None, - description='Template string evaluating when to stop paginating.', + description="Template string evaluating when to stop paginating.", examples=[ - '{{ response.data.has_more is false }}', + "{{ response.data.has_more is false }}", "{{ 'next' not in headers['link'] }}", ], - title='Stop Condition', + title="Stop Condition", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DatetimeBasedCursor(BaseModel): - type: Literal['DatetimeBasedCursor'] + type: Literal["DatetimeBasedCursor"] cursor_field: str = Field( ..., - description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.', - examples=['created_at', "{{ config['record_cursor'] }}"], - title='Cursor Field', + description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", + examples=["created_at", "{{ config['record_cursor'] }}"], + title="Cursor Field", ) datetime_format: str = Field( ..., - description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms'], - title='Outgoing Datetime Format', + description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms"], + title="Outgoing Datetime Format", ) start_datetime: Union[str, MinMaxDatetime] = Field( ..., - description='The datetime that determines the earliest record that should be synced.', - examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"], - title='Start Datetime', + description="The datetime that determines the earliest record that should be synced.", + examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], + title="Start Datetime", ) cursor_datetime_formats: Optional[List[str]] = Field( None, - description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.', - title='Cursor Datetime Formats', + description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", + title="Cursor Datetime Formats", ) cursor_granularity: Optional[str] = Field( None, - description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.', - examples=['PT1S'], - title='Cursor Granularity', + description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", + examples=["PT1S"], + title="Cursor Granularity", ) end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( None, - description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.', - examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'], - title='End Datetime', + description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", + examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], + title="End Datetime", ) end_time_option: Optional[RequestOption] = Field( None, - description='Optionally configures how the end datetime will be sent in requests to the source API.', - title='Inject End Time Into Outgoing HTTP Request', + description="Optionally configures how the end datetime will be sent in requests to the source API.", + title="Inject End Time Into Outgoing HTTP Request", ) is_data_feed: Optional[bool] = Field( None, - description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.', - title='Whether the target API is formatted as a data feed', + description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", + title="Whether the target API is formatted as a data feed", ) lookback_window: Optional[str] = Field( None, - description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.', - examples=['P1D', "P{{ config['lookback_days'] }}D"], - title='Lookback Window', + description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", + examples=["P1D", "P{{ config['lookback_days'] }}D"], + title="Lookback Window", ) partition_field_end: Optional[str] = Field( None, - description='Name of the partition start time field.', - examples=['ending_time'], - title='Partition Field End', + description="Name of the partition start time field.", + examples=["ending_time"], + title="Partition Field End", ) partition_field_start: Optional[str] = Field( None, - description='Name of the partition end time field.', - examples=['starting_time'], - title='Partition Field Start', + description="Name of the partition end time field.", + examples=["starting_time"], + title="Partition Field Start", ) start_time_option: Optional[RequestOption] = Field( None, - description='Optionally configures how the start datetime will be sent in requests to the source API.', - title='Inject Start Time Into Outgoing HTTP Request', + description="Optionally configures how the start datetime will be sent in requests to the source API.", + title="Inject Start Time Into Outgoing HTTP Request", ) step: Optional[str] = Field( None, - description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.', - examples=['P1W', "{{ config['step_increment'] }}"], - title='Step', + description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", + examples=["P1W", "{{ config['step_increment'] }}"], + title="Step", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DefaultErrorHandler(BaseModel): - type: Literal['DefaultErrorHandler'] + type: Literal["DefaultErrorHandler"] backoff_strategies: Optional[ List[ Union[ @@ -897,144 +889,142 @@ class DefaultErrorHandler(BaseModel): ] ] = Field( None, - description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.', - title='Backoff Strategies', + description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", + title="Backoff Strategies", ) max_retries: Optional[int] = Field( 5, - description='The maximum number of time to retry a retryable request before giving up and failing.', + description="The maximum number of time to retry a retryable request before giving up and failing.", examples=[5, 0, 10], - title='Max Retry Count', + title="Max Retry Count", ) response_filters: Optional[List[HttpResponseFilter]] = Field( None, description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", - title='Response Filters', + title="Response Filters", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DefaultPaginator(BaseModel): - type: Literal['DefaultPaginator'] - pagination_strategy: Union[ - CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement - ] = Field( + type: Literal["DefaultPaginator"] + pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement] = Field( ..., - description='Strategy defining how records are paginated.', - title='Pagination Strategy', + description="Strategy defining how records are paginated.", + title="Pagination Strategy", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) page_size_option: Optional[RequestOption] = None page_token_option: Optional[Union[RequestOption, RequestPath]] = None - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DpathExtractor(BaseModel): - type: Literal['DpathExtractor'] + type: Literal["DpathExtractor"] field_path: List[str] = Field( ..., description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', examples=[ - ['data'], - ['data', 'records'], - ['data', '{{ parameters.name }}'], - ['data', '*', 'record'], + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], ], - title='Field Path', + title="Field Path", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenRequestApiKeyAuthenticator(BaseModel): - type: Literal['ApiKey'] + type: Literal["ApiKey"] inject_into: RequestOption = Field( ..., - description='Configure how the API Key will be sent in requests to the source API.', + description="Configure how the API Key will be sent in requests to the source API.", examples=[ - {'inject_into': 'header', 'field_name': 'Authorization'}, - {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, ], - title='Inject API Key Into Outgoing HTTP Request', + title="Inject API Key Into Outgoing HTTP Request", ) class ListPartitionRouter(BaseModel): - type: Literal['ListPartitionRouter'] + type: Literal["ListPartitionRouter"] cursor_field: str = Field( ..., description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', - examples=['section', "{{ config['section_key'] }}"], - title='Current Partition Value Identifier', + examples=["section", "{{ config['section_key'] }}"], + title="Current Partition Value Identifier", ) values: Union[str, List[str]] = Field( ..., - description='The list of attributes being iterated over and used as input for the requests made to the source API.', - examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"], - title='Partition Values', + description="The list of attributes being iterated over and used as input for the requests made to the source API.", + examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], + title="Partition Values", ) request_option: Optional[RequestOption] = Field( None, - description='A request option describing where the list value should be injected into and under what field name if applicable.', - title='Inject Partition Value Into Outgoing HTTP Request', + description="A request option describing where the list value should be injected into and under what field name if applicable.", + title="Inject Partition Value Into Outgoing HTTP Request", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RecordSelector(BaseModel): - type: Literal['RecordSelector'] + type: Literal["RecordSelector"] extractor: Union[CustomRecordExtractor, DpathExtractor] record_filter: Optional[RecordFilter] = Field( None, - description='Responsible for filtering records to be emitted by the Source.', - title='Record Filter', + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class Spec(BaseModel): - type: Literal['Spec'] + type: Literal["Spec"] connection_specification: Dict[str, Any] = Field( ..., - description='A connection specification describing how a the connector can be configured.', - title='Connection Specification', + description="A connection specification describing how a the connector can be configured.", + title="Connection Specification", ) documentation_url: Optional[str] = Field( None, description="URL of the connector's documentation page.", - examples=['https://docs.airbyte.com/integrations/sources/dremio'], - title='Documentation URL', + examples=["https://docs.airbyte.com/integrations/sources/dremio"], + title="Documentation URL", ) advanced_auth: Optional[AuthFlow] = Field( None, - description='Advanced specification for configuring the authentication flow.', - title='Advanced Auth', + description="Advanced specification for configuring the authentication flow.", + title="Advanced Auth", ) class CompositeErrorHandler(BaseModel): - type: Literal['CompositeErrorHandler'] + type: Literal["CompositeErrorHandler"] error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( ..., - description='List of error handlers to iterate on to determine how to handle a failed response.', - title='Error Handlers', + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DeclarativeSource(BaseModel): class Config: extra = Extra.forbid - type: Literal['DeclarativeSource'] + type: Literal["DeclarativeSource"] check: CheckStream streams: List[DeclarativeStream] version: str @@ -1043,7 +1033,7 @@ class Config: spec: Optional[Spec] = None metadata: Optional[Dict[str, Any]] = Field( None, - description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.', + description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", ) @@ -1051,101 +1041,91 @@ class DeclarativeStream(BaseModel): class Config: extra = Extra.allow - type: Literal['DeclarativeStream'] + type: Literal["DeclarativeStream"] retriever: Union[CustomRetriever, SimpleRetriever] = Field( ..., - description='Component used to coordinate how records are extracted across stream slices and request pages.', - title='Retriever', + description="Component used to coordinate how records are extracted across stream slices and request pages.", + title="Retriever", ) - incremental_sync: Optional[ - Union[CustomIncrementalSync, DatetimeBasedCursor] - ] = Field( + incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field( None, - description='Component used to fetch data incrementally based on a time field in the data.', - title='Incremental Sync', - ) - name: Optional[str] = Field( - '', description='The stream name.', example=['Users'], title='Name' - ) - primary_key: Optional[PrimaryKey] = Field( - '', description='The primary key of the stream.', title='Primary Key' + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", ) + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key") schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field( None, - description='Component used to retrieve the schema for the current stream.', - title='Schema Loader', + description="Component used to retrieve the schema for the current stream.", + title="Schema Loader", ) - transformations: Optional[ - List[Union[AddFields, CustomTransformation, RemoveFields]] - ] = Field( + transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = Field( None, - description='A list of transformations to be applied to each output record.', - title='Transformations', + description="A list of transformations to be applied to each output record.", + title="Transformations", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenAuthenticator(BaseModel): - type: Literal['SessionTokenAuthenticator'] + type: Literal["SessionTokenAuthenticator"] login_requester: HttpRequester = Field( ..., - description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.', + description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", examples=[ { - 'type': 'HttpRequester', - 'url_base': 'https://my_api.com', - 'path': '/login', - 'authenticator': { - 'type': 'BasicHttpAuthenticator', - 'username': '{{ config.username }}', - 'password': '{{ config.password }}', + "type": "HttpRequester", + "url_base": "https://my_api.com", + "path": "/login", + "authenticator": { + "type": "BasicHttpAuthenticator", + "username": "{{ config.username }}", + "password": "{{ config.password }}", }, } ], - title='Login Requester', + title="Login Requester", ) session_token_path: List[str] = Field( ..., - description='The path in the response body returned from the login requester to the session token.', - examples=[['access_token'], ['result', 'token']], - title='Session Token Path', + description="The path in the response body returned from the login requester to the session token.", + examples=[["access_token"], ["result", "token"]], + title="Session Token Path", ) expiration_duration: Optional[str] = Field( None, - description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.', - examples=['PT1H', 'P1D'], - title='Expiration Duration', + description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", + examples=["PT1H", "P1D"], + title="Expiration Duration", ) - request_authentication: Union[ - SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator - ] = Field( + request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator] = Field( ..., - description='Authentication method to use for requests sent to the API, specifying how to inject the session token.', - title='Data Request Authentication', + description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", + title="Data Request Authentication", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class HttpRequester(BaseModel): - type: Literal['HttpRequester'] + type: Literal["HttpRequester"] url_base: str = Field( ..., - description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ - 'https://connect.squareup.com/v2', + "https://connect.squareup.com/v2", "{{ config['base_url'] or 'https://app.posthog.com'}}/api/", ], - title='API Base URL', + title="API Base URL", ) path: str = Field( ..., - description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ - '/products', + "/products", "/quotes/{{ stream_partition['id'] }}/quote_line_groups", "/trades/{{ config['symbol_id'] }}/history", ], - title='URL Path', + title="URL Path", ) authenticator: Optional[ Union[ @@ -1160,96 +1140,92 @@ class HttpRequester(BaseModel): ] ] = Field( None, - description='Authentication method to use for requests sent to the API.', - title='Authenticator', + description="Authentication method to use for requests sent to the API.", + title="Authenticator", ) - error_handler: Optional[ - Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] - ] = Field( + error_handler: Optional[Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]] = Field( None, - description='Error handler component that defines how to handle errors.', - title='Error Handler', + description="Error handler component that defines how to handle errors.", + title="Error Handler", ) http_method: Optional[Union[str, HttpMethodEnum]] = Field( - 'GET', - description='The HTTP method used to fetch data from the source (can be GET or POST).', - examples=['GET', 'POST'], - title='HTTP Method', + "GET", + description="The HTTP method used to fetch data from the source (can be GET or POST).", + examples=["GET", "POST"], + title="HTTP Method", ) request_body_data: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.', + description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", examples=[ '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' ], - title='Request Body Payload (Non-JSON)', + title="Request Body Payload (Non-JSON)", ) request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( None, - description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.', + description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", examples=[ - {'sort_order': 'ASC', 'sort_field': 'CREATED_AT'}, - {'key': "{{ config['value'] }}"}, - {'sort': {'field': 'updated_at', 'order': 'ascending'}}, + {"sort_order": "ASC", "sort_field": "CREATED_AT"}, + {"key": "{{ config['value'] }}"}, + {"sort": {"field": "updated_at", "order": "ascending"}}, ], - title='Request Body JSON Payload', + title="Request Body JSON Payload", ) request_headers: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.', - examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}], - title='Request Headers', + description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", + examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], + title="Request Headers", ) request_parameters: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.', + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", examples=[ - {'unit': 'day'}, + {"unit": "day"}, { - 'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' }, - {'searchIn': "{{ ','.join(config.get('search_in', [])) }}"}, - {'sort_by[asc]': 'updated_at'}, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, ], - title='Query Parameters', + title="Query Parameters", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ParentStreamConfig(BaseModel): - type: Literal['ParentStreamConfig'] + type: Literal["ParentStreamConfig"] parent_key: str = Field( ..., - description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.', - examples=['id', "{{ config['parent_record_id'] }}"], - title='Parent Key', - ) - stream: DeclarativeStream = Field( - ..., description='Reference to the parent stream.', title='Parent Stream' + description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", + examples=["id", "{{ config['parent_record_id'] }}"], + title="Parent Key", ) + stream: DeclarativeStream = Field(..., description="Reference to the parent stream.", title="Parent Stream") partition_field: str = Field( ..., - description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.', - examples=['parent_id', "{{ config['parent_partition_field'] }}"], - title='Current Parent Key Value Identifier', + description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", + examples=["parent_id", "{{ config['parent_partition_field'] }}"], + title="Current Parent Key Value Identifier", ) request_option: Optional[RequestOption] = Field( None, - description='A request option describing where the parent key value should be injected into and under what field name if applicable.', - title='Request Option', + description="A request option describing where the parent key value should be injected into and under what field name if applicable.", + title="Request Option", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SimpleRetriever(BaseModel): - type: Literal['SimpleRetriever'] + type: Literal["SimpleRetriever"] record_selector: RecordSelector = Field( ..., - description='Component that describes how to extract records from a HTTP response.', + description="Component that describes how to extract records from a HTTP response.", ) requester: Union[CustomRequester, HttpRequester] = Field( ..., - description='Requester component that describes how to prepare HTTP requests to send to the source API.', + description="Requester component that describes how to prepare HTTP requests to send to the source API.", ) paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( None, @@ -1260,28 +1236,24 @@ class SimpleRetriever(BaseModel): CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, - List[ - Union[ - CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter - ] - ], + List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], ] ] = Field( [], - description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.', - title='Partition Router', + description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", + title="Partition Router", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SubstreamPartitionRouter(BaseModel): - type: Literal['SubstreamPartitionRouter'] + type: Literal["SubstreamPartitionRouter"] parent_stream_configs: List[ParentStreamConfig] = Field( ..., - description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.', - title='Parent Stream Configs', + description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", + title="Parent Stream Configs", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") CompositeErrorHandler.update_forward_refs() diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java index 9625259aa007..0c5408b475ea 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java @@ -19,15 +19,11 @@ import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; -import io.airbyte.integrations.destination.s3.avro.JsonToAvroSchemaConverter; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.io.IOException; import java.util.Map; -import java.util.function.BiFunction; -import java.util.function.Function; import javax.annotation.Nullable; -import org.apache.avro.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java index 3d21bf42b420..fb9114b990bc 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java @@ -13,7 +13,6 @@ import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.List; import java.util.Map; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 09087c4cc530..80373997df5c 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -395,7 +395,7 @@ private TyperDeduper buildTyperDeduper(final BigQuerySqlGenerator sqlGenerator, parsedCatalog, migrator, v2RawTableMigrator, - 8); + 8); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java index 21417c2927d6..eaf077d986d7 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java @@ -133,29 +133,29 @@ public void copyIntoTableFromStage(final String datasetId, LOGGER.info("Uploading records from staging files to target table {} (dataset {}): {}", tableId, datasetId, stagedFileName); - final String fullFilePath = String.format("gs://%s/%s%s", gcsConfig.getBucketName(), getStagingFullPath(datasetId, stream), stagedFileName); - LOGGER.info("Uploading staged file: {}", fullFilePath); - final LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, fullFilePath) - .setFormatOptions(FormatOptions.csv()) - .setSchema(tableSchema) - .setWriteDisposition(WriteDisposition.WRITE_APPEND) - .setJobTimeoutMs(60000L) - .build(); - - final Job loadJob = this.bigQuery.create(JobInfo.of(configuration)); - LOGGER.info("[{}] Created a new job to upload record(s) to target table {} (dataset {}): {}", loadJob.getJobId(), - tableId, datasetId, loadJob); - - try { - BigQueryUtils.waitForJobFinish(loadJob); - LOGGER.info("[{}] Target table {} (dataset {}) is successfully appended with staging files", loadJob.getJobId(), - tableId, datasetId); - } catch (final BigQueryException | InterruptedException e) { - throw new RuntimeException( - String.format("[%s] Failed to upload staging files to destination table %s (%s)", loadJob.getJobId(), - tableId, datasetId), - e); - } + final String fullFilePath = String.format("gs://%s/%s%s", gcsConfig.getBucketName(), getStagingFullPath(datasetId, stream), stagedFileName); + LOGGER.info("Uploading staged file: {}", fullFilePath); + final LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, fullFilePath) + .setFormatOptions(FormatOptions.csv()) + .setSchema(tableSchema) + .setWriteDisposition(WriteDisposition.WRITE_APPEND) + .setJobTimeoutMs(60000L) + .build(); + + final Job loadJob = this.bigQuery.create(JobInfo.of(configuration)); + LOGGER.info("[{}] Created a new job to upload record(s) to target table {} (dataset {}): {}", loadJob.getJobId(), + tableId, datasetId, loadJob); + + try { + BigQueryUtils.waitForJobFinish(loadJob); + LOGGER.info("[{}] Target table {} (dataset {}) is successfully appended with staging files", loadJob.getJobId(), + tableId, datasetId); + } catch (final BigQueryException | InterruptedException e) { + throw new RuntimeException( + String.format("[%s] Failed to upload staging files to destination table %s (%s)", loadJob.getJobId(), + tableId, datasetId), + e); + } } @Override diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java index 73ce256e1dc1..9996b0d16711 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java @@ -69,8 +69,8 @@ public SerializedAirbyteMessageConsumer createAsync( } /** - * Out BigQuery's uploader threads use a fair amount of memory. We believe this is largely - * due to the sdk client we use. + * Out BigQuery's uploader threads use a fair amount of memory. We believe this is largely due to + * the sdk client we use. * * @return number of bytes to make available for message buffering. */ @@ -79,10 +79,10 @@ private long getBigQueryBufferMemoryLimit() { } private Map createWriteConfigs(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final ParsedCatalog parsedCatalog, - final Function recordFormatterCreator, - final Function tmpTableNameTransformer) { + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog, + final Function recordFormatterCreator, + final Function tmpTableNameTransformer) { return catalog.getStreams().stream() .map(configuredStream -> { Preconditions.checkNotNull(configuredStream.getDestinationSyncMode(), "Undefined destination sync mode"); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java index 148c7c95382a..7e51dcdfce74 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java @@ -46,4 +46,5 @@ public BigQueryWriteConfig(final String streamName, tableSchema, syncMode); } + } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java index 7f27d5e83d6d..affaef8a3d4d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java @@ -72,8 +72,7 @@ public void execute(final String sql) throws InterruptedException { if (job.getStatus().getError() != null) { throw new BigQueryException(Streams.concat( Stream.of(job.getStatus().getError()), - job.getStatus().getExecutionErrors().stream() - ).toList()); + job.getStatus().getExecutionErrors().stream()).toList()); } final JobStatistics.QueryStatistics statistics = job.getStatistics(); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java index 3e5c73260a1d..2d5a8644fa33 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java @@ -17,7 +17,6 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.nio.file.Path; import java.util.HashSet; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py index 5b6c892dd7a2..6e3d0882130f 100644 --- a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py +++ b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py @@ -14,14 +14,7 @@ import duckdb from airbyte_cdk import AirbyteLogger from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import ( - AirbyteConnectionStatus, - AirbyteMessage, - ConfiguredAirbyteCatalog, - DestinationSyncMode, - Status, - Type, -) +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type logger = getLogger("airbyte") @@ -46,9 +39,7 @@ def _get_destination_path(destination_path: str) -> str: Get a normalized version of the destination path. Automatically append /local/ to the start of the path """ - if destination_path.startswith("md:") or destination_path.startswith( - "motherduck:" - ): + if destination_path.startswith("md:") or destination_path.startswith("motherduck:"): return destination_path if not destination_path.startswith("/local"): @@ -57,8 +48,7 @@ def _get_destination_path(destination_path: str) -> str: destination_path = os.path.normpath(destination_path) if not destination_path.startswith("/local"): raise ValueError( - f"destination_path={destination_path} is not a valid path." - "A valid path shall start with /local or no / prefix" + f"destination_path={destination_path} is not a valid path." "A valid path shall start with /local or no / prefix" ) return destination_path @@ -140,9 +130,7 @@ def write( data = message.record.data stream = message.record.stream if stream not in streams: - logger.debug( - f"Stream {stream} was not present in configured streams, skipping" - ) + logger.debug(f"Stream {stream} was not present in configured streams, skipping") continue # add to buffer @@ -167,9 +155,7 @@ def write( con.executemany(query, buffer[stream_name]) con.commit() - def check( - self, logger: AirbyteLogger, config: Mapping[str, Any] - ) -> AirbyteConnectionStatus: + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: """ Tests if the input configuration can be used to successfully connect to the destination with the needed permissions e.g: if a provided API token or password can be used to connect and write to the destination. @@ -198,6 +184,4 @@ def check( return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: - return AirbyteConnectionStatus( - status=Status.FAILED, message=f"An exception occurred: {repr(e)}" - ) + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java index 644f2c54e857..e5c7339c28a4 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java @@ -85,7 +85,8 @@ public void execute(final String sql) throws Exception { database.execute(sql); } catch (final SnowflakeSQLException e) { LOGGER.error("Sql {} failed", queryId, e); - // Snowflake SQL exceptions by default may not be super helpful, so we try to extract the relevant part of the message. + // Snowflake SQL exceptions by default may not be super helpful, so we try to extract the relevant + // part of the message. final String trimmedMessage; if (e.getMessage().startsWith(EXCEPTION_COMMON_PREFIX)) { // The first line is a pretty generic message, so just remove it diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java index 7138ce1abb4d..e0af9b0dcfba 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java @@ -36,7 +36,8 @@ public class SnowflakeSqlGenerator implements SqlGenerator RESERVED_COLUMN_NAMES = ImmutableList.of( "CURRENT_DATE", "CURRENT_TIME", @@ -62,8 +63,8 @@ public ColumnId buildColumnId(final String name, final String suffix) { // No escaping needed, as far as I can tell. We quote all our identifier names. final String nameWithSuffix = name + suffix; return new ColumnId(prefixReservedColumnName(escapeSqlIdentifier(name).toUpperCase()) + suffix, - nameWithSuffix, - nameWithSuffix.toUpperCase()); + nameWithSuffix, + nameWithSuffix.toUpperCase()); } public String toDialectType(final AirbyteType type) { @@ -196,18 +197,22 @@ private String extractAndCast(final ColumnId column, final AirbyteType airbyteTy } /** - * The `${` bigram causes problems inside script blocks. For example, a perfectly innocuous query like - * `SELECT "_airbyte_data":"${foo}" FROM ...` works fine normally, but running this block will throw - * an error: - * 
{@code
+   * The `${` bigram causes problems inside script blocks. For example, a perfectly innocuous query
+   * like `SELECT "_airbyte_data":"${foo}" FROM ...` works fine normally, but running this block will
+   * throw an error:
+   *
+   * 
+   * {@code
    * EXECUTE IMMEDIATE 'BEGIN
    * LET x INTEGER := (SELECT "_airbyte_data":"${foo}" FROM ...);
    * END;';
-   * }

+   * }
+   *
*

- * This method is a workaround for this behavior. We switch to using the {@code get} method to extract - * JSON values, and avoid the `${` sequence by using string concatenation. This generates a sql statement - * like {@code SELECT TRY_CAST((get("_airbyte_data", '$' + '{foo}')::text) as INTEGER) FROM ...}. + * This method is a workaround for this behavior. We switch to using the {@code get} method to + * extract JSON values, and avoid the `${` sequence by using string concatenation. This generates a + * sql statement like {@code SELECT TRY_CAST((get("_airbyte_data", '$' + '{foo}')::text) as INTEGER) + * FROM ...}. */ private String extractAndCastInsideScript(final ColumnId column, final AirbyteType airbyteType) { final String[] parts = column.originalName().split("\\$\\{", -1); @@ -561,8 +566,8 @@ public String migrateFromV1toV2(final StreamId streamId, final String namespace, } /** - * Snowflake json object access is done using double-quoted strings, e.g. `SELECT "_airbyte_data":"foo"`. - * As such, we need to escape double-quotes in the field name. + * Snowflake json object access is done using double-quoted strings, e.g. `SELECT + * "_airbyte_data":"foo"`. As such, we need to escape double-quotes in the field name. */ public static String escapeJsonIdentifier(final String identifier) { // Note that we don't need to escape backslashes here! @@ -592,8 +597,7 @@ public static String escapeSqlIdentifier(String identifier) { } private static String prefixReservedColumnName(final String columnName) { - return RESERVED_COLUMN_NAMES.stream().anyMatch(k -> k.equalsIgnoreCase(columnName)) ? - "_" + columnName : columnName; + return RESERVED_COLUMN_NAMES.stream().anyMatch(k -> k.equalsIgnoreCase(columnName)) ? "_" + columnName : columnName; } public static String escapeSingleQuotedString(final String str) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java index 26c28944495a..0cf13c579d13 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java @@ -37,7 +37,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.sql.DataSource; -import net.snowflake.client.jdbc.SnowflakeSQLException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; import org.junit.jupiter.api.AfterAll; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java index ed92be45193e..49cc80abecec 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.snowflake.typing_deduping; import static org.junit.jupiter.api.Assertions.assertAll; @@ -19,20 +23,15 @@ void columnNameSpecialCharacterHandling() { new ColumnId( "__FOO_", "${foo}", - "${FOO}" - ), - generator.buildColumnId("${foo}") - ), + "${FOO}"), + generator.buildColumnId("${foo}")), // But normally, we should leave those characters untouched. () -> assertEquals( new ColumnId( "{FO$O}", "{fo$o}", - "{FO$O}" - ), - generator.buildColumnId("{fo$o}") - ) - ); + "{FO$O}"), + generator.buildColumnId("{fo$o}"))); } /** @@ -48,10 +47,8 @@ void streamNameSpecialCharacterHandling() { "airbyte_internal", "__foo__raw__stream___bar_", "${foo}", - "${bar}" - ), - generator.buildStreamId("${foo}", "${bar}", "airbyte_internal") - ), + "${bar}"), + generator.buildStreamId("${foo}", "${bar}", "airbyte_internal")), () -> assertEquals( new StreamId( "{FO$O}", @@ -59,10 +56,8 @@ void streamNameSpecialCharacterHandling() { "airbyte_internal", "{fo$o}_raw__stream_{ba$r}", "{fo$o}", - "{ba$r}" - ), - generator.buildStreamId("{fo$o}", "{ba$r}", "airbyte_internal") - ) - ); + "{ba$r}"), + generator.buildStreamId("{fo$o}", "{ba$r}", "airbyte_internal"))); } + } diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index d7b2322d7c7f..4b3349457147 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -7,7 +7,7 @@ data: enabled: true cloud: enabled: true - dockerImageTag: 0.2.0 # https://github.com/airbytehq/airbyte/issues/30478 + dockerImageTag: 0.2.0 # https://github.com/airbytehq/airbyte/issues/30478 connectorSubtype: api connectorType: source definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json index 9eb524b550d7..2b0206c69c64 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json @@ -4,191 +4,121 @@ "stream": { "name": "customers", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "date_modified" - ] + "default_cursor_field": ["date_modified"] }, "sync_mode": "incremental", - "cursor_field": [ - "date_modified" - ], + "cursor_field": ["date_modified"], "destination_sync_mode": "append" }, { "stream": { "name": "orders", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "date_modified" - ] + "default_cursor_field": ["date_modified"] }, "sync_mode": "incremental", - "cursor_field": [ - "date_modified" - ], + "cursor_field": ["date_modified"], "destination_sync_mode": "append" }, { "stream": { "name": "transactions", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" }, { "stream": { "name": "pages", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" }, { "stream": { "name": "brands", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" }, { "stream": { "name": "categories", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" }, { "stream": { "name": "products", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "date_modified" - ] + "default_cursor_field": ["date_modified"] }, "sync_mode": "incremental", - "cursor_field": [ - "date_modified" - ], + "cursor_field": ["date_modified"], "destination_sync_mode": "append" }, { "stream": { "name": "channels", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "date_modified" - ] + "default_cursor_field": ["date_modified"] }, "sync_mode": "incremental", - "cursor_field": [ - "date_modified" - ], + "cursor_field": ["date_modified"], "destination_sync_mode": "append" }, { "stream": { "name": "store", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "store_id" - ] + "default_cursor_field": ["store_id"] }, "sync_mode": "incremental", - "cursor_field": [ - "store_id" - ], + "cursor_field": ["store_id"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "order_products", "json_schema": {}, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, - "default_cursor_field": [ - "id" - ] + "default_cursor_field": ["id"] }, "sync_mode": "incremental", - "cursor_field": [ - "id" - ], + "cursor_field": ["id"], "destination_sync_mode": "append" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json index 6e6e1d09f59c..78e6edc8221e 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json @@ -36,7 +36,7 @@ }, "icon_url": { "type": ["null", "string"] - }, + }, "is_enabled": { "type": ["null", "boolean"] } diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/pages.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/pages.json index 00fded5a8333..003984b52eea 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/pages.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/pages.json @@ -58,6 +58,6 @@ }, "id": { "type": ["null", "integer"] - } + } } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json index b2f6edc47478..ce96ff0a6a99 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json @@ -28,7 +28,7 @@ "bid_strategy": { "type": ["null", "string"] }, - "boosted_object_id":{ + "boosted_object_id": { "type": ["null", "string"] }, "budget_rebalance_flag": { diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json index c812570a11d4..4601d3d2a510 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -6,12 +6,22 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["segments.date"], ["segments.ad_network_type"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["segments.date"], ["segments.ad_network_type"]], + "primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["segments.date"], + ["segments.ad_network_type"] + ], "cursor_field": ["segments.date"] }, { @@ -22,23 +32,23 @@ "source_defined_cursor": true, "default_cursor_field": ["segments.date"], "source_defined_primary_key": [ - ["customer.id"], - ["campaign_budget.id"], - ["segments.date"], - ["segments.budget_campaign_association_status.campaign"], - ["segments.budget_campaign_association_status.status"] + ["customer.id"], + ["campaign_budget.id"], + ["segments.date"], + ["segments.budget_campaign_association_status.campaign"], + ["segments.budget_campaign_association_status.status"] ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], "primary_key": [ - ["customer.id"], - ["campaign_budget.id"], - ["segments.date"], - ["segments.budget_campaign_association_status.campaign"], - ["segments.budget_campaign_association_status.status"] - ] + ["customer.id"], + ["campaign_budget.id"], + ["segments.date"], + ["segments.budget_campaign_association_status.campaign"], + ["segments.budget_campaign_association_status.status"] + ] }, { "stream": { @@ -57,13 +67,23 @@ "name": "account_performance_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["customer.id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "source_defined_primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["customer.id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "cursor_field": ["segments.date"] }, { @@ -72,26 +92,44 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["click_view.gclid"], ["segments.date"], ["segments.ad_network_type"]], + "source_defined_primary_key": [ + ["click_view.gclid"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["click_view.gclid"], ["segments.date"], ["segments.ad_network_type"]] + "primary_key": [ + ["click_view.gclid"], + ["segments.date"], + ["segments.ad_network_type"] + ] }, { "stream": { "name": "geographic_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["customer.id"], ["geographic_view.country_criterion_id"], ["geographic_view.location_type"], ["segments.date"]], + "source_defined_primary_key": [ + ["customer.id"], + ["geographic_view.country_criterion_id"], + ["geographic_view.location_type"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["customer.id"], ["geographic_view.country_criterion_id"], ["geographic_view.location_type"], ["segments.date"]], + "primary_key": [ + ["customer.id"], + ["geographic_view.country_criterion_id"], + ["geographic_view.location_type"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { @@ -99,13 +137,21 @@ "name": "keyword_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["ad_group_criterion.criterion_id"], ["ad_group.id"], ["segments.date"]], + "source_defined_primary_key": [ + ["ad_group_criterion.criterion_id"], + ["ad_group.id"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group_criterion.criterion_id"], ["ad_group.id"], ["segments.date"]], + "primary_key": [ + ["ad_group_criterion.criterion_id"], + ["ad_group.id"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { @@ -113,13 +159,25 @@ "name": "display_keyword_performance_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "cursor_field": ["segments.date"] }, { @@ -127,12 +185,24 @@ "name": "display_topics_performance_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", - "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"], ["segments.date"], ["segments.ad_network_type"], ["segments.device"]], + "primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"] }, @@ -154,7 +224,11 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["segments.date"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["segments.date"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", @@ -197,12 +271,22 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["segments.date"], - "source_defined_primary_key": [["campaign.id"], ["segments.date"], ["segments.hour"], ["segments.ad_network_type"]] + "source_defined_primary_key": [ + ["campaign.id"], + ["segments.date"], + ["segments.hour"], + ["segments.ad_network_type"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["campaign.id"], ["segments.date"], ["segments.hour"], ["segments.ad_network_type"]] + "primary_key": [ + ["campaign.id"], + ["segments.date"], + ["segments.hour"], + ["segments.ad_network_type"] + ] }, { "stream": { @@ -231,7 +315,11 @@ "name": "ad_group_ad_labels", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["label.id"]] + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["label.id"] + ] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", @@ -243,12 +331,24 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["customer.id"], ["user_location_view.country_criterion_id"], ["user_location_view.targeting_location"], ["segments.date"], ["segments.ad_network_type"]], + "source_defined_primary_key": [ + ["customer.id"], + ["user_location_view.country_criterion_id"], + ["user_location_view.targeting_location"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["customer.id"], ["user_location_view.country_criterion_id"], ["user_location_view.targeting_location"], ["segments.date"], ["segments.ad_network_type"]], + "primary_key": [ + ["customer.id"], + ["user_location_view.country_criterion_id"], + ["user_location_view.targeting_location"], + ["segments.date"], + ["segments.ad_network_type"] + ], "cursor_field": ["segments.date"] }, { @@ -321,13 +421,21 @@ "name": "campaign_bidding_strategies", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["campaign.id"], ["bidding_strategy.id"], ["segments.date"]], + "source_defined_primary_key": [ + ["campaign.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "primary_key": [["campaign.id"], ["bidding_strategy.id"], ["segments.date"]], + "primary_key": [ + ["campaign.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { @@ -335,13 +443,21 @@ "name": "ad_group_bidding_strategies", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["ad_group.id"], ["bidding_strategy.id"], ["segments.date"]], + "source_defined_primary_key": [ + ["ad_group.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group.id"], ["bidding_strategy.id"], ["segments.date"]], + "primary_key": [ + ["ad_group.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { @@ -371,7 +487,9 @@ "name": "ad_group_criterion_labels", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group_criterion_label.resource_name"]] + "source_defined_primary_key": [ + ["ad_group_criterion_label.resource_name"] + ] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json index 95d5f3756347..ce29ca2dbd2d 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json @@ -40,7 +40,7 @@ } } }, - { + { "type": "STREAM", "stream": { "stream_descriptor": { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json index bf47ea0d9690..5e20a30c4498 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json @@ -263,7 +263,7 @@ } } }, - { + { "type": "STREAM", "stream": { "stream_descriptor": { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json index 842b35eff4f1..8f212a2ae02b 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json @@ -819,7 +819,11 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["yearWeek"], - "source_defined_primary_key": [["property_id"], ["yearWeek"], ["eventName"]] + "source_defined_primary_key": [ + ["property_id"], + ["yearWeek"], + ["eventName"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json index ee9f5e39d8f0..559c52f37852 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json @@ -264,50 +264,24 @@ ] }, { - "name" : "events_report", - "dimensions" : [ - "date", - "eventName" - ], - "metrics" : [ - "eventCount", - "totalUsers", - "eventCountPerUser", - "totalRevenue" - ] + "name": "events_report", + "dimensions": ["date", "eventName"], + "metrics": ["eventCount", "totalUsers", "eventCountPerUser", "totalRevenue"] }, { - "name" : "weekly_events_report", - "dimensions" : [ - "yearWeek", - "eventName" - ], - "metrics" : [ - "eventCount", - "totalUsers", - "eventCountPerUser", - "totalRevenue" - ] + "name": "weekly_events_report", + "dimensions": ["yearWeek", "eventName"], + "metrics": ["eventCount", "totalUsers", "eventCountPerUser", "totalRevenue"] }, { - "name" : "conversions_report", - "dimensions" : [ - "date", - "eventName" - ], - "metrics" : [ - "conversions", - "totalUsers", - "totalRevenue" - ] + "name": "conversions_report", + "dimensions": ["date", "eventName"], + "metrics": ["conversions", "totalUsers", "totalRevenue"] }, { - "name" : "pages_title_and_screen_class_report", - "dimensions" : [ - "date", - "unifiedScreenClass" - ], - "metrics" : [ + "name": "pages_title_and_screen_class_report", + "dimensions": ["date", "unifiedScreenClass"], + "metrics": [ "screenPageViews", "totalUsers", "newUsers", @@ -318,12 +292,9 @@ ] }, { - "name" : "pages_path_report", - "dimensions" : [ - "date", - "pagePath" - ], - "metrics" : [ + "name": "pages_path_report", + "dimensions": ["date", "pagePath"], + "metrics": [ "screenPageViews", "totalUsers", "newUsers", @@ -334,12 +305,9 @@ ] }, { - "name" : "pages_title_and_screen_name_report", - "dimensions" : [ - "date", - "unifiedScreenName" - ], - "metrics" : [ + "name": "pages_title_and_screen_name_report", + "dimensions": ["date", "unifiedScreenName"], + "metrics": [ "screenPageViews", "totalUsers", "newUsers", @@ -350,12 +318,9 @@ ] }, { - "name" : "content_group_report", - "dimensions" : [ - "date", - "contentGroup" - ], - "metrics" : [ + "name": "content_group_report", + "dimensions": ["date", "contentGroup"], + "metrics": [ "screenPageViews", "totalUsers", "newUsers", @@ -366,12 +331,9 @@ ] }, { - "name" : "ecommerce_purchases_item_name_report", - "dimensions" : [ - "date", - "itemName" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_name_report", + "dimensions": ["date", "itemName"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -381,12 +343,9 @@ ] }, { - "name" : "ecommerce_purchases_item_id_report", - "dimensions" : [ - "date", - "itemId" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_id_report", + "dimensions": ["date", "itemId"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -396,8 +355,8 @@ ] }, { - "name" : "ecommerce_purchases_item_category_report_combined", - "dimensions" : [ + "name": "ecommerce_purchases_item_category_report_combined", + "dimensions": [ "date", "itemCategory", "itemCategory2", @@ -405,7 +364,7 @@ "itemCategory4", "itemCategory5" ], - "metrics" : [ + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -415,12 +374,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_report", - "dimensions" : [ - "date", - "itemCategory" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_report", + "dimensions": ["date", "itemCategory"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -430,12 +386,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_2_report", - "dimensions" : [ - "date", - "itemCategory2" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_2_report", + "dimensions": ["date", "itemCategory2"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -445,12 +398,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_3_report", - "dimensions" : [ - "date", - "itemCategory3" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_3_report", + "dimensions": ["date", "itemCategory3"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -460,12 +410,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_4_report", - "dimensions" : [ - "date", - "itemCategory4" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_4_report", + "dimensions": ["date", "itemCategory4"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -475,12 +422,9 @@ ] }, { - "name" : "ecommerce_purchases_item_category_5_report", - "dimensions" : [ - "date", - "itemCategory5" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_category_5_report", + "dimensions": ["date", "itemCategory5"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -490,12 +434,9 @@ ] }, { - "name" : "ecommerce_purchases_item_brand_report", - "dimensions" : [ - "date", - "itemBrand" - ], - "metrics" : [ + "name": "ecommerce_purchases_item_brand_report", + "dimensions": ["date", "itemBrand"], + "metrics": [ "cartToViewRate", "purchaseToViewRate", "itemsPurchased", @@ -505,12 +446,9 @@ ] }, { - "name" : "publisher_ads_ad_unit_report", - "dimensions" : [ - "date", - "adUnitName" - ], - "metrics" : [ + "name": "publisher_ads_ad_unit_report", + "dimensions": ["date", "adUnitName"], + "metrics": [ "publisherAdImpressions", "adUnitExposure", "publisherAdClicks", @@ -518,12 +456,9 @@ ] }, { - "name" : "publisher_ads_page_path_report", - "dimensions" : [ - "date", - "pagePath" - ], - "metrics" : [ + "name": "publisher_ads_page_path_report", + "dimensions": ["date", "pagePath"], + "metrics": [ "publisherAdImpressions", "adUnitExposure", "publisherAdClicks", @@ -531,12 +466,9 @@ ] }, { - "name" : "publisher_ads_ad_format_report", - "dimensions" : [ - "date", - "adFormat" - ], - "metrics" : [ + "name": "publisher_ads_ad_format_report", + "dimensions": ["date", "adFormat"], + "metrics": [ "publisherAdImpressions", "adUnitExposure", "publisherAdClicks", @@ -544,12 +476,9 @@ ] }, { - "name" : "publisher_ads_ad_source_report", - "dimensions" : [ - "date", - "adSourceName" - ], - "metrics" : [ + "name": "publisher_ads_ad_source_report", + "dimensions": ["date", "adSourceName"], + "metrics": [ "publisherAdImpressions", "adUnitExposure", "publisherAdClicks", @@ -557,12 +486,9 @@ ] }, { - "name" : "demographic_country_report", - "dimensions" : [ - "date", - "country" - ], - "metrics" : [ + "name": "demographic_country_report", + "dimensions": ["date", "country"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -572,12 +498,9 @@ ] }, { - "name" : "demographic_region_report", - "dimensions" : [ - "date", - "region" - ], - "metrics" : [ + "name": "demographic_region_report", + "dimensions": ["date", "region"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -587,12 +510,9 @@ ] }, { - "name" : "demographic_city_report", - "dimensions" : [ - "date", - "city" - ], - "metrics" : [ + "name": "demographic_city_report", + "dimensions": ["date", "city"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -602,12 +522,9 @@ ] }, { - "name" : "demographic_language_report", - "dimensions" : [ - "date", - "language" - ], - "metrics" : [ + "name": "demographic_language_report", + "dimensions": ["date", "language"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -617,12 +534,9 @@ ] }, { - "name" : "demographic_age_report", - "dimensions" : [ - "date", - "userAgeBracket" - ], - "metrics" : [ + "name": "demographic_age_report", + "dimensions": ["date", "userAgeBracket"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -632,12 +546,9 @@ ] }, { - "name" : "demographic_gender_report", - "dimensions" : [ - "date", - "userGender" - ], - "metrics" : [ + "name": "demographic_gender_report", + "dimensions": ["date", "userGender"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -647,12 +558,9 @@ ] }, { - "name" : "demographic_interest_report", - "dimensions" : [ - "date", - "brandingInterest" - ], - "metrics" : [ + "name": "demographic_interest_report", + "dimensions": ["date", "brandingInterest"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -662,12 +570,9 @@ ] }, { - "name" : "tech_browser_report", - "dimensions" : [ - "date", - "browser" - ], - "metrics" : [ + "name": "tech_browser_report", + "dimensions": ["date", "browser"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -678,12 +583,9 @@ ] }, { - "name" : "tech_device_category_report", - "dimensions" : [ - "date", - "deviceCategory" - ], - "metrics" : [ + "name": "tech_device_category_report", + "dimensions": ["date", "deviceCategory"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -694,12 +596,9 @@ ] }, { - "name" : "tech_device_model_report", - "dimensions" : [ - "date", - "deviceModel" - ], - "metrics" : [ + "name": "tech_device_model_report", + "dimensions": ["date", "deviceModel"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -710,12 +609,9 @@ ] }, { - "name" : "tech_screen_resolution_report", - "dimensions" : [ - "date", - "screenResolution" - ], - "metrics" : [ + "name": "tech_screen_resolution_report", + "dimensions": ["date", "screenResolution"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -726,12 +622,9 @@ ] }, { - "name" : "tech_app_version_report", - "dimensions" : [ - "date", - "appVersion" - ], - "metrics" : [ + "name": "tech_app_version_report", + "dimensions": ["date", "appVersion"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -742,12 +635,9 @@ ] }, { - "name" : "tech_platform_report", - "dimensions" : [ - "date", - "platform" - ], - "metrics" : [ + "name": "tech_platform_report", + "dimensions": ["date", "platform"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -758,13 +648,9 @@ ] }, { - "name" : "tech_platform_device_category_report", - "dimensions" : [ - "date", - "platform", - "deviceCategory" - ], - "metrics" : [ + "name": "tech_platform_device_category_report", + "dimensions": ["date", "platform", "deviceCategory"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -775,12 +661,9 @@ ] }, { - "name" : "tech_operating_system_report", - "dimensions" : [ - "date", - "operatingSystem" - ], - "metrics" : [ + "name": "tech_operating_system_report", + "dimensions": ["date", "operatingSystem"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", @@ -791,12 +674,9 @@ ] }, { - "name" : "tech_os_with_version_report", - "dimensions" : [ - "date", - "operatingSystemWithVersion" - ], - "metrics" : [ + "name": "tech_os_with_version_report", + "dimensions": ["date", "operatingSystemWithVersion"], + "metrics": [ "totalUsers", "newUsers", "engagedSessions", diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json index 487f0f84b0a9..0d1074e212d0 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json @@ -215,7 +215,7 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, - { + { "stream": { "name": "issue_transitions", "json_schema": {}, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json index 44c717228f56..e3c19d2e8876 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json @@ -8,8 +8,8 @@ "readOnly": true }, "filterId": { - "type" : ["null", "string"], - "description" : "Id of the related filter" + "type": ["null", "string"], + "description": "Id of the related filter" }, "type": { "type": "string", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json index a9202899f541..538bf16bfac6 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json @@ -13,7 +13,7 @@ "readOnly": true }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue.", "readOnly": true }, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json index 6fd0b569f954..1bada7505928 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json @@ -7,8 +7,8 @@ "description": "The ID of the context." }, "fieldId": { - "type" : ["null", "string"], - "description" : "Id of the related field" + "type": ["null", "string"], + "description": "Id of the related field" }, "name": { "type": "string", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json index 6f4b18e666ad..faaa84ba2ec9 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json @@ -7,7 +7,7 @@ "description": "The key of the property. Required on create and update." }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue.", "readOnly": true }, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json index b058f9341495..102a5fba5563 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json @@ -7,7 +7,7 @@ "description": "The ID of the link." }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue." }, "self": { diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json index 903925b030bc..913638b8b4f7 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json @@ -8,7 +8,7 @@ "readOnly": true }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue.", "readOnly": true }, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json index 19c489177737..069c52a189d3 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json @@ -8,7 +8,7 @@ "readOnly": true }, "issueId": { - "type" : ["null", "string"], + "type": ["null", "string"], "description": "Id of the related issue." }, "isWatching": { diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json index 2e8b46c62da2..afb993abe8e3 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json @@ -7,8 +7,8 @@ "description": "The ID of the avatar." }, "projectId": { - "type" : ["null", "string"], - "description" : "Id of the related project." + "type": ["null", "string"], + "description": "Id of the related project." }, "owner": { "type": "string", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json index 2452d5435b61..ff7c28191eec 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json @@ -7,8 +7,8 @@ "description": "The URL of the issue level security item." }, "projectId": { - "type" : ["null", "string"], - "description" : "Id of the related project." + "type": ["null", "string"], + "description": "Id of the related project." }, "id": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json index 74466e57701d..0180e3dd6d18 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json @@ -12,12 +12,12 @@ "description": "The name of the screen tab field. Required on create and update. The maximum length is 255 characters." }, "screenId": { - "type" : ["null", "integer"], - "description" : "Id of the related screen." + "type": ["null", "integer"], + "description": "Id of the related screen." }, "tabId": { - "type" : ["null", "integer"], - "description" : "Id of the related tab." + "type": ["null", "integer"], + "description": "Id of the related tab." } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json index 1806c4b2ff40..013814e7057d 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json @@ -13,8 +13,8 @@ "description": "The name of the screen tab. The maximum length is 255 characters." }, "screenId": { - "type" : ["null", "integer"], - "description" : "Id of the related screen." + "type": ["null", "integer"], + "description": "Id of the related screen." } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py index 9f2cf2ec5d48..16fdfb000a4c 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import logging from datetime import datetime from typing import Any, Dict diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py index 0bee338a005c..38d5f334e776 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import pytest from source_jira.source import SourceJira from source_jira.streams import ApplicationRoles diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json b/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json index 52d5f046634e..8745a889bf96 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json @@ -1 +1,10 @@ -{"api_token": "invalid_token", "domain": "invaliddomain.atlassian.net", "email": "no-reply@domain.com", "start_date": "2023-01-01T00:00:00Z", "projects": ["IT1", "IT1", "IT1"], "expand_issue_changelog": true, "render_fields": true, "expand_issue_transition": false} \ No newline at end of file +{ + "api_token": "invalid_token", + "domain": "invaliddomain.atlassian.net", + "email": "no-reply@domain.com", + "start_date": "2023-01-01T00:00:00Z", + "projects": ["IT1", "IT1", "IT1"], + "expand_issue_changelog": true, + "render_fields": true, + "expand_issue_transition": false +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java index 6056c76778a6..c469c24f3faa 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java @@ -9,9 +9,10 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; +import io.airbyte.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; import io.airbyte.integrations.source.relationaldb.models.CdcState; import java.util.Optional; -import io.airbyte.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; + public class MssqlCdcSavedInfoFetcher implements CdcSavedInfoFetcher { private final JsonNode savedOffset; diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlPkStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlPkStrictEncryptJdbcSourceAcceptanceTest.java index 6079164515d3..b32eb8f452ed 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlPkStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlPkStrictEncryptJdbcSourceAcceptanceTest.java @@ -344,10 +344,10 @@ void testReadMultipleTablesIncrementally() throws Exception { database.query(connection -> { connection.fetch(String.format("USE %s;", getDefaultNamespace())); connection.fetch(String.format("CREATE TABLE %s (\n" - + " id int PRIMARY KEY,\n" - + " name VARCHAR(200) NOT NULL,\n" - + " updated_at VARCHAR(200) NOT NULL\n" - + ");", streamOneName)); + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at VARCHAR(200) NOT NULL\n" + + ");", streamOneName)); connection.execute( String.format( "INSERT INTO %s(id, name, updated_at) VALUES (1,'picard', '2004-10-19')", @@ -369,22 +369,22 @@ void testReadMultipleTablesIncrementally() throws Exception { // Insert records into second table database.query(ctx -> { ctx.fetch(String.format("CREATE TABLE %s (\n" - + " id int PRIMARY KEY,\n" - + " name VARCHAR(200) NOT NULL,\n" - + " updated_at DATE NOT NULL\n" - + ");", streamTwoName)); + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at DATE NOT NULL\n" + + ");", streamTwoName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (40,'Jean Luc','2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (40,'Jean Luc','2006-10-19')", + streamTwoFullyQualifiedName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (41, 'Groot', '2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (41, 'Groot', '2006-10-19')", + streamTwoFullyQualifiedName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (42, 'Thanos','2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (42, 'Thanos','2006-10-19')", + streamTwoFullyQualifiedName)); return null; }); // Create records list that we expect to see in the state message @@ -425,7 +425,7 @@ void testReadMultipleTablesIncrementally() throws Exception { // All records in the 2 configured streams should be present assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf( Stream.concat(getTestMessages(streamOneName).stream().parallel(), - streamTwoExpectedRecords.stream().parallel()).collect(toList())); + streamTwoExpectedRecords.stream().parallel()).collect(toList())); final List actualFirstSyncState = extractStateMessage(messagesFromFirstSync); // Since we are emitting a state message after each record, we should have 1 state for each record - @@ -463,12 +463,12 @@ void testReadMultipleTablesIncrementally() throws Exception { // Extract the incremental states of each stream's first and second state message final List streamOneIncrementalStatesFromFirstSync = List.of(streamOneStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), - streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); final JsonNode streamOneFinalStreamStateFromFirstSync = streamOneStateMessagesFromFirstSync.get(2).getStream().getStreamState(); final List streamTwoIncrementalStatesFromFirstSync = List.of(streamTwoStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), - streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); final JsonNode streamTwoFinalStreamStateFromFirstSync = streamTwoStateMessagesFromFirstSync.get(2).getStream().getStreamState(); // The incremental_state of each stream's first and second incremental states is expected @@ -485,8 +485,8 @@ void testReadMultipleTablesIncrementally() throws Exception { // switch final List messagesFromSecondSyncWithMixedStates = MoreIterators .toList(source.read(config, configuredCatalog, - Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), - streamTwoStateMessagesFromFirstSync.get(1))))); + Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), + streamTwoStateMessagesFromFirstSync.get(1))))); // Extract only state messages for each stream after second sync final List streamOneStateMessagesFromSecondSync = @@ -515,19 +515,19 @@ void testReadMultipleTablesIncrementally() throws Exception { database.query(ctx -> { ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (4,'Hooper','2006-10-19')", - getFullyQualifiedTableName(streamOneName))); + + "VALUES (4,'Hooper','2006-10-19')", + getFullyQualifiedTableName(streamOneName))); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (43, 'Iron Man', '2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (43, 'Iron Man', '2006-10-19')", + streamTwoFullyQualifiedName)); return null; }); final List messagesFromThirdSync = MoreIterators .toList(source.read(config, configuredCatalog, - Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), - streamTwoStateMessagesFromSecondSync.get(0))))); + Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), + streamTwoStateMessagesFromSecondSync.get(0))))); // Extract only state messages, state type, and cursor for each stream after second sync final List streamOneStateMessagesFromThirdSync = @@ -649,32 +649,32 @@ protected List getTestMessages(final String streamName) { return List.of( new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_1, - COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19")))), + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_2, - COL_NAME, "crusher", - COL_UPDATED_AT, - "2005-10-19")))), + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_3, - COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19"))))); + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19"))))); } private AirbyteStream getAirbyteStream(final String tableName, final String namespace) { return CatalogHelpers.createAirbyteStream( - tableName, - namespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + tableName, + namespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))); } diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index ac542d79abc7..0be68fc641b1 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -188,7 +188,6 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); } - @Test void testStrictSSLUnsecuredNoTunnel() throws Exception { final String PASSWORD = "Passw0rd"; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java index 902ce156a35e..147c3281a686 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java @@ -69,10 +69,11 @@ private static Properties commonProperties(final JdbcDatabase database) { if (sourceConfig.get("replication_method").has("server_time_zone")) { final String serverTimeZone = sourceConfig.get("replication_method").get("server_time_zone").asText(); if (!serverTimeZone.isEmpty()) { - /** Per Debezium docs, https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-temporal-types - * this property is now connectionTimeZone - * {@link com.mysql.cj.conf.PropertyKey#connectionTimeZone} - **/ + /** + * Per Debezium docs, + * https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-temporal-types + * this property is now connectionTimeZone {@link com.mysql.cj.conf.PropertyKey#connectionTimeZone} + **/ props.setProperty("database.connectionTimeZone", serverTimeZone); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java index 4fe64b18ee0e..a25149da6ab1 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java @@ -66,4 +66,5 @@ public AirbyteMessage saveStateAfterCompletionOfSnapshotOfNewStreams() { public boolean compressSchemaHistoryForState() { return COMPRESSION_ENABLED; } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java index d6598bd5677e..3727ddcafad0 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java @@ -146,7 +146,6 @@ public static Map getTableSizeInf return tableSizeInfoMap; } - /** * Iterates through each stream and find the max cursor value and the record count which has that * value based on each cursor field provided by the customer per stream This information is saved in diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java index 4c53b8845229..c9d656ee3aae 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.source.mysql.cursor_based; import com.google.common.collect.Lists; @@ -78,4 +82,5 @@ private CursorBasedStatus generateDbStreamState(final AirbyteStreamNameNamespace } return state; } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java index a2428ad84edd..9bdfa3727efc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java @@ -218,10 +218,10 @@ private ConfiguredAirbyteCatalog getConfiguredCatalogWithPartialColumns() { .withSyncMode(INCREMENTAL) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - Field.of("id", JsonSchemaType.NUMBER) - /* no name field */) + String.format("%s", STREAM_NAME), + String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), + Field.of("id", JsonSchemaType.NUMBER) + /* no name field */) .withSourceDefinedCursor(true) .withSourceDefinedPrimaryKey(List.of(List.of("id"))) .withSupportedSyncModes( @@ -230,10 +230,10 @@ private ConfiguredAirbyteCatalog getConfiguredCatalogWithPartialColumns() { .withSyncMode(INCREMENTAL) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME2), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - /* no name field */ - Field.of("id", JsonSchemaType.NUMBER)) + String.format("%s", STREAM_NAME2), + String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), + /* no name field */ + Field.of("id", JsonSchemaType.NUMBER)) .withSourceDefinedCursor(true) .withSourceDefinedPrimaryKey(List.of(List.of("id"))) .withSupportedSyncModes( diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java index 81e251fce590..309cfd258e57 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java @@ -55,4 +55,5 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc grantCorrectPermissions(); createAndPopulateTables(); } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java index 011769bc6279..5ae7ebf22fbe 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java @@ -52,4 +52,5 @@ protected void setupEnvironment(final TestDestinationEnv environment) { private void alterUserRequireSsl() { executeQuery("ALTER USER " + container.getUsername() + " REQUIRE SSL;"); } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java index 21554cffa3f5..9a214a0da6ea 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java @@ -66,8 +66,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Random; import java.util.Properties; +import java.util.Random; import java.util.Set; import java.util.stream.Collectors; import javax.sql.DataSource; @@ -705,9 +705,11 @@ public void testTwoStreamSync() throws Exception { } /** - * This test creates lots of tables increasing the schema history size above the limit of {@link AirbyteSchemaHistoryStorage#SIZE_LIMIT_TO_COMPRESS_MB} - * forcing the {@link AirbyteSchemaHistoryStorage#read()} method to compress the schema history blob as part of the state message - * which allows us to test that the next sync is able to work fine when provided with a compressed blob in the state. + * This test creates lots of tables increasing the schema history size above the limit of + * {@link AirbyteSchemaHistoryStorage#SIZE_LIMIT_TO_COMPRESS_MB} forcing the + * {@link AirbyteSchemaHistoryStorage#read()} method to compress the schema history blob as part of + * the state message which allows us to test that the next sync is able to work fine when provided + * with a compressed blob in the state. */ @Test public void testCompressedSchemaHistory() throws Exception { @@ -785,4 +787,5 @@ private static String generateRandomStringOf32Characters() { return randomString.toString(); } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MysqlPkJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MysqlPkJdbcSourceAcceptanceTest.java index 8f96c99ff379..b7eed3cba392 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MysqlPkJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MysqlPkJdbcSourceAcceptanceTest.java @@ -90,7 +90,6 @@ class MySqlPkJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { protected Database database; protected DSLContext dslContext; - @BeforeAll static void init() throws Exception { container = new MySQLContainer<>("mysql:8.0") @@ -107,21 +106,21 @@ static void init() throws Exception { public void setup() throws Exception { environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) - .put(JdbcUtils.USERNAME_KEY, TEST_USER) - .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) - .put("standard_via_pk", true) - .build()); + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) + .put(JdbcUtils.USERNAME_KEY, TEST_USER) + .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) + .put("standard_via_pk", true) + .build()); dslContext = DSLContextFactory.create( config.get(JdbcUtils.USERNAME_KEY).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText()), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asText()), SQLDialect.MYSQL); database = new Database(dslContext); @@ -174,10 +173,10 @@ void testReadMultipleTablesIncrementally() throws Exception { database.query(connection -> { connection.fetch(String.format("USE %s;", getDefaultNamespace())); connection.fetch(String.format("CREATE TABLE %s (\n" - + " id int PRIMARY KEY,\n" - + " name VARCHAR(200) NOT NULL,\n" - + " updated_at VARCHAR(200) NOT NULL\n" - + ");", streamOneName)); + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at VARCHAR(200) NOT NULL\n" + + ");", streamOneName)); connection.execute( String.format( "INSERT INTO %s(id, name, updated_at) VALUES (1,'picard', '2004-10-19')", @@ -199,22 +198,22 @@ void testReadMultipleTablesIncrementally() throws Exception { // Insert records into second table database.query(ctx -> { ctx.fetch(String.format("CREATE TABLE %s (\n" - + " id int PRIMARY KEY,\n" - + " name VARCHAR(200) NOT NULL,\n" - + " updated_at DATE NOT NULL\n" - + ");", streamTwoName)); + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at DATE NOT NULL\n" + + ");", streamTwoName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (40,'Jean Luc','2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (40,'Jean Luc','2006-10-19')", + streamTwoFullyQualifiedName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (41, 'Groot', '2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (41, 'Groot', '2006-10-19')", + streamTwoFullyQualifiedName)); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (42, 'Thanos','2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (42, 'Thanos','2006-10-19')", + streamTwoFullyQualifiedName)); return null; }); // Create records list that we expect to see in the state message @@ -255,7 +254,7 @@ void testReadMultipleTablesIncrementally() throws Exception { // All records in the 2 configured streams should be present assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf( Stream.concat(getTestMessages(streamOneName).stream().parallel(), - streamTwoExpectedRecords.stream().parallel()).collect(toList())); + streamTwoExpectedRecords.stream().parallel()).collect(toList())); final List actualFirstSyncState = extractStateMessage(messagesFromFirstSync); // Since we are emitting a state message after each record, we should have 1 state for each record - @@ -293,12 +292,12 @@ void testReadMultipleTablesIncrementally() throws Exception { // Extract the incremental states of each stream's first and second state message final List streamOneIncrementalStatesFromFirstSync = List.of(streamOneStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), - streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); final JsonNode streamOneFinalStreamStateFromFirstSync = streamOneStateMessagesFromFirstSync.get(2).getStream().getStreamState(); final List streamTwoIncrementalStatesFromFirstSync = List.of(streamTwoStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), - streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); final JsonNode streamTwoFinalStreamStateFromFirstSync = streamTwoStateMessagesFromFirstSync.get(2).getStream().getStreamState(); // The incremental_state of each stream's first and second incremental states is expected @@ -311,11 +310,12 @@ void testReadMultipleTablesIncrementally() throws Exception { // Sync should work with a primaryKey state AND a cursor-based state from each stream // Forcing a sync with // - stream one state still being the first record read via Primary Key. - // - stream two state being the Primary Key state before the final emitted state before the cursor switch + // - stream two state being the Primary Key state before the final emitted state before the cursor + // switch final List messagesFromSecondSyncWithMixedStates = MoreIterators .toList(source.read(config, configuredCatalog, - Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), - streamTwoStateMessagesFromFirstSync.get(1))))); + Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), + streamTwoStateMessagesFromFirstSync.get(1))))); // Extract only state messages for each stream after second sync final List streamOneStateMessagesFromSecondSync = @@ -344,19 +344,19 @@ void testReadMultipleTablesIncrementally() throws Exception { database.query(ctx -> { ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (4,'Hooper','2006-10-19')", - getFullyQualifiedTableName(streamOneName))); + + "VALUES (4,'Hooper','2006-10-19')", + getFullyQualifiedTableName(streamOneName))); ctx.execute( String.format("INSERT INTO %s(id, name, updated_at)" - + "VALUES (43, 'Iron Man', '2006-10-19')", - streamTwoFullyQualifiedName)); + + "VALUES (43, 'Iron Man', '2006-10-19')", + streamTwoFullyQualifiedName)); return null; }); final List messagesFromThirdSync = MoreIterators .toList(source.read(config, configuredCatalog, - Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), - streamTwoStateMessagesFromSecondSync.get(0))))); + Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), + streamTwoStateMessagesFromSecondSync.get(0))))); // Extract only state messages, state type, and cursor for each stream after second sync final List streamOneStateMessagesFromThirdSync = @@ -471,17 +471,17 @@ protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configure protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(ImmutableMap - .of(COL_ID, ID_VALUE_4, - COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19"))))); + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19"))))); expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(ImmutableMap - .of(COL_ID, ID_VALUE_5, - COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19"))))); + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19"))))); final DbStreamState state = new CursorBasedStatus() .withStateType(StateType.CURSOR_BASED) .withVersion(2L) @@ -509,34 +509,32 @@ protected List getTestMessages(final String streamName) { return List.of( new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_1, - COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19")))), + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_2, - COL_NAME, "crusher", - COL_UPDATED_AT, - "2005-10-19")))), + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_3, - COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19"))))); + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19"))))); } - - private AirbyteStream getAirbyteStream(final String tableName, final String namespace) { return CatalogHelpers.createAirbyteStream( - tableName, - namespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + tableName, + namespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))); } @@ -545,27 +543,27 @@ private AirbyteStream getAirbyteStream(final String tableName, final String name protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(Lists.newArrayList( CatalogHelpers.createAirbyteStream( - TABLE_NAME, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(Collections.emptyList()), CatalogHelpers.createAirbyteStream( - TABLE_NAME_COMPOSITE_PK, - defaultNamespace, - Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), - Field.of(COL_LAST_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + TABLE_NAME_COMPOSITE_PK, + defaultNamespace, + Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), + Field.of(COL_LAST_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey( List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); @@ -575,29 +573,29 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { @Override protected List createExpectedTestMessages(final List states) { return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteMessage().withType(Type.STATE) - .withState( - new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s))))) - .collect( - Collectors.toList()) - : List.of(new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY))); + ? states.stream() + .map(s -> new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s))))) + .collect( + Collectors.toList()) + : List.of(new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY))); } @Override protected List createState(final List states) { return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s)))) - .collect( - Collectors.toList()) - : List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY)); + ? states.stream() + .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s)))) + .collect( + Collectors.toList()) + : List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY)); } @Override @@ -609,5 +607,5 @@ protected JsonNode getStateData(final AirbyteMessage airbyteMessage, final Strin throw new IllegalArgumentException("Stream not found in state message: " + streamName); } -} +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index ee3c4b66de8d..dc44c707ad24 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -21,4 +21,4 @@ } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml index c1f09efbb0c4..1ae2572cc8c5 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml @@ -16,7 +16,7 @@ definitions: requester: type: HttpRequester - url_base: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/reporting/' + url_base: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/reporting/' path: "{{ parameters.path }}" http_method: GET request_headers: @@ -24,11 +24,11 @@ definitions: authenticator: type: CustomAuthenticator class_name: source_paypal_transaction.components.PayPalOauth2Authenticator - client_id: '{{ config[''client_id''] }}' - client_secret: '{{ config[''client_secret''] }}' + client_id: "{{ config['client_id'] }}" + client_secret: "{{ config['client_secret'] }}" refresh_request_body: Content-Type: application/x-www-form-urlencoded - token_refresh_endpoint: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/oauth2/token' + token_refresh_endpoint: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/oauth2/token' grant_type: client_credentials expires_in_name: expires_in access_token_name: access_token @@ -78,18 +78,18 @@ definitions: fields: - path: - transaction_id - value: '{{ record[''transaction_info''][''transaction_id''] }}' + value: "{{ record['transaction_info']['transaction_id'] }}" incremental_sync: type: DatetimeBasedCursor cursor_field: transaction_updated_date cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%SZ' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime datetime: >- {{ max( format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: start_date @@ -100,8 +100,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" step: P15D cursor_granularity: PT1S $parameters: @@ -132,18 +132,18 @@ definitions: fields: - path: - as_of_time - value: '{{ format_datetime(record[''as_of_time''], ''%Y-%m-%dT%H:%M:%SZ'') }}' + value: "{{ format_datetime(record['as_of_time'], '%Y-%m-%dT%H:%M:%SZ') }}" incremental_sync: type: DatetimeBasedCursor cursor_field: as_of_time cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%SZ' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime datetime: >- {{ max( format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: as_of_time @@ -201,4 +201,4 @@ spec: type: "string" title: "Refresh token" description: "The key to refresh the expired access token." - airbyte_secret: true \ No newline at end of file + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json index afa3150857dd..58031b32fb5e 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json @@ -4,7 +4,7 @@ "stream": { "name": "deals", "json_schema": {}, - "supported_sync_modes": ["full_refresh","incremental"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["update_time"] }, diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml index 42240785560b..319bcde916bc 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml @@ -12,12 +12,12 @@ definitions: extractor: type: DpathExtractor field_path: ["data", "*", "data"] - + selector_users: type: RecordSelector extractor: type: DpathExtractor - field_path: ["data", "*", "data","*"] + field_path: ["data", "*", "data", "*"] requester: type: HttpRequester @@ -107,7 +107,7 @@ definitions: path: "/recents" lead_labels_stream: - # Does not support pagination + # Does not support pagination $ref: "#/definitions/base_stream" name: "lead_labels" retriever: @@ -148,7 +148,7 @@ definitions: path: "/recents" activity_types_stream: -# This stream didnt have limit as query_parameter + # This stream didnt have limit as query_parameter $ref: "#/definitions/base_stream" name: "activity_types" retriever: @@ -302,7 +302,6 @@ definitions: parent_key: "id" partition_field: "parent_id" - streams: - "#/definitions/deals_stream" - "#/definitions/deal_fields_stream" @@ -351,9 +350,10 @@ spec: airbyte_secret: true replication_start_date: title: Start Date - description: UTC date and time in the format 2017-01-25T00:00:00Z. Any data + description: + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. When specified and not None, then stream will behave as incremental examples: - - '2017-01-25 00:00:00Z' + - "2017-01-25 00:00:00Z" type: string diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json index 997124c2b50f..2901d9c336f4 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json @@ -57,7 +57,7 @@ "type": ["null", "boolean"] }, "json_column_flag": { - "type": ["null", "boolean"] + "type": ["null", "boolean"] }, "parent_id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json index 4b06133de601..805265c21864 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json @@ -102,7 +102,7 @@ } }, "mandatory_flag": { - "type": ["null", "object","boolean","integer"] + "type": ["null", "object", "boolean", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json index f24cb9d06a69..dff11aa1ddba 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json @@ -59,19 +59,19 @@ "type": ["null", "boolean"] }, "parent_id": { - "type": ["null", "integer"] + "type": ["null", "integer"] }, "id_suffix": { - "type": ["null", "string"] + "type": ["null", "string"] }, "is_subfield": { - "type": ["null", "boolean"] + "type": ["null", "boolean"] }, "json_column_flag": { - "type": ["null", "boolean"] + "type": ["null", "boolean"] }, "group_id": { - "type": ["null", "integer"] + "type": ["null", "integer"] }, "options": { "type": ["null", "array"], diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json index 858901f328f1..62d35b1099b0 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json @@ -198,10 +198,10 @@ "type": ["null", "string"] }, "category_id": { - "type": ["null", "integer"] + "type": ["null", "integer"] }, "delete_time": { - "type": ["null", "string"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java index a524860752b5..706dca1a885e 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java @@ -321,8 +321,8 @@ public static List filterStreamsUnderVacuumForCtidSync( } public static Map getTableMaxTupleForStreams(final JdbcDatabase database, - final List streams, - final String quoteString) { + final List streams, + final String quoteString) { final Map tableMaxTupleEstimates = new HashMap<>(); streams.forEach(stream -> { final AirbyteStreamNameNamespacePair namespacePair = @@ -334,8 +334,8 @@ public static Map getTableMaxTupleForSt } public static int getTableMaxTupleForStream(final JdbcDatabase database, - final AirbyteStreamNameNamespacePair stream, - final String quoteString) { + final AirbyteStreamNameNamespacePair stream, + final String quoteString) { try { final String streamName = stream.getName(); final String schemaName = stream.getNamespace(); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 56c6a20656d8..7c365a7373ac 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -494,8 +494,8 @@ public List> getIncrementalIterators(final getQuoteString()); final Map tablesMaxTuple = - CtidUtils.isTidRangeScanCapableDBServer(database) ? null : - PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); if (!streamsCategorised.ctidStreams().streamsForCtidSync().isEmpty()) { LOGGER.info("Streams to be synced via ctid : {}", finalListOfStreamsToBeSyncedViaCtid.size()); LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(finalListOfStreamsToBeSyncedViaCtid)); @@ -563,8 +563,8 @@ public List> getIncrementalIterators(final getQuoteString()); final Map tablesMaxTuple = - CtidUtils.isTidRangeScanCapableDBServer(database) ? null : - PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); if (finalListOfStreamsToBeSyncedViaCtid.isEmpty()) { LOGGER.info("No Streams will be synced via ctid."); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java index a153d08e8dc4..17594f414711 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java @@ -143,8 +143,8 @@ public static List> cdcCtidIteratorsCombin quoteString); final Map tablesMaxTuple = - CtidUtils.isTidRangeScanCapableDBServer(database) ? null : - PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, quoteString); + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, quoteString); final PostgresCtidHandler ctidHandler = new PostgresCtidHandler(sourceConfig, database, ctidPostgresSourceOperations, diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java index 89dc92149a96..25f27d817948 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java @@ -6,8 +6,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; -import io.airbyte.integrations.source.relationaldb.models.CdcState; import io.airbyte.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.integrations.source.relationaldb.models.CdcState; import java.util.Optional; public class PostgresCdcSavedInfoFetcher implements CdcSavedInfoFetcher { diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java index e3284c0ae082..bd6b4ae9bb4a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java @@ -75,4 +75,5 @@ public int hashCode() { public static Ctid inc(final Ctid ctid, final long maxTuple) { return (ctid.tuple + 1 > maxTuple) ? Ctid.of(ctid.page + 1, 1) : Ctid.of(ctid.page, ctid.tuple + 1); } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java index 0f26c218817e..68228b06ae90 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java @@ -12,7 +12,6 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.SQLException; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -61,8 +60,9 @@ public record StreamsCategorised (CtidStreams ctidStreams, } /** - * Postgres servers version 14 and above are capable of running a tid range scan. - * Used by ctid queries + * Postgres servers version 14 and above are capable of running a tid range scan. Used by ctid + * queries + * * @param database database * @return true for Tid scan capable server */ diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java index 1aa7d3039d29..ef490c34cfad 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java @@ -66,6 +66,7 @@ public class InitialSyncCtidIterator extends AbstractIterator i private int numberOfTimesReSynced = 0; private boolean subQueriesInitialized = false; private final boolean tidRangeScanCapableDBServer; + public InitialSyncCtidIterator(final CtidStateManager ctidStateManager, final JdbcDatabase database, final CtidPostgresSourceOperations sourceOperations, diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java index b3a912a52a6a..0441cd9cdb20 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java @@ -78,6 +78,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc } protected abstract String getServerImageName(); + public abstract ImmutableMap getCertificateConfiguration(); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java index a5a3c6146d69..44299c3e8881 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java @@ -1,8 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.io.airbyte.integration_tests.sources; -public class CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest extends CDCPostgresSourceCaCertificateSslAcceptanceTest{ +public class CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest extends CDCPostgresSourceCaCertificateSslAcceptanceTest { + @Override protected String getServerImageName() { return "postgres:12-bullseye"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java index 0936751640fd..1e052877768c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java @@ -21,5 +21,4 @@ protected String getServerImageName() { return "postgres:15-bullseye"; } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java index cd083c1ba883..20b264e3a0cf 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java @@ -23,4 +23,5 @@ public ImmutableMap getCertificateConfiguration() { protected String getServerImageName() { return "postgres:15-bullseye"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java index 42b8312bb28c..7671d7d29d01 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java @@ -1,8 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.io.airbyte.integration_tests.sources; -public class CdcPostgresSourceAcceptanceLegacyCtidTest extends CdcPostgresSourceAcceptanceTest{ +public class CdcPostgresSourceAcceptanceLegacyCtidTest extends CdcPostgresSourceAcceptanceTest { + @Override protected String getServerImageName() { return "postgres:13-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index f96cb2c49cee..8583922b36f5 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -225,4 +225,5 @@ private void verifyFieldNotExist(final List records, final protected String getServerImageName() { return "postgres:15-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java index 58bb28ab99cb..f0fce358a3c0 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java @@ -1,9 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.io.airbyte.integration_tests.sources; -public class PostgresSourceAcceptanceLegacyCtidTest extends PostgresSourceAcceptanceTest{ +public class PostgresSourceAcceptanceLegacyCtidTest extends PostgresSourceAcceptanceTest { @Override protected String getServerImageName() { return "postgres:13-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java index 6c284994bdec..2461331a5ca1 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.source.postgres; public class CdcPostgresSourceLegacyCtidTest extends CdcPostgresSourceTest { @@ -6,4 +10,5 @@ public class CdcPostgresSourceLegacyCtidTest extends CdcPostgresSourceTest { protected String getServerImageName() { return "debezium/postgres:13-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index fefdb69b29f9..a036cde99d2f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -993,11 +993,16 @@ protected void ctidIteratorPageSizeTest() throws Exception { } @Override - protected void compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBeforeSync(final CdcTargetPosition targetPosition, final AirbyteRecordMessage record) { - // The LSN from records should be either equal or grater than the position value before the sync started. - // The current Write-Ahead Log (WAL) position can move ahead even without any data modifications (INSERT, UPDATE, DELETE) - // The start and end of transactions, even read-only ones, are recorded in the WAL. So, simply starting and committing a transaction can cause the WAL location to move forward. - // Periodic checkpoints, which write dirty pages from memory to disk to ensure database consistency, generate WAL records. Checkpoints happen even if there are no active data modifications + protected void compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBeforeSync(final CdcTargetPosition targetPosition, + final AirbyteRecordMessage record) { + // The LSN from records should be either equal or grater than the position value before the sync + // started. + // The current Write-Ahead Log (WAL) position can move ahead even without any data modifications + // (INSERT, UPDATE, DELETE) + // The start and end of transactions, even read-only ones, are recorded in the WAL. So, simply + // starting and committing a transaction can cause the WAL location to move forward. + // Periodic checkpoints, which write dirty pages from memory to disk to ensure database consistency, + // generate WAL records. Checkpoints happen even if there are no active data modifications assert targetPosition instanceof PostgresCdcTargetPosition; assertTrue(extractPosition(record.getData()).targetLsn.compareTo(((PostgresCdcTargetPosition) targetPosition).targetLsn) >= 0); } @@ -1005,4 +1010,5 @@ protected void compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBefo protected String getServerImageName() { return "debezium/postgres:15-alpine"; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java index a6f897cf4dd7..03b23fdf4b03 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java @@ -50,8 +50,8 @@ void testCtidQueryBounds() { Pair.of(Ctid.ZERO, null)); assertEquals(expected, chunks); - chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.of(8,1), 819200L, 81920L, 50, EIGHT_KB); - expected = List.of(Pair.of(Ctid.of(8,1), Ctid.of(9,0)), Pair.of(Ctid.of(9,0), Ctid.of(10,0))); + chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.of(8, 1), 819200L, 81920L, 50, EIGHT_KB); + expected = List.of(Pair.of(Ctid.of(8, 1), Ctid.of(9, 0)), Pair.of(Ctid.of(9, 0), Ctid.of(10, 0))); assertEquals(expected, chunks); } @@ -59,58 +59,58 @@ void testCtidQueryBounds() { void testLegacyCtidQueryBounds() { var chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(0, 184), 3805450321L, 8192L, 50, GIGABYTE, 185); var expected = List.of( - Pair.of(Ctid.of (0,185), Ctid.of(27027,185)), - Pair.of(Ctid.of(27028,1), Ctid.of(54054,185)), - Pair.of(Ctid.of(54055,1), Ctid.of(81081,185)), - Pair.of(Ctid.of(81082,1), Ctid.of(108108,185)), - Pair.of(Ctid.of(108109,1), Ctid.of(135135,185)), - Pair.of(Ctid.of(135136,1), Ctid.of(162162,185)), - Pair.of(Ctid.of(162163,1), Ctid.of(189189,185)), - Pair.of(Ctid.of(189190,1), Ctid.of(216216,185)), - Pair.of(Ctid.of(216217,1), Ctid.of(243243,185)), - Pair.of(Ctid.of(243244,1), Ctid.of(270270,185)), - Pair.of(Ctid.of(270271,1), Ctid.of(297297,185)), - Pair.of(Ctid.of(297298,1), Ctid.of(324324,185)), - Pair.of(Ctid.of(324325,1), Ctid.of(351351,185)), - Pair.of(Ctid.of(351352,1), Ctid.of(378378,185)), - Pair.of(Ctid.of(378379,1), Ctid.of(405405,185)), - Pair.of(Ctid.of(405406,1), Ctid.of(432432,185)), - Pair.of(Ctid.of(432433,1), Ctid.of(459459,185)), - Pair.of(Ctid.of(459460,1), Ctid.of(486486,185)), - Pair.of(Ctid.of(486487,1), Ctid.of(513513,185))); + Pair.of(Ctid.of(0, 185), Ctid.of(27027, 185)), + Pair.of(Ctid.of(27028, 1), Ctid.of(54054, 185)), + Pair.of(Ctid.of(54055, 1), Ctid.of(81081, 185)), + Pair.of(Ctid.of(81082, 1), Ctid.of(108108, 185)), + Pair.of(Ctid.of(108109, 1), Ctid.of(135135, 185)), + Pair.of(Ctid.of(135136, 1), Ctid.of(162162, 185)), + Pair.of(Ctid.of(162163, 1), Ctid.of(189189, 185)), + Pair.of(Ctid.of(189190, 1), Ctid.of(216216, 185)), + Pair.of(Ctid.of(216217, 1), Ctid.of(243243, 185)), + Pair.of(Ctid.of(243244, 1), Ctid.of(270270, 185)), + Pair.of(Ctid.of(270271, 1), Ctid.of(297297, 185)), + Pair.of(Ctid.of(297298, 1), Ctid.of(324324, 185)), + Pair.of(Ctid.of(324325, 1), Ctid.of(351351, 185)), + Pair.of(Ctid.of(351352, 1), Ctid.of(378378, 185)), + Pair.of(Ctid.of(378379, 1), Ctid.of(405405, 185)), + Pair.of(Ctid.of(405406, 1), Ctid.of(432432, 185)), + Pair.of(Ctid.of(432433, 1), Ctid.of(459459, 185)), + Pair.of(Ctid.of(459460, 1), Ctid.of(486486, 185)), + Pair.of(Ctid.of(486487, 1), Ctid.of(513513, 185))); assertEquals(expected, chunks); chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(new Ctid("(1314328,123)"), 9805450321L, 8192L, 45, GIGABYTE, 10005); expected = List.of( - Pair.of(Ctid.of("(1314328,124)"), Ctid.of(1314827,10005)), - Pair.of(Ctid.of("(1314828,1)"), Ctid.of(1315326,10005)), - Pair.of(Ctid.of("(1315327,1)"), Ctid.of(1315825,10005)), - Pair.of(Ctid.of("(1315826,1)"), Ctid.of(1316324,10005)), - Pair.of(Ctid.of("(1316325,1)"), Ctid.of(1316823,10005))); + Pair.of(Ctid.of("(1314328,124)"), Ctid.of(1314827, 10005)), + Pair.of(Ctid.of("(1314828,1)"), Ctid.of(1315326, 10005)), + Pair.of(Ctid.of("(1315327,1)"), Ctid.of(1315825, 10005)), + Pair.of(Ctid.of("(1315826,1)"), Ctid.of(1316324, 10005)), + Pair.of(Ctid.of("(1316325,1)"), Ctid.of(1316823, 10005))); assertEquals(expected, chunks); chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(0, 0), 380545L, 8192L, 45, GIGABYTE, 55); - expected = List.of(Pair.of(Ctid.of(0,1), Ctid.of(90909,55))); + expected = List.of(Pair.of(Ctid.of(0, 1), Ctid.of(90909, 55))); assertEquals(expected, chunks); chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(9876, 5432), 380545L, 8192L, 45, GIGABYTE, 5); expected = List.of( - Pair.of(Ctid.of(9877, 1), Ctid.of(1009877,5))); + Pair.of(Ctid.of(9877, 1), Ctid.of(1009877, 5))); assertEquals(expected, chunks); chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.ZERO, 4096L, 8192L, 45, GIGABYTE, 226); expected = List.of( - Pair.of(Ctid.of(0,1), Ctid.of(22123,226))); + Pair.of(Ctid.of(0, 1), Ctid.of(22123, 226))); assertEquals(expected, chunks); // Simulate an empty table - expected to generate an empty query plan chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.ZERO, 4096L, 8192L, 45, GIGABYTE, 1); - expected = List.of(Pair.of(Ctid.of(0,1), Ctid.of(5000000,1))); + expected = List.of(Pair.of(Ctid.of(0, 1), Ctid.of(5000000, 1))); assertEquals(expected, chunks); - chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(8,1), 819200L, 81920L, 50, EIGHT_KB, 1); - expected = List.of(Pair.of(Ctid.of(9,1), Ctid.of(10,1)), Pair.of(Ctid.of(11,1), Ctid.of(11,1))); + chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(8, 1), 819200L, 81920L, 50, EIGHT_KB, 1); + expected = List.of(Pair.of(Ctid.of(9, 1), Ctid.of(10, 1)), Pair.of(Ctid.of(11, 1), Ctid.of(11, 1))); assertEquals(expected, chunks); } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json index 01308d418942..219fe41cace1 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "type": {"type" : ["null", "string"]}, + "type": { "type": ["null", "string"] }, "eventID": { "type": "string" }, @@ -72,95 +72,129 @@ }, "_meta": { "type": ["null", "object"], - "properties" : { - "context": {"type" : ["null", "string"]}, - "contexts": {"type" : ["null", "string"]}, - "entries": {"type" : ["null", "object"]}, - "message": {"type" : ["null", "string"]}, - "packages": {"type" : ["null", "string"]}, - "sdk": {"type" : ["null", "string"]}, - "tags": {"type" : ["null", "object"]}, - "user": {"type" : ["null", "string"]} + "properties": { + "context": { "type": ["null", "string"] }, + "contexts": { "type": ["null", "string"] }, + "entries": { "type": ["null", "object"] }, + "message": { "type": ["null", "string"] }, + "packages": { "type": ["null", "string"] }, + "sdk": { "type": ["null", "string"] }, + "tags": { "type": ["null", "object"] }, + "user": { "type": ["null", "string"] } } }, "context": { - "type" : ["null", "object"], - "properties" : { + "type": ["null", "object"], + "properties": { "emptyList": { - "type" : ["null", "array"] + "type": ["null", "array"] }, "emptyMap": { - "type" : ["null", "object"] + "type": ["null", "object"] }, - "length": {"type" : ["null", "integer"]}, - "results": {"type" : ["null", "array"], "items" : {"type" : "integer"}}, - "session": {"type" : ["null", "object"], "properties" : { - "foo": { - "type" : ["null", "string"] + "length": { "type": ["null", "integer"] }, + "results": { + "type": ["null", "array"], + "items": { "type": "integer" } + }, + "session": { + "type": ["null", "object"], + "properties": { + "foo": { + "type": ["null", "string"] + } } - }}, - "unauthorized": {"type" : ["null", "boolean"]}, - "url": {"type" : ["null", "string"]} + }, + "unauthorized": { "type": ["null", "boolean"] }, + "url": { "type": ["null", "string"] } } }, "contexts": { - "type" : ["null", "object"], - "properties" : { - "browser": {"type" : ["null", "object"], "properties" : { - "name": {"type" : ["null", "string"]}, - "type": {"type" : ["null", "string"]}, - "version": {"type" : ["null", "string"]} - }}, - "client_os": {"type" : ["null", "object"], "properties" : { - "name": {"type" : ["null", "string"]}, - "type": {"type" : ["null", "string"]}, - "version": {"type" : ["null", "string"]} - }} + "type": ["null", "object"], + "properties": { + "browser": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "version": { "type": ["null", "string"] } + } + }, + "client_os": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "version": { "type": ["null", "string"] } + } + } + } + }, + "crashFile": { "type": ["null", "string"] }, + "culprit": { "type": ["null", "string"] }, + "dateReceived": { "type": ["null", "string"], "format": "date-time" }, + "dist": { "type": ["null", "string"] }, + "entries": { + "type": ["null", "array"], + "items": { "type": ["null", "object"] }, + "properties": { + "data": { + "type": ["null", "object"], + "properties": { + "formatted": { "type": ["null", "string"] }, + "frames": { + "type": ["null", "object"], + "properties": { + "absPath": { "type": ["null", "string"] }, + "colNo": { "type": ["null", "string"] }, + "context": { "type": ["null", "array"] }, + "filename": { "type": ["null", "string"] }, + "function": { "type": ["null", "string"] }, + "inApp": { "type": ["null", "boolean"] }, + "lineNo": { "type": ["null", "integer"] }, + "lock": { "type": ["null", "integer"] }, + "module": { "type": ["null", "string"] }, + "package": { "type": ["null", "string"] }, + "platform": { "type": ["null", "string"] }, + "rawFunction": { "type": ["null", "string"] }, + "sourceLink": { "type": ["null", "string"] }, + "symbol": { "type": ["null", "string"] }, + "symbolAddr": { "type": ["null", "string"] }, + "trust": { "type": ["null", "string"] } + } + } + } + }, + "type": { "type": ["null", "string"] } + } + }, + "errors": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "fingerprints": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "groupingConfig": { + "type": ["null", "object"], + "properties": { + "enhancements": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] } + } + }, + "location": { "type": ["null", "string"] }, + "metadata": { + "type": ["null", "object"], + "properties": { + "in_app_frame_mix": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } } }, - "crashFile": {"type" : ["null", "string"]}, - "culprit": {"type" : ["null", "string"]}, - "dateReceived": {"type" : ["null", "string"], "format" : "date-time"}, - "dist": {"type" : ["null", "string"]}, - "entries": {"type" : ["null", "array"], "items" : {"type" : ["null", "object"]}, "properties" : { - "data": {"type" : ["null", "object"], "properties" : { - "formatted": {"type" : ["null", "string"]}, - "frames": {"type" : ["null", "object"], "properties" : { - "absPath": {"type" : ["null", "string"]}, - "colNo": {"type" : ["null", "string"]}, - "context": {"type" : ["null", "array"]}, - "filename": {"type" : ["null", "string"]}, - "function": {"type" : ["null", "string"]}, - "inApp": {"type" : ["null", "boolean"]}, - "lineNo": {"type" : ["null", "integer"]}, - "lock": {"type" : ["null", "integer"]}, - "module": {"type" : ["null", "string"]}, - "package": {"type" : ["null", "string"]}, - "platform": {"type" : ["null", "string"]}, - "rawFunction": {"type" : ["null", "string"]}, - "sourceLink": {"type" : ["null", "string"]}, - "symbol": {"type" : ["null", "string"]}, - "symbolAddr": {"type" : ["null", "string"]}, - "trust": {"type" : ["null", "string"]} - }} - }}, - "type": {"type" : ["null", "string"]} - }}, - "errors": {"type" : ["null", "array"], "items" : {"type" : ["null", "string"]}}, - "fingerprints": {"type" : ["null", "array"], "items" : {"type" : ["null", "string"]}}, - "groupingConfig": {"type" : ["null", "object"], "properties" : { - "enhancements": {"type" : ["null", "string"]}, - "id": {"type" : ["null", "string"]} - }}, - "location": {"type" : ["null", "string"]}, - "metadata": {"type" : ["null", "object"], "properties" : { - "in_app_frame_mix": {"type" : ["null", "string"]}, - "title": {"type" : ["null", "string"]} - }}, - "occurrence": {"type" : ["null", "string"]}, - "packages": {"type" : ["null", "object"]}, - "projectID": {"type" : ["null", "string"]}, - "sdk": {"type" : ["null", "string"]}, - "size": {"type" : ["null", "integer"]} + "occurrence": { "type": ["null", "string"] }, + "packages": { "type": ["null", "object"] }, + "projectID": { "type": ["null", "string"] }, + "sdk": { "type": ["null", "string"] }, + "size": { "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json index 53957231edbc..c5660d0dc62c 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json @@ -130,19 +130,19 @@ "type": "integer" }, "isUnhandled": { - "type" : ["null", "boolean"] + "type": ["null", "boolean"] }, "issueCategory": { - "type" : ["null", "string"] + "type": ["null", "string"] }, "issueType": { - "type" : ["null", "string"] + "type": ["null", "string"] }, "platform": { - "type" : ["null", "string"] + "type": ["null", "string"] }, "substatus": { - "type" : ["null", "string"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json index 061aa7221bda..a78ca92f0f5f 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json @@ -399,49 +399,49 @@ "type": "boolean" }, "access": { - "type" : ["null", "array"], - "items" : { - "type" : ["null", "string"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] } }, "builtinSymbolSources": { - "type" : ["null", "array"], - "items" : { - "type" : ["null", "string"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] } }, "dynamicSamplingBiases": { - "type" : ["null", "array"], - "items" : { - "type" : ["null", "object"], - "properties" : { - "id": {"type" : ["null", "string"]}, - "active": {"type" : ["null", "boolean"]} + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "active": { "type": ["null", "boolean"] } } } }, "eventProcessing": { - "type" : ["null", "object"], - "properties" : { + "type": ["null", "object"], + "properties": { "symbolicationDegraded": { - "type" : ["null", "boolean"] + "type": ["null", "boolean"] } } }, - "fingerprintingRules": {"type" : ["null", "string"]}, - "firstTransactionEvent": {"type" : ["null", "boolean"]}, - "groupingAutoUpdate": {"type" : ["null", "boolean"]}, - "groupingConfig": {"type" : ["null", "string"]}, - "groupingEnhancements": {"type" : ["null", "string"]}, - "groupingEnhancementsBase": {"type" : ["null", "string"]}, - "hasMinifiedStackTrace": {"type" : ["null", "boolean"]}, - "hasMonitors": {"type" : ["null", "boolean"]}, - "hasProfiles": {"type" : ["null", "boolean"]}, - "hasReplays": {"type" : ["null", "boolean"]}, - "hasSessions": {"type" : ["null", "boolean"]}, - "recapServerUrl": {"type" : ["null", "string"]}, - "secondaryGroupingConfig": {"type" : ["null", "string"]}, - "secondaryGroupingExpiry": {"type" : ["null", "integer"]}, - "symbolSources": {"type" : ["null", "string"]} + "fingerprintingRules": { "type": ["null", "string"] }, + "firstTransactionEvent": { "type": ["null", "boolean"] }, + "groupingAutoUpdate": { "type": ["null", "boolean"] }, + "groupingConfig": { "type": ["null", "string"] }, + "groupingEnhancements": { "type": ["null", "string"] }, + "groupingEnhancementsBase": { "type": ["null", "string"] }, + "hasMinifiedStackTrace": { "type": ["null", "boolean"] }, + "hasMonitors": { "type": ["null", "boolean"] }, + "hasProfiles": { "type": ["null", "boolean"] }, + "hasReplays": { "type": ["null", "boolean"] }, + "hasSessions": { "type": ["null", "boolean"] }, + "recapServerUrl": { "type": ["null", "string"] }, + "secondaryGroupingConfig": { "type": ["null", "string"] }, + "secondaryGroupingExpiry": { "type": ["null", "integer"] }, + "symbolSources": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json index 62a2a56ea03c..2883aad1db39 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json @@ -116,16 +116,16 @@ "enum": ["active", "disabled", "pending_deletion", "deletion_in_progress"] }, "access": { - "type" : ["null", "array"], - "items" : { - "type" : ["null", "string"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] } }, - "firstTransactionEvent": {"type" : ["null", "boolean"]}, - "hasMonitors": {"type" : ["null", "boolean"]}, - "hasProfiles": {"type" : ["null", "boolean"]}, - "hasReplays": {"type" : ["null", "boolean"]}, - "hasSessions": {"type" : ["null", "boolean"]}, - "hasMinifiedStackTrace": {"type" : ["null", "boolean"]} + "firstTransactionEvent": { "type": ["null", "boolean"] }, + "hasMonitors": { "type": ["null", "boolean"] }, + "hasProfiles": { "type": ["null", "boolean"] }, + "hasReplays": { "type": ["null", "boolean"] }, + "hasSessions": { "type": ["null", "boolean"] }, + "hasMinifiedStackTrace": { "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json index c581d5430c72..d6ea85853ac0 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json @@ -85,24 +85,26 @@ "type": ["null", "string"] }, "currentProjectMeta": { - "type" : ["null", "object"] - }, - "status": {"type" : ["null", "string"]}, - "userAgent": {"type" : ["null", "string"]}, - "versionInfo": {"type" : ["null", "object"], - "properties" : { - "buildHash": {"type" : ["null", "string"]}, - "description": {"type" : ["null", "string"]}, - "package": {"type" : ["null", "string"]}, - "version": {"type" : ["null", "object"], - "properties" : { - "buildCode": {"type" : ["null", "string"]}, - "components": {"type" : ["null", "integer"]}, - "major": {"type" : ["null", "integer"]}, - "minor": {"type" : ["null", "integer"]}, - "patch": {"type" : ["null", "integer"]}, - "pre": {"type" : ["null", "string"]}, - "raw": {"type" : ["null", "string"]} + "type": ["null", "object"] + }, + "status": { "type": ["null", "string"] }, + "userAgent": { "type": ["null", "string"] }, + "versionInfo": { + "type": ["null", "object"], + "properties": { + "buildHash": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "package": { "type": ["null", "string"] }, + "version": { + "type": ["null", "object"], + "properties": { + "buildCode": { "type": ["null", "string"] }, + "components": { "type": ["null", "integer"] }, + "major": { "type": ["null", "integer"] }, + "minor": { "type": ["null", "integer"] }, + "patch": { "type": ["null", "integer"] }, + "pre": { "type": ["null", "string"] }, + "raw": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json index bc5819d2c48a..29376d5432ca 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -1,601 +1,601 @@ [ - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "customers" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "draft_orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "products" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updatedAt": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "products_graph_ql" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "abandoned_checkouts" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafields" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999 - }, - "stream_descriptor": { - "name": "collects" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "custom_collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "created_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "order_refunds" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "id": 99999999999999 - }, - "stream_descriptor": { - "name": "order_risks" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "created_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "transactions" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "processed_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "tender_transactions" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "pages" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "price_rules" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "price_rules": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "discount_codes" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "inventory_items" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "locations": {}, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "inventory_levels" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "id": 99999999999999 - }, - "stream_descriptor": { - "name": "fulfillment_orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "fulfillments" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": {}, - "stream_descriptor": { - "name": "balance_transactions" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "stream_descriptor": { - "name": "articles" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "articles": { - "id": 99999999999999, - "deleted": {} - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_articles" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "stream_descriptor": { - "name": "blogs" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00", - "blogs": { - "id": 99999999999999 - } - }, - "stream_descriptor": { - "name": "metafield_blogs" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "customers": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_customers" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "orders": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "draft_orders": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_draft_orders" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_products" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - } - }, - "stream_descriptor": { - "name": "product_images" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_product_images" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - } - }, - "stream_descriptor": { - "name": "product_variants" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "products": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_product_variants" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "collects": { - "id": 99999999999999 - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "collects": { - "id": 99999999999999 - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "smart_collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "smart_collections": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_smart_collections" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "pages": { - "updated_at": "2027-07-11T13:07:45-07:00", - "deleted": { - "deleted_at": "2027-07-11T13:07:45-07:00" - } - }, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_pages" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_shops" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "locations": {}, - "updated_at": "2027-07-11T13:07:45-07:00" - }, - "stream_descriptor": { - "name": "metafield_locations" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": {}, - "stream_descriptor": { - "name": "disputes" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": {}, - "stream_descriptor": { - "name": "customer_saved_search" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "id": 99999999999999, - "customers": { - "updated_at": "2027-07-11T13:07:45-07:00" - } - }, - "stream_descriptor": { - "name": "customer_address" - } - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "customers" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "draft_orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "products" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updatedAt": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "products_graph_ql" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "abandoned_checkouts" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafields" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999 + }, + "stream_descriptor": { + "name": "collects" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "custom_collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "created_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "order_refunds" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "id": 99999999999999 + }, + "stream_descriptor": { + "name": "order_risks" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "created_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "transactions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "processed_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "tender_transactions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "pages" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "price_rules" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "price_rules": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "discount_codes" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "inventory_items" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "locations": {}, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "inventory_levels" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "id": 99999999999999 + }, + "stream_descriptor": { + "name": "fulfillment_orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "fulfillments" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": {}, + "stream_descriptor": { + "name": "balance_transactions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "stream_descriptor": { + "name": "articles" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "articles": { + "id": 99999999999999, + "deleted": {} + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_articles" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "stream_descriptor": { + "name": "blogs" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00", + "blogs": { + "id": 99999999999999 + } + }, + "stream_descriptor": { + "name": "metafield_blogs" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "customers": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_customers" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "draft_orders": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_draft_orders" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_products" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + } + }, + "stream_descriptor": { + "name": "product_images" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_product_images" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + } + }, + "stream_descriptor": { + "name": "product_variants" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_product_variants" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "collects": { + "id": 99999999999999 + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "collects": { + "id": 99999999999999 + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "smart_collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "smart_collections": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_smart_collections" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "pages": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_pages" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_shops" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "locations": {}, + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "stream_descriptor": { + "name": "metafield_locations" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": {}, + "stream_descriptor": { + "name": "disputes" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": {}, + "stream_descriptor": { + "name": "customer_saved_search" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "customers": { + "updated_at": "2027-07-11T13:07:45-07:00" + } + }, + "stream_descriptor": { + "name": "customer_address" + } } + } ] diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json index f96c4878b03f..25734860ba98 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -1,257 +1,257 @@ { + "customers": { + "updated_at": "2023-07-11T13:07:45-07:00" + }, + "orders": { + "deleted": { + "deleted_at": "" + }, + "updated_at": "2023-04-24T11:00:10-07:00" + }, + "draft_orders": { + "updated_at": "2023-07-11T12:57:55-07:00" + }, + "products": { + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + }, + "updated_at": "2023-09-05T07:12:05-07:00" + }, + "products_graph_ql": { + "updatedAt": "2023-09-05T14:12:05Z" + }, + "abandoned_checkouts": { + "updated_at": "2023-07-11T13:07:45-07:00" + }, + "metafields": { + "updated_at": "2022-05-30T23:42:02-07:00" + }, + "collects": { + "id": 29427031703741 + }, + "custom_collections": { + "deleted": { + "deleted_at": "2023-09-06T03:34:39-07:00" + }, + "updated_at": "2023-09-05T07:06:59-07:00" + }, + "order_refunds": { + "orders": { + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } + }, + "created_at": "2022-10-10T06:21:53-07:00" + }, + "order_risks": { + "orders": { + "updated_at": "2023-04-24T07:03:06-07:00", + "deleted": { + "deleted_at": "" + } + }, + "id": 6446736474301 + }, + "transactions": { + "orders": { + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } + }, + "created_at": "2023-04-24T11:00:08-07:00" + }, + "tender_transactions": { + "processed_at": "2023-04-24T11:00:08-07:00" + }, + "pages": { + "deleted": { + "deleted_at": "2023-09-06T03:37:06-07:00" + }, + "updated_at": "2023-04-24T11:08:41-07:00" + }, + "price_rules": { + "deleted": { + "deleted_at": "2023-09-06T03:48:46-07:00" + }, + "updated_at": "2023-04-24T05:52:22-07:00" + }, + "discount_codes": { + "price_rules": { + "updated_at": "2023-04-24T05:52:22-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-24T05:52:22-07:00" + }, + "inventory_items": { + "products": { + "updated_at": "2023-04-20T04:12:51-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-14T03:29:27-07:00" + }, + "inventory_levels": { + "locations": {}, + "updated_at": "2023-04-24T11:00:10-07:00" + }, + "fulfillment_orders": { + "orders": { + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } + }, + "id": 5985636450493 + }, + "fulfillments": { + "orders": { + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-24T11:00:09-07:00" + }, + "balance_transactions": {}, + "articles": { + "id": 558999404733, + "deleted": { + "deleted_at": "2023-09-05T14:02:00-07:00" + } + }, + "metafield_articles": { + "articles": { + "id": 558627979453, + "deleted": {} + }, + "updated_at": "2023-04-14T03:19:18-07:00" + }, + "blogs": { + "id": 85733114045, + "deleted": { + "deleted_at": "2023-09-06T03:30:22-07:00" + } + }, + "metafield_blogs": { + "updated_at": "2022-10-07T06:05:23-07:00", + "blogs": { + "id": 80417685693 + } + }, + "metafield_customers": { "customers": { - "updated_at": "2023-07-11T13:07:45-07:00" + "updated_at": "2023-04-24T06:53:48-07:00" }, + "updated_at": "2023-04-13T04:50:10-07:00" + }, + "metafield_orders": { "orders": { - "deleted": { - "deleted_at": "" - }, - "updated_at": "2023-04-24T11:00:10-07:00" - }, + "updated_at": "2023-04-24T10:59:00-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-14T03:52:40-07:00" + }, + "metafield_draft_orders": { "draft_orders": { - "updated_at": "2023-07-11T12:57:55-07:00" + "updated_at": "2023-04-24T07:18:06-07:00" }, + "updated_at": "2023-04-24T07:18:06-07:00" + }, + "metafield_products": { "products": { - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - }, - "updated_at": "2023-09-05T07:12:05-07:00" - }, - "products_graph_ql": { - "updatedAt": "2023-09-05T14:12:05Z" - }, - "abandoned_checkouts": { - "updated_at": "2023-07-11T13:07:45-07:00" - }, - "metafields": { - "updated_at": "2022-05-30T23:42:02-07:00" - }, + "updated_at": "2023-04-20T04:12:59-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-14T04:04:46-07:00" + }, + "product_images": { + "id": 33290489659581, + "products": { + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + } + }, + "metafield_product_images": { + "products": { + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + }, + "updated_at": "2023-04-24T10:32:19-07:00" + }, + "product_variants": { + "id": 42778150305981, + "products": { + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + } + }, + "metafield_product_variants": { + "products": { + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + }, + "updated_at": "2023-04-14T03:29:27-07:00" + }, + "collections": { "collects": { - "id": 29427031703741 - }, - "custom_collections": { - "deleted": { - "deleted_at": "2023-09-06T03:34:39-07:00" - }, - "updated_at": "2023-09-05T07:06:59-07:00" - }, - "order_refunds": { - "orders": { - "updated_at": "2023-04-24T11:00:10-07:00", - "deleted": { - "deleted_at": "" - } - }, - "created_at": "2022-10-10T06:21:53-07:00" - }, - "order_risks": { - "orders": { - "updated_at": "2023-04-24T07:03:06-07:00", - "deleted": { - "deleted_at": "" - } - }, - "id": 6446736474301 - }, - "transactions": { - "orders": { - "updated_at": "2023-04-24T11:00:10-07:00", - "deleted": { - "deleted_at": "" - } - }, - "created_at": "2023-04-24T11:00:08-07:00" - }, - "tender_transactions": { - "processed_at": "2023-04-24T11:00:08-07:00" - }, - "pages": { - "deleted": { - "deleted_at": "2023-09-06T03:37:06-07:00" - }, - "updated_at": "2023-04-24T11:08:41-07:00" - }, - "price_rules": { - "deleted": { - "deleted_at": "2023-09-06T03:48:46-07:00" - }, - "updated_at": "2023-04-24T05:52:22-07:00" - }, - "discount_codes": { - "price_rules": { - "updated_at": "2023-04-24T05:52:22-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-24T05:52:22-07:00" - }, - "inventory_items": { - "products": { - "updated_at": "2023-04-20T04:12:51-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-14T03:29:27-07:00" - }, - "inventory_levels": { - "locations": {}, - "updated_at": "2023-04-24T11:00:10-07:00" - }, - "fulfillment_orders": { - "orders": { - "updated_at": "2023-04-24T11:00:10-07:00", - "deleted": { - "deleted_at": "" - } - }, - "id": 5985636450493 - }, - "fulfillments": { - "orders": { - "updated_at": "2023-04-24T11:00:10-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-24T11:00:09-07:00" - }, - "balance_transactions": {}, - "articles": { - "id": 558999404733, - "deleted": { - "deleted_at": "2023-09-05T14:02:00-07:00" - } - }, - "metafield_articles": { - "articles": { - "id": 558627979453, - "deleted": {} - }, - "updated_at": "2023-04-14T03:19:18-07:00" - }, - "blogs": { - "id": 85733114045, - "deleted": { - "deleted_at": "2023-09-06T03:30:22-07:00" - } - }, - "metafield_blogs": { - "updated_at": "2022-10-07T06:05:23-07:00", - "blogs": { - "id": 80417685693 - } - }, - "metafield_customers": { - "customers": { - "updated_at": "2023-04-24T06:53:48-07:00" - }, - "updated_at": "2023-04-13T04:50:10-07:00" - }, - "metafield_orders": { - "orders": { - "updated_at": "2023-04-24T10:59:00-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-14T03:52:40-07:00" - }, - "metafield_draft_orders": { - "draft_orders": { - "updated_at": "2023-04-24T07:18:06-07:00" - }, - "updated_at": "2023-04-24T07:18:06-07:00" - }, - "metafield_products": { - "products": { - "updated_at": "2023-04-20T04:12:59-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-14T04:04:46-07:00" - }, - "product_images": { - "id": 33290489659581, - "products": { - "updated_at": "", - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - } - } - }, - "metafield_product_images": { - "products": { - "updated_at": "", - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - } - }, - "updated_at": "2023-04-24T10:32:19-07:00" - }, - "product_variants": { - "id": 42778150305981, - "products": { - "updated_at": "", - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - } - } - }, - "metafield_product_variants": { - "products": { - "updated_at": "", - "deleted": { - "deleted_at": "2023-09-05T13:32:22-07:00" - } - }, - "updated_at": "2023-04-14T03:29:27-07:00" - }, - "collections": { - "collects": { - "id": 29427031703741 - }, - "updated_at": "2023-03-20T06:08:12-07:00" - }, - "metafield_collections": { - "collects": { - "id": 29427031703741 - }, - "updated_at": "2022-10-08T04:44:51-07:00" + "id": 29427031703741 }, + "updated_at": "2023-03-20T06:08:12-07:00" + }, + "metafield_collections": { + "collects": { + "id": 29427031703741 + }, + "updated_at": "2022-10-08T04:44:51-07:00" + }, + "smart_collections": { + "updated_at": "2023-09-05T07:12:04-07:00" + }, + "metafield_smart_collections": { "smart_collections": { - "updated_at": "2023-09-05T07:12:04-07:00" - }, - "metafield_smart_collections": { - "smart_collections": { - "updated_at": "2023-09-05T07:12:04-07:00" - }, - "updated_at": "2023-04-14T05:21:58-07:00" - }, - "metafield_pages": { - "pages": { - "updated_at": "2023-04-24T11:08:41-07:00", - "deleted": { - "deleted_at": "" - } - }, - "updated_at": "2023-04-24T11:08:41-07:00" - }, - "metafield_shops": { - "updated_at": "2023-04-24T11:12:38-07:00" + "updated_at": "2023-09-05T07:12:04-07:00" }, - "metafield_locations": { - "locations": {}, - "updated_at": "2022-10-12T02:21:35-07:00" - }, - "disputes": {}, - "customer_saved_search": {}, - "customer_address": { - "id": 8212915650749, - "customers": { - "updated_at": "2023-07-11T13:07:45-07:00" - } + "updated_at": "2023-04-14T05:21:58-07:00" + }, + "metafield_pages": { + "pages": { + "updated_at": "2023-04-24T11:08:41-07:00", + "deleted": { + "deleted_at": "" + } + }, + "updated_at": "2023-04-24T11:08:41-07:00" + }, + "metafield_shops": { + "updated_at": "2023-04-24T11:12:38-07:00" + }, + "metafield_locations": { + "locations": {}, + "updated_at": "2022-10-12T02:21:35-07:00" + }, + "disputes": {}, + "customer_saved_search": {}, + "customer_address": { + "id": 8212915650749, + "customers": { + "updated_at": "2023-07-11T13:07:45-07:00" } + } } diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json index 88686d74f32b..8e298abbbbc6 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json @@ -23,7 +23,7 @@ "float_ts": { "type": ["null", "number"] }, - "subtype": { + "subtype": { "type": ["null", "string"] }, "is_locked": { diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json index 45b2339775bb..5468844c8226 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json @@ -1,98 +1,50 @@ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "object": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "created": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "display_name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "effective_percentage": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "inclusive": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "jurisdiction": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "livemode": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "metadata": { - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "percentage": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "state": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "tax_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml b/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml index 7f762698cfc8..51b80752d8b5 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml +++ b/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml @@ -49,12 +49,11 @@ definitions: pagination_strategy: type: CursorPagination page_size: 500 - cursor_value: '{{ (last_records|last)[''id''] }}' - stop_condition: '{{ not last_records }}' + cursor_value: "{{ (last_records|last)['id'] }}" + stop_condition: "{{ not last_records }}" board_id_partition_router: - type: CustomPartitionRouter - class_name: - source_trello.components.OrderIdsPartitionRouter + class_name: source_trello.components.OrderIdsPartitionRouter parent_stream_configs: - type: ParentStreamConfig parent_key: id @@ -76,7 +75,7 @@ definitions: $ref: "#/definitions/requester" path: members/me/boards request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" record_selector: $ref: "#/definitions/boards_selector" organizations_stream: @@ -89,7 +88,7 @@ definitions: $ref: "#/definitions/requester" path: members/me/organizations request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" record_selector: $ref: "#/definitions/selector" actions_stream: @@ -111,20 +110,20 @@ definitions: type: DatetimeBasedCursor cursor_field: date cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%S.%fZ' - datetime_format: '%Y-%m-%dT%H:%M:%S.%fZ' + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" start_datetime: type: MinMaxDatetime - datetime: '{{ config[''start_date''] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: since inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" cards_stream: type: DeclarativeStream name: cards @@ -135,13 +134,13 @@ definitions: $ref: "#/definitions/requester" path: boards/{{ stream_partition.id }}/cards/all request_parameters: - list: 'true' - sort: '-id' - since: '{{ config[''start_date''] }}' - members: 'true' - pluginData: 'true' - actions_display: 'true' - customFieldItems: 'true' + list: "true" + sort: "-id" + since: "{{ config['start_date'] }}" + members: "true" + pluginData: "true" + actions_display: "true" + customFieldItems: "true" record_selector: $ref: "#/definitions/selector" paginator: @@ -158,7 +157,7 @@ definitions: $ref: "#/definitions/requester" path: boards/{{ stream_partition.id }}/checklists request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" fields: all checkItem_fields: all record_selector: @@ -175,7 +174,7 @@ definitions: $ref: "#/definitions/requester" path: boards/{{ stream_partition.id }}/lists request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" record_selector: $ref: "#/definitions/selector" partition_router: @@ -190,7 +189,7 @@ definitions: $ref: "#/definitions/requester" path: boards/{{ stream_partition.id }}/members request_parameters: - since: '{{ config[''start_date''] }}' + since: "{{ config['start_date'] }}" record_selector: $ref: "#/definitions/selector" partition_router: @@ -235,7 +234,7 @@ spec: pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" description: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. examples: - - '2021-03-01T00:00:00Z' + - "2021-03-01T00:00:00Z" format: date-time order: 2 board_ids: diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json index e6e72140e9f7..6f370225b0aa 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json @@ -6,13 +6,13 @@ "actor": { "type": ["null", "object"], "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - } + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] } + } }, "id": { "type": ["null", "integer"] @@ -23,7 +23,7 @@ "description": { "type": ["null", "string"] }, - "deleted_at": { + "deleted_at": { "type": ["null", "string"], "format": "date-time" },