diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 165d4475750e..48b176f41a22 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.63.3 +current_version = 0.63.4 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.github/workflows/connectors_version_increment_check.yml b/.github/workflows/connectors_version_increment_check.yml index 280a199a4285..9339440ff14c 100644 --- a/.github/workflows/connectors_version_increment_check.yml +++ b/.github/workflows/connectors_version_increment_check.yml @@ -41,7 +41,7 @@ jobs: uses: ./.github/actions/run-airbyte-ci with: context: "pull_request" - dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} + # dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} Commenting this out as we believe Dagger cloud caching is causing excessively long jobs for such a small check docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 8f7bb15e61c9..9cff7e2029cd 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -174,14 +174,18 @@ corresponds to that version. | Version | Date | Pull Request | Subject | |:--------| :--------- | :--------------------------------------------------------- |:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.40.9 | 2024-07-01 | [\#39473](https://github.com/airbytehq/airbyte/pull/39473) | minor changes around error logging and testing | +| 0.40.8 | 2024-07-01 | [\#40499](https://github.com/airbytehq/airbyte/pull/40499) | Make JdbcDatabase SQL statement logging optional; add generation_id support to JdbcSqlGenerator | +| 0.40.7 | 2024-07-01 | [\#40516](https://github.com/airbytehq/airbyte/pull/40516) | Remove dbz hearbeat. | +| ~~0.40.6~~ | | | (this version does not exist) | | 0.40.5 | 2024-06-26 | [\#40517](https://github.com/airbytehq/airbyte/pull/40517) | JdbcDatabase.executeWithinTransaction allows disabling SQL statement logging | -| 0.35.16 | 2024-06-25 | [\#40517](https://github.com/airbytehq/airbyte/pull/40517) | (backport) JdbcDatabase.executeWithinTransaction allows disabling SQL statement logging | | 0.40.4 | 2024-06-18 | [\#40254](https://github.com/airbytehq/airbyte/pull/40254) | Destinations: Do not throw on unrecognized airbyte message type (ignore message instead) | | 0.40.3 | 2024-06-18 | [\#39526](https://github.com/airbytehq/airbyte/pull/39526) | Destinations: INCOMPLETE stream status is a TRANSIENT error rather than SYSTEM | | 0.40.2 | 2024-06-18 | [\#39552](https://github.com/airbytehq/airbyte/pull/39552) | Destinations: Throw error if the ConfiguredCatalog has no streams | | 0.40.1 | 2024-06-14 | [\#39349](https://github.com/airbytehq/airbyte/pull/39349) | Source stats for full refresh streams | | 0.40.0 | 2024-06-17 | [\#38622](https://github.com/airbytehq/airbyte/pull/38622) | Destinations: Implement refreshes logic in AbstractStreamOperation | | 0.39.0 | 2024-06-17 | [\#38067](https://github.com/airbytehq/airbyte/pull/38067) | Destinations: Breaking changes for refreshes (fail on INCOMPLETE stream status; ignore OVERWRITE sync mode) | +| 0.38.3 | 2024-06-25 | [\#40499](https://github.com/airbytehq/airbyte/pull/40499) | (backport) Make JdbcDatabase SQL statement logging optional; add generation_id support to JdbcSqlGenerator | | 0.38.2 | 2024-06-14 | [\#39460](https://github.com/airbytehq/airbyte/pull/39460) | Bump postgres JDBC driver version | | 0.38.1 | 2024-06-13 | [\#39445](https://github.com/airbytehq/airbyte/pull/39445) | Sources: More CDK changes to handle big initial snapshots. | | 0.38.0 | 2024-06-11 | [\#39405](https://github.com/airbytehq/airbyte/pull/39405) | Sources: Debezium properties manager interface changed to accept a list of streams to scope to | @@ -194,6 +198,7 @@ corresponds to that version. | 0.36.4 | 2024-05-31 | [\#38824](https://github.com/airbytehq/airbyte/pull/38824) | Param marked as non-null to nullable in JdbcDestinationHandler for NPE fix | | 0.36.2 | 2024-05-29 | [\#38538](https://github.com/airbytehq/airbyte/pull/38357) | Exit connector when encountering a config error. | | 0.36.0 | 2024-05-29 | [\#38358](https://github.com/airbytehq/airbyte/pull/38358) | Plumb generation_id / sync_id to destinations code | +| 0.35.16 | 2024-06-25 | [\#40517](https://github.com/airbytehq/airbyte/pull/40517) | (backport) JdbcDatabase.executeWithinTransaction allows disabling SQL statement logging | | 0.35.15 | 2024-05-31 | [\#38824](https://github.com/airbytehq/airbyte/pull/38824) | Param marked as non-null to nullable in JdbcDestinationHandler for NPE fix | | 0.35.14 | 2024-05-28 | [\#38738](https://github.com/airbytehq/airbyte/pull/38738) | make ThreadCreationInfo cast as nullable | | 0.35.13 | 2024-05-28 | [\#38632](https://github.com/airbytehq/airbyte/pull/38632) | minor changes to allow conversion of snowflake tests to kotlin | diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/db/DbAnalyticsUtils.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/db/DbAnalyticsUtils.kt index 75eda5761212..adde870d15af 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/db/DbAnalyticsUtils.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/db/DbAnalyticsUtils.kt @@ -14,6 +14,7 @@ object DbAnalyticsUtils { const val CDC_CURSOR_INVALID_KEY: String = "db-sources-cdc-cursor-invalid" const val DATA_TYPES_SERIALIZATION_ERROR_KEY = "db-sources-data-serialization-error" const val CDC_SNAPSHOT_FORCE_SHUTDOWN_KEY = "db-sources-snapshot-force-shutdown" + const val DEBEZIUM_CLOSE_REASON_KEY = "db-sources-debezium-close-reason" @JvmStatic fun cdcCursorInvalidMessage(): AirbyteAnalyticsTraceMessage { @@ -33,4 +34,9 @@ object DbAnalyticsUtils { .withType(CDC_SNAPSHOT_FORCE_SHUTDOWN_KEY) .withValue("1") } + + @JvmStatic + fun debeziumCloseReasonMessage(reason: String): AirbyteAnalyticsTraceMessage { + return AirbyteAnalyticsTraceMessage().withType(DEBEZIUM_CLOSE_REASON_KEY).withValue(reason) + } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/IntegrationRunner.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/IntegrationRunner.kt index b22cf8c74dcb..f707c6b5cfec 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/IntegrationRunner.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/IntegrationRunner.kt @@ -213,6 +213,7 @@ internal constructor( } } } catch (e: Exception) { + LOGGER.error(e) { "caught exception!" } // Many of the exceptions thrown are nested inside layers of RuntimeExceptions. An // attempt is made // to diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index bd1da13e5e26..899bf5c00e3d 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.40.5 +version=0.40.9 diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.kt index f30c69b7f0d5..4ccc77d75256 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.kt @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import io.airbyte.cdk.db.jdbc.JdbcDatabase import io.airbyte.cdk.integrations.base.JavaBaseConstants +import io.airbyte.cdk.integrations.base.JavaBaseConstants.DestinationColumns import io.airbyte.cdk.integrations.destination.jdbc.ColumnDefinition import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil.getResultsOrLogAndThrowFirst @@ -50,7 +51,8 @@ abstract class JdbcDestinationHandler( protected val catalogName: String?, protected val jdbcDatabase: JdbcDatabase, protected val rawTableNamespace: String, - private val dialect: SQLDialect + private val dialect: SQLDialect, + private val columns: DestinationColumns = DestinationColumns.V2_WITH_GENERATION, ) : DestinationHandler { protected val dslContext: DSLContext get() = DSL.using(dialect) @@ -363,6 +365,14 @@ abstract class JdbcDestinationHandler( ) } + protected open fun isAirbyteGenerationColumnMatch(existingTable: TableDefinition): Boolean { + return toJdbcTypeName(AirbyteProtocolType.INTEGER) + .equals( + existingTable.columns.getValue(JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID).type, + ignoreCase = true, + ) + } + open protected fun existingSchemaMatchesStreamConfig( stream: StreamConfig?, existingTable: TableDefinition @@ -375,7 +385,11 @@ abstract class JdbcDestinationHandler( JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT ) && isAirbyteExtractedAtColumnMatch(existingTable)) || !(existingTable.columns.containsKey(JavaBaseConstants.COLUMN_NAME_AB_META) && - isAirbyteMetaColumnMatch(existingTable)) + isAirbyteMetaColumnMatch(existingTable)) || + (columns == DestinationColumns.V2_WITH_GENERATION && + !(existingTable.columns.containsKey( + JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID + ) && isAirbyteGenerationColumnMatch(existingTable))) ) { // Missing AB meta columns from final table, we need them to do proper T+D so trigger // soft-reset diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.kt index b66b003aef0e..656408b43bbb 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.kt @@ -5,6 +5,7 @@ package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping import com.google.common.annotations.VisibleForTesting import io.airbyte.cdk.integrations.base.JavaBaseConstants +import io.airbyte.cdk.integrations.base.JavaBaseConstants.DestinationColumns import io.airbyte.cdk.integrations.destination.NamingConventionTransformer import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType @@ -23,12 +24,11 @@ import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf import io.airbyte.protocol.models.v0.DestinationSyncMode import java.sql.Timestamp import java.time.Instant -import java.util.* -import kotlin.Any -import kotlin.Boolean -import kotlin.IllegalArgumentException +import java.util.Locale +import java.util.Optional import kotlin.Int import org.jooq.Condition +import org.jooq.CreateTableColumnStep import org.jooq.DSLContext import org.jooq.DataType import org.jooq.Field @@ -37,6 +37,7 @@ import org.jooq.Name import org.jooq.Record import org.jooq.SQLDialect import org.jooq.SelectConditionStep +import org.jooq.SelectFieldOrAsterisk import org.jooq.conf.ParamType import org.jooq.impl.DSL import org.jooq.impl.SQLDataType @@ -45,7 +46,9 @@ abstract class JdbcSqlGenerator @JvmOverloads constructor( protected val namingTransformer: NamingConventionTransformer, - private val cascadeDrop: Boolean = false + private val cascadeDrop: Boolean = false, + @VisibleForTesting + internal val columns: DestinationColumns = DestinationColumns.V2_WITH_GENERATION, ) : SqlGenerator { protected val cdcDeletedAtColumn: ColumnId = buildColumnId("_ab_cdc_deleted_at") @@ -199,6 +202,9 @@ constructor( SQLDataType.VARCHAR(36).nullable(false) metaColumns[JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT] = timestampWithTimeZoneType.nullable(false) + if (columns == DestinationColumns.V2_WITH_GENERATION) { + metaColumns[JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID] = SQLDataType.BIGINT + } if (includeMetaColumn) metaColumns[JavaBaseConstants.COLUMN_NAME_AB_META] = structType.nullable(false) return metaColumns @@ -332,38 +338,50 @@ constructor( rawTableName: Name, namespace: String, tableName: String - ) = - dslContext - .createTable(rawTableName) - .column( - JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, - SQLDataType.VARCHAR(36).nullable(false), - ) - .column( - JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, - timestampWithTimeZoneType.nullable(false), - ) - .column( - JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, - timestampWithTimeZoneType.nullable(true), - ) - .column(JavaBaseConstants.COLUMN_NAME_DATA, structType.nullable(false)) - .column(JavaBaseConstants.COLUMN_NAME_AB_META, structType.nullable(true)) - .`as`( - DSL.select( - DSL.field(JavaBaseConstants.COLUMN_NAME_AB_ID) - .`as`(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), - DSL.field(JavaBaseConstants.COLUMN_NAME_EMITTED_AT) - .`as`(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), - DSL.cast(null, timestampWithTimeZoneType) - .`as`(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT), - DSL.field(JavaBaseConstants.COLUMN_NAME_DATA) - .`as`(JavaBaseConstants.COLUMN_NAME_DATA), - DSL.cast(null, structType).`as`(JavaBaseConstants.COLUMN_NAME_AB_META), - ) - .from(DSL.table(DSL.name(namespace, tableName))), + ): String { + val hasGenerationId = columns == DestinationColumns.V2_WITH_GENERATION + + val createTable: CreateTableColumnStep = + dslContext + .createTable(rawTableName) + .column( + JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, + SQLDataType.VARCHAR(36).nullable(false), + ) + .column( + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, + timestampWithTimeZoneType.nullable(false), + ) + .column( + JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, + timestampWithTimeZoneType.nullable(true), + ) + .column(JavaBaseConstants.COLUMN_NAME_DATA, structType.nullable(false)) + .column(JavaBaseConstants.COLUMN_NAME_AB_META, structType.nullable(true)) + if (hasGenerationId) { + createTable.column(JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID, SQLDataType.BIGINT) + } + + val selectColumns: MutableList = + mutableListOf( + DSL.field(JavaBaseConstants.COLUMN_NAME_AB_ID) + .`as`(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), + DSL.field(JavaBaseConstants.COLUMN_NAME_EMITTED_AT) + .`as`(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT), + DSL.cast(null, timestampWithTimeZoneType) + .`as`(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT), + DSL.field(JavaBaseConstants.COLUMN_NAME_DATA) + .`as`(JavaBaseConstants.COLUMN_NAME_DATA), + DSL.cast(null, structType).`as`(JavaBaseConstants.COLUMN_NAME_AB_META), ) + if (hasGenerationId) { + selectColumns += DSL.value(0).`as`(JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID) + } + + return createTable + .`as`(DSL.select(selectColumns).from(DSL.table(DSL.name(namespace, tableName)))) .getSQL(ParamType.INLINED) + } override fun clearLoadedAt(streamId: StreamId): Sql { return of( diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.kt index 40406c36cdeb..c8350282cfbc 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.kt @@ -7,12 +7,14 @@ import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.db.jdbc.JdbcDatabase import io.airbyte.cdk.integrations.base.JavaBaseConstants import io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT +import io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID import io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID import io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT import io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META import io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID import io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA import io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT +import io.airbyte.cdk.integrations.base.JavaBaseConstants.DestinationColumns import io.airbyte.cdk.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType @@ -90,7 +92,7 @@ abstract class JdbcSqlGeneratorIntegrationTest) { insertRecords( DSL.name(streamId.rawNamespace, streamId.rawName), - JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES, + sqlGenerator.columns.rawColumns, records, COLUMN_NAME_DATA, COLUMN_NAME_AB_META @@ -143,9 +147,12 @@ abstract class JdbcSqlGeneratorIntegrationTest, generationId: Long, ) { - // TODO handle generation ID val columnNames = - if (includeCdcDeletedAt) FINAL_TABLE_COLUMN_NAMES_CDC else FINAL_TABLE_COLUMN_NAMES + (if (includeCdcDeletedAt) FINAL_TABLE_COLUMN_NAMES_CDC else FINAL_TABLE_COLUMN_NAMES) + .toMutableList() + if (sqlGenerator.columns == DestinationColumns.V2_WITH_GENERATION) { + columnNames += COLUMN_NAME_AB_GENERATION_ID + } insertRecords( DSL.name(streamId.finalNamespace, streamId.finalName + suffix), columnNames, diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.kt b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.kt index 59e7efbb038f..ce424a4254bf 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.kt @@ -80,20 +80,21 @@ class AirbyteDebeziumHandler( cdcSavedInfoFetcher.savedOffset, if (addDbNameToOffsetState) Optional.ofNullable(config[JdbcUtils.DATABASE_KEY].asText()) - else Optional.empty() + else Optional.empty(), ) val schemaHistoryManager: Optional = if (trackSchemaHistory) Optional.of( AirbyteSchemaHistoryStorage.Companion.initializeDBHistory( cdcSavedInfoFetcher.savedSchemaHistory, - cdcStateHandler.compressSchemaHistoryForState() - ) + cdcStateHandler.compressSchemaHistoryForState(), + ), ) else Optional.empty() val publisher = DebeziumRecordPublisher(debeziumPropertiesManager) val queue: CapacityReportingBlockingQueue> = CapacityReportingBlockingQueue(queueSize) + publisher.start(queue, offsetManager, schemaHistoryManager) // handle state machine around pub/sub logic. val eventIterator: AutoCloseableIterator = @@ -102,13 +103,14 @@ class AirbyteDebeziumHandler( targetPosition, { publisher.hasClosed() }, DebeziumShutdownProcedure(queue, { publisher.close() }, { publisher.hasClosed() }), - firstRecordWaitTime + firstRecordWaitTime, + config ) val syncCheckpointDuration = if (config.has(DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY)) Duration.ofSeconds( - config[DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY].asLong() + config[DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY].asLong(), ) else DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION val syncCheckpointRecords = @@ -122,7 +124,7 @@ class AirbyteDebeziumHandler( targetPosition, eventConverter, offsetManager, - schemaHistoryManager + schemaHistoryManager, ) // Usually sourceStateIterator requires airbyteStream as input. For DBZ iterator, stream is @@ -133,7 +135,7 @@ class AirbyteDebeziumHandler( eventIterator, null, messageProducer, - StateEmitFrequency(syncCheckpointRecords, syncCheckpointDuration) + StateEmitFrequency(syncCheckpointRecords, syncCheckpointDuration), ) return AutoCloseableIterators.fromIterator(iterator) } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.kt b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.kt index 9822c298d122..1e2389b43cbf 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.kt @@ -3,8 +3,11 @@ */ package io.airbyte.cdk.integrations.debezium.internals +import com.fasterxml.jackson.databind.JsonNode import com.google.common.annotations.VisibleForTesting import com.google.common.collect.AbstractIterator +import io.airbyte.cdk.db.DbAnalyticsUtils.debeziumCloseReasonMessage +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility import io.airbyte.cdk.integrations.debezium.CdcTargetPosition import io.airbyte.commons.lang.MoreBooleans import io.airbyte.commons.util.AutoCloseableIterator @@ -36,6 +39,7 @@ class DebeziumRecordIterator( private val publisherStatusSupplier: Supplier, private val debeziumShutdownProcedure: DebeziumShutdownProcedure>, private val firstRecordWaitTime: Duration, + private val config: JsonNode ) : AbstractIterator(), AutoCloseableIterator { private val heartbeatEventSourceField: MutableMap?>, Field?> = HashMap(1) @@ -82,7 +86,8 @@ class DebeziumRecordIterator( String.format( "No records were returned by Debezium in the timeout seconds %s, closing the engine and iterator", waitTime.seconds - ) + ), + DebeziumCloseReason.TIMEOUT ) } LOGGER.info { "no record found. polling again." } @@ -101,12 +106,16 @@ class DebeziumRecordIterator( // too long if (targetPosition.reachedTargetPosition(heartbeatPos)) { requestClose( - "Closing: Heartbeat indicates sync is done by reaching the target position" + "Closing: Heartbeat indicates sync is done by reaching the target position", + DebeziumCloseReason.HEARTBEAT_REACHED_TARGET_POSITION ) } else if ( heartbeatPos == this.lastHeartbeatPosition && heartbeatPosNotChanging() ) { - requestClose("Closing: Heartbeat indicates sync is not progressing") + requestClose( + "Closing: Heartbeat indicates sync is not progressing", + DebeziumCloseReason.HEARTBEAT_NOT_PROGRESSING + ) } if (heartbeatPos != lastHeartbeatPosition) { @@ -122,7 +131,10 @@ class DebeziumRecordIterator( // if the last record matches the target file position, it is time to tell the producer // to shutdown. if (targetPosition.reachedTargetPosition(changeEventWithMetadata)) { - requestClose("Closing: Change event reached target position") + requestClose( + "Closing: Change event reached target position", + DebeziumCloseReason.CHANGE_EVENT_REACHED_TARGET_POSITION + ) } this.tsLastHeartbeat = null this.receivedFirstRecord = true @@ -175,7 +187,7 @@ class DebeziumRecordIterator( */ @Throws(Exception::class) override fun close() { - requestClose("Closing: Iterator closing") + requestClose("Closing: Iterator closing", DebeziumCloseReason.ITERATOR_CLOSE) } private fun isHeartbeatEvent(event: ChangeEvent): Boolean { @@ -185,23 +197,25 @@ class DebeziumRecordIterator( } private fun heartbeatPosNotChanging(): Boolean { - if (this.tsLastHeartbeat == null) { + // Closing debezium due to heartbeat position not changing only exists as an escape hatch + // for + // testing setups. In production, we rely on the platform heartbeats to kill the sync + if (!isTest() || this.tsLastHeartbeat == null) { return false } val timeElapsedSinceLastHeartbeatTs = Duration.between(this.tsLastHeartbeat, LocalDateTime.now()) - LOGGER.info { - "Time since last hb_pos change ${timeElapsedSinceLastHeartbeatTs.toSeconds()}s" - } - // wait time for no change in heartbeat position is half of initial waitTime return timeElapsedSinceLastHeartbeatTs.compareTo(firstRecordWaitTime.dividedBy(2)) > 0 } - private fun requestClose(closeLogMessage: String) { + private fun requestClose(closeLogMessage: String, closeReason: DebeziumCloseReason) { if (signalledDebeziumEngineShutdown) { return } LOGGER.info { closeLogMessage } + AirbyteTraceMessageUtility.emitAnalyticsTrace( + debeziumCloseReasonMessage(closeReason.toString()) + ) debeziumShutdownProcedure.initiateShutdownProcedure() signalledDebeziumEngineShutdown = true } @@ -212,6 +226,10 @@ class DebeziumRecordIterator( } } + private fun isTest(): Boolean { + return config.has("is_test") && config["is_test"].asBoolean() + } + /** * [DebeziumRecordIterator.heartbeatEventSourceField] acts as a cache so that we avoid using * reflection to setAccessible for each event @@ -246,5 +264,13 @@ class DebeziumRecordIterator( } } + enum class DebeziumCloseReason() { + TIMEOUT, + ITERATOR_CLOSE, + HEARTBEAT_REACHED_TARGET_POSITION, + CHANGE_EVENT_REACHED_TARGET_POSITION, + HEARTBEAT_NOT_PROGRESSING + } + companion object {} } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIteratorTest.kt b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIteratorTest.kt index 3370d83405f9..372dc05b8637 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIteratorTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIteratorTest.kt @@ -3,6 +3,8 @@ */ package io.airbyte.cdk.integrations.debezium.internals +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper import io.airbyte.cdk.integrations.debezium.CdcTargetPosition import io.debezium.engine.ChangeEvent import java.time.Duration @@ -34,6 +36,7 @@ class DebeziumRecordIteratorTest { { false }, mock(), Duration.ZERO, + getTestConfig(), // Heartbeats should not be ignored for tests. ) val lsn = debeziumRecordIterator.getHeartbeatPosition( @@ -44,7 +47,7 @@ class DebeziumRecordIteratorTest { Collections.singletonMap("lsn", 358824993496L), null, null, - null + null, ) override fun key(): String? { @@ -62,9 +65,15 @@ class DebeziumRecordIteratorTest { fun sourceRecord(): SourceRecord { return sourceRecord } - } + }, ) Assertions.assertEquals(lsn, 358824993496L) } + + fun getTestConfig(): JsonNode { + val mapper: ObjectMapper = ObjectMapper() + val testConfig = "{\"is_test\": true}" + return mapper.readTree(testConfig) + } } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/kotlin/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/kotlin/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.kt index 553ff6c162f3..aa16052c071f 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/kotlin/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.kt +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/kotlin/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.kt @@ -135,7 +135,7 @@ constructor( @VisibleForTesting fun toStreamConfig(stream: ConfiguredAirbyteStream): StreamConfig { - if (stream.generationId == null) { + if (stream.generationId == null || stream.minimumGenerationId == null) { throw ConfigErrorException( "You must upgrade your platform version to use this connector version. Either downgrade your connector or upgrade platform to 0.63.0" ) diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.kt index e696cb4803b4..afd75dde9758 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.kt @@ -40,8 +40,6 @@ import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.function.Executable import org.junit.jupiter.api.parallel.Execution import org.junit.jupiter.api.parallel.ExecutionMode -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.EnumSource private val LOGGER = KotlinLogging.logger {} /** @@ -230,15 +228,11 @@ abstract class BaseTypingDedupingTest { * Starting with an empty destination, execute a full refresh overwrite sync. Verify that the * records are written to the destination table. Then run a second sync, and verify that the * records are overwritten. - * - * Parameterized on destination sync mode. After the refreshes project, APPEND and OVERWRITE - * behave identically. */ - @ParameterizedTest - @EnumSource(DestinationSyncMode::class, names = ["APPEND", "OVERWRITE"]) @Throws(Exception::class) - fun truncateRefresh() { - val catalog = + @Test + open fun truncateRefresh() { + val catalog1 = io.airbyte.protocol.models.v0 .ConfiguredAirbyteCatalog() .withStreams( @@ -247,8 +241,8 @@ abstract class BaseTypingDedupingTest { .withSyncId(42) .withGenerationId(43) .withMinimumGenerationId(43) + .withDestinationSyncMode(DestinationSyncMode.APPEND) .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) .withStream( AirbyteStream() .withNamespace(streamNamespace) @@ -261,7 +255,7 @@ abstract class BaseTypingDedupingTest { // First sync val messages1 = readMessages("dat/sync1_messages.jsonl") - runSync(catalog, messages1) + runSync(catalog1, messages1) val expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl") val expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl") @@ -269,13 +263,34 @@ abstract class BaseTypingDedupingTest { // Second sync val messages2 = readMessages("dat/sync2_messages.jsonl") + val catalog2 = + io.airbyte.protocol.models.v0 + .ConfiguredAirbyteCatalog() + .withStreams( + java.util.List.of( + ConfiguredAirbyteStream() + .withSyncId(42) + .withGenerationId(44) + .withMinimumGenerationId(44) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withSyncMode(SyncMode.FULL_REFRESH) + .withStream( + AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA) + ) + ) + ) - runSync(catalog, messages2) + runSync(catalog2, messages2) val expectedRawRecords2 = - readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl") + readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_raw.jsonl") val expectedFinalRecords2 = - readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl") + readRecords( + "dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_final.jsonl" + ) verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()) } @@ -283,14 +298,11 @@ abstract class BaseTypingDedupingTest { * Starting with an empty destination, execute a full refresh append sync. Verify that the * records are written to the destination table. Then run a second sync, and verify that the old * and new records are all present. - * - * Similar to [truncateRefresh], this is parameterized on sync mode. */ - @ParameterizedTest - @EnumSource(DestinationSyncMode::class, names = ["APPEND", "OVERWRITE"]) @Throws(Exception::class) - fun mergeRefresh() { - val catalog = + @Test + open fun mergeRefresh() { + val catalog1 = io.airbyte.protocol.models.v0 .ConfiguredAirbyteCatalog() .withStreams( @@ -299,8 +311,8 @@ abstract class BaseTypingDedupingTest { .withSyncId(42) .withGenerationId(43) .withMinimumGenerationId(0) - .withSyncMode(SyncMode.FULL_REFRESH) .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withSyncMode(SyncMode.FULL_REFRESH) .withStream( AirbyteStream() .withNamespace(streamNamespace) @@ -313,7 +325,7 @@ abstract class BaseTypingDedupingTest { // First sync val messages1 = readMessages("dat/sync1_messages.jsonl") - runSync(catalog, messages1) + runSync(catalog1, messages1) val expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl") val expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl") @@ -321,12 +333,31 @@ abstract class BaseTypingDedupingTest { // Second sync val messages2 = readMessages("dat/sync2_messages.jsonl") + val catalog2 = + io.airbyte.protocol.models.v0 + .ConfiguredAirbyteCatalog() + .withStreams( + java.util.List.of( + ConfiguredAirbyteStream() + .withSyncId(42) + .withGenerationId(44) + .withMinimumGenerationId(0) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withSyncMode(SyncMode.FULL_REFRESH) + .withStream( + AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA) + ) + ) + ) - runSync(catalog, messages2) + runSync(catalog2, messages2) - val expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl") + val expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_with_new_gen_id_raw.jsonl") val expectedFinalRecords2 = - readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl") + readRecords("dat/sync2_expectedrecords_fullrefresh_append_with_new_gen_id_final.jsonl") verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()) } @@ -1024,6 +1055,7 @@ abstract class BaseTypingDedupingTest { disableFinalTableComparison: Boolean ) { val actualRawRecords = dumpRawTableRecords(streamNamespace, streamName) + if (disableFinalTableComparison) { DIFFER!!.diffRawTableRecords(expectedRawRecords, actualRawRecords) } else { diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py index bec0cf5b1c4f..9406b8e5488f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py @@ -107,7 +107,7 @@ def sleep_on_ratelimit(details: Mapping[str, Any]) -> None: def log_give_up(details: Mapping[str, Any]) -> None: _, exc, _ = sys.exc_info() if isinstance(exc, RequestException): - logger.error(f"Max retry limit reached. Request: {exc.request}, Response: {exc.response}") + logger.error(f"Max retry limit reached in {details['elapsed']}s. Request: {exc.request}, Response: {exc.response}") else: logger.error("Max retry limit reached for unknown request and response") diff --git a/airbyte-cdk/python/unit_tests/conftest.py b/airbyte-cdk/python/unit_tests/conftest.py new file mode 100644 index 000000000000..a5883fe095a5 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/conftest.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import datetime + +import freezegun +import pytest + + +@pytest.fixture() +def mock_sleep(monkeypatch): + with freezegun.freeze_time(datetime.datetime.now(), ignore=['_pytest.runner', '_pytest.terminal']) as frozen_datetime: + monkeypatch.setattr('time.sleep', lambda x: frozen_datetime.tick(x)) + yield diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index d8f219053ea4..77e94778ea2f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -649,6 +649,7 @@ def test_join_url(test_name, base_url, path, expected_full_url): assert sent_request.url == expected_full_url +@pytest.mark.usefixtures("mock_sleep") def test_request_attempt_count_is_tracked_across_retries(http_requester_factory): request_mock = MagicMock(spec=requests.PreparedRequest) request_mock.headers = {} @@ -669,6 +670,7 @@ def test_request_attempt_count_is_tracked_across_retries(http_requester_factory) assert http_requester._http_client._request_attempt_count.get(request_mock) == http_requester._http_client._max_retries + 1 +@pytest.mark.usefixtures("mock_sleep") def test_request_attempt_count_with_exponential_backoff_strategy(http_requester_factory): request_mock = MagicMock(spec=requests.PreparedRequest) request_mock.headers = {} diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 8af1199dea7e..50bd3d8faf7a 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -222,6 +222,7 @@ def test_parse_refresh_token_lifespan( assert isinstance(expires_datetime, pendulum.DateTime) assert ("access_token", expected_token_expiry_date) == (token, expires_datetime) + @pytest.mark.usefixtures("mock_sleep") @pytest.mark.parametrize("error_code", (429, 500, 502, 504)) def test_refresh_access_token_retry(self, error_code, requests_mock): oauth = Oauth2Authenticator( diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py index 9bf2f080a8b3..77c3562d926a 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py @@ -265,6 +265,7 @@ def test_raises_backoff_exception_with_response_with_unmapped_error(mocker, back http_client._send(prepared_request, {}) +@pytest.mark.usefixtures("mock_sleep") def test_send_request_given_retry_response_action_retries_and_returns_valid_response(): mocked_session = MagicMock(spec=requests.Session) valid_response = MagicMock(spec=requests.Response) @@ -361,8 +362,8 @@ def test_send_handles_response_action_given_session_send_raises_request_exceptio assert e.failure_type == error_resolution.failure_type +@pytest.mark.usefixtures("mock_sleep") def test_send_request_given_request_exception_and_retry_response_action_retries_and_returns_valid_response(): - mocked_session = MagicMock(spec=requests.Session) def update_response(*args, **kwargs): @@ -414,8 +415,8 @@ def backoff_time(self, *args, **kwargs): assert mocked_send.call_count == 1 +@pytest.mark.usefixtures("mock_sleep") def test_default_max_retries(): - class BackoffStrategy: def backoff_time(self, *args, **kwargs): return 0.001 @@ -435,8 +436,8 @@ def backoff_time(self, *args, **kwargs): assert mocked_send.call_count == 6 +@pytest.mark.usefixtures("mock_sleep") def test_backoff_strategy_max_retries(): - class BackoffStrategy: def backoff_time(self, *args, **kwargs): return 0.001 @@ -458,6 +459,7 @@ def backoff_time(self, *args, **kwargs): assert mocked_send.call_count == retries + 1 +@pytest.mark.usefixtures("mock_sleep") def test_backoff_strategy_max_time(): error_handler = HttpStatusErrorHandler(logger=MagicMock(), error_mapping={requests.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}, max_retries=10, max_time=timedelta(seconds=2)) diff --git a/airbyte-cdk/python/unit_tests/utils/test_rate_limiting.py b/airbyte-cdk/python/unit_tests/utils/test_rate_limiting.py index d1ed294b930d..bc9d3ece61b8 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_rate_limiting.py +++ b/airbyte-cdk/python/unit_tests/utils/test_rate_limiting.py @@ -11,6 +11,7 @@ def helper_with_exceptions(exception_type): raise exception_type +@pytest.mark.usefixtures("mock_sleep") @pytest.mark.parametrize( "max_tries, max_time, factor, exception_to_raise", [ diff --git a/airbyte-ci/connectors/live-tests/README.md b/airbyte-ci/connectors/live-tests/README.md index 52e9ab4831e3..3f32a64dcdf4 100644 --- a/airbyte-ci/connectors/live-tests/README.md +++ b/airbyte-ci/connectors/live-tests/README.md @@ -280,6 +280,10 @@ The traffic recorded on the control connector is passed to the target connector ## Changelog +### 0.17.5 + +Performance improvements using caching. + ### 0.17.4 Fix control image when running tests in CI. diff --git a/airbyte-ci/connectors/live-tests/pyproject.toml b/airbyte-ci/connectors/live-tests/pyproject.toml index 8f548872a626..76b760ac923a 100644 --- a/airbyte-ci/connectors/live-tests/pyproject.toml +++ b/airbyte-ci/connectors/live-tests/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "live-tests" -version = "0.17.4" +version = "0.17.5" description = "Contains utilities for testing connectors against live data." authors = ["Airbyte "] license = "MIT" diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py index 9062661317d3..bd59825ed767 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py @@ -8,6 +8,7 @@ from collections.abc import Iterable, Iterator, MutableMapping from dataclasses import dataclass, field from enum import Enum +from functools import cache from pathlib import Path from typing import Any, Dict, List, Optional @@ -348,6 +349,7 @@ def get_status_messages_per_stream(self, stream: str) -> Dict[str, List[AirbyteS statuses[message.trace.stream_status.stream_descriptor.name].append(message.trace.stream_status) return statuses + @cache def get_message_count_per_type(self) -> dict[AirbyteMessageType, int]: message_count: dict[AirbyteMessageType, int] = defaultdict(int) for message in self.airbyte_messages: @@ -440,6 +442,9 @@ def update_configuration(self) -> None: self.logger.error(f"Response: {response.text}") self.logger.info(f"Updated configuration for {self.connector_under_test.name}, actor {self.actor_id}") + def __hash__(self): + return hash(self.connector_under_test.version) + @dataclass(kw_only=True) class ConnectionObjects: diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/report.py b/airbyte-ci/connectors/live-tests/src/live_tests/report.py index 741b39921d68..6d1da72984ca 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/report.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/report.py @@ -8,6 +8,7 @@ from collections.abc import Iterable, MutableMapping from copy import deepcopy from enum import Enum +from functools import cache from pathlib import Path from typing import TYPE_CHECKING, Any, Optional @@ -170,13 +171,17 @@ def get_record_count_per_stream( stream_schemas: Iterable = result.stream_schemas or [] for stream in stream_schemas: - per_stream_count[stream][source] = sum(1 for _ in result.get_records_per_stream(stream)) # type: ignore + per_stream_count[stream][source] = self._get_record_count_for_stream(result, stream) for stream in per_stream_count: per_stream_count[stream]["difference"] = per_stream_count[stream]["target"] - per_stream_count[stream]["control"] record_count_per_command_and_stream[control_result.command] = per_stream_count # type: ignore return record_count_per_command_and_stream + @cache + def _get_record_count_for_stream(self, result: ExecutionResult, stream: str) -> int: + return sum(1 for _ in result.get_records_per_stream(stream)) # type: ignore + def get_untested_streams(self) -> list[str]: streams_with_data: set[str] = set() for stream_count in self.get_record_count_per_stream().values(): diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md index c5fa6e820878..b9bd966b4794 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 3.8.3 + +Add handling for global state messages (for use by DB sources). + +## 3.8.2 + +Allow tests to access the setup/teardown container. + +## 3.8.1 + +Provide the ability for users to perform setup/teardown by building and running a container before each test. + ## 3.8.0 Add `TestDiscovery.test_primary_keys_data_type`, which validates that primary keys are not of type `array` or `object` in discovered catalog. diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index d38fe9a167dc..2b044bd355c2 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -158,12 +158,13 @@ def no_unsupported_types_when_skip_test(cls, skip_test: bool, values: Dict[str, return skip_test -class SetupTeardownConfig(BaseConfig): - setup_teardown_dockerfile_path: str = Field( +class ClientContainerConfig(BaseConfig): + client_container_dockerfile_path: str = Field( None, description="Path to Dockerfile to run before each test for which a config is provided." ) - setup_command: List[str] = Field(None, description="Command for running the setup/teardown container for setup") - teardown_command: List[str] = Field(None, description="Command for running the setup/teardown container for teardown") + setup_command: List[str] = Field(None, description="Command for running the setup/teardown container for setup.") + teardown_command: List[str] = Field(None, description="Command for running the setup/teardown container for teardown.") + between_syncs_command: Optional[List[str]] = Field(None, description="Command to run between syncs that occur in a test.") class BasicReadTestConfig(BaseConfig): @@ -189,9 +190,8 @@ class BasicReadTestConfig(BaseConfig): default_factory=FileTypesConfig, description="For file-based connectors, unsupported by source file types can be configured or a test can be skipped at all", ) - setup_teardown_config: Optional[SetupTeardownConfig] = Field( - default_factory=SetupTeardownConfig, - description="Information required to run a setup & teardown Docker container before each test.", + client_container_config: Optional[ClientContainerConfig] = Field( + description="Information required to run a client Docker container before each test.", ) @@ -209,9 +209,8 @@ class FullRefreshConfig(BaseConfig): ignored_fields: Optional[Mapping[str, List[IgnoredFieldsConfiguration]]] = Field( description="For each stream, list of fields path ignoring in sequential reads test" ) - setup_teardown_config: Optional[SetupTeardownConfig] = Field( - default_factory=SetupTeardownConfig, - description="Information required to run a setup & teardown Docker container before each test.", + client_container_config: Optional[ClientContainerConfig] = Field( + description="Information required to run a client Docker container before each test.", ) @@ -249,9 +248,8 @@ class IncrementalConfig(BaseConfig): skip_comprehensive_incremental_tests: Optional[bool] = Field( description="Determines whether to skip more granular testing for incremental syncs", default=False ) - setup_teardown_config: Optional[SetupTeardownConfig] = Field( - default_factory=SetupTeardownConfig, - description="Information required to run a setup & teardown Docker container before each test.", + client_container_config: Optional[ClientContainerConfig] = Field( + description="Information required to run a client Docker container before each test.", ) class Config: diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py index 6c10bbe24550..d38543aa06fd 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py @@ -20,22 +20,22 @@ from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConnectorSpecification, Type from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import ( + ClientContainerConfig, Config, EmptyStreamConfiguration, ExpectedRecordsConfig, IgnoredFieldsConfiguration, - SetupTeardownConfig, ) from connector_acceptance_test.tests import TestBasicRead from connector_acceptance_test.utils import ( SecretDict, build_configured_catalog_from_custom_catalog, build_configured_catalog_from_discovered_catalog_and_empty_streams, + client_container_runner, connector_runner, filter_output, load_config, load_yaml_or_json_path, - setup_and_teardown_runner, ) @@ -131,11 +131,11 @@ def connector_config_fixture(base_path, connector_config_path) -> SecretDict: return SecretDict(json.loads(contents)) -@pytest.fixture(name="setup_teardown_dockerfile_config") -def setup_teardown_dockerfile_config_fixture(inputs, base_path, acceptance_test_config) -> Optional[SetupTeardownConfig]: +@pytest.fixture(name="client_container_config") +def client_container_config_fixture(inputs, base_path, acceptance_test_config) -> Optional[ClientContainerConfig]: """Fixture with connector's setup/teardown Dockerfile path, if it exists.""" - if hasattr(inputs, "setup_teardown_config"): - return getattr(inputs, "setup_teardown_config") + if hasattr(inputs, "setup_teardown_config") and inputs.setup_teardown_config: + return inputs.setup_teardown_config @pytest.fixture(name="invalid_connector_config") @@ -201,30 +201,42 @@ def docker_runner_fixture( @pytest.fixture(autouse=True) -async def setup_and_teardown( +async def client_container( base_path: Path, connector_config: SecretDict, dagger_client: dagger.Client, - setup_teardown_dockerfile_config: Optional[SetupTeardownConfig], + client_container_config: Optional[ClientContainerConfig], +) -> Optional[dagger.Container]: + if client_container_config: + return await client_container_runner.get_client_container( + dagger_client, + base_path, + base_path / client_container_config.client_dockerfile_path, + ) + + +@pytest.fixture(autouse=True) +async def setup_and_teardown( + client_container: dagger.Container, + connector_config: SecretDict, + client_container_config: Optional[ClientContainerConfig], ): - setup_teardown_container = None - if setup_teardown_dockerfile_config: + if client_container: logging.info("Running setup") - setup_teardown_container = await setup_and_teardown_runner.do_setup( - dagger_client, - base_path / setup_teardown_dockerfile_config.setup_teardown_dockerfile_path, - setup_teardown_dockerfile_config.setup_command, + setup_teardown_container = await client_container_runner.do_setup( + client_container, + client_container_config.setup_command, connector_config, ) - print(f">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> {await setup_teardown_container.stdout()}") + logging.info(f"Setup stdout: {await setup_teardown_container.stdout()}") yield None - if setup_teardown_container: + if client_container: logging.info("Running teardown") - setup_teardown_container = await setup_and_teardown_runner.do_teardown( - setup_teardown_container, - setup_teardown_dockerfile_config.teardown_command, + setup_teardown_container = await client_container_runner.do_teardown( + client_container, + client_container_config.teardown_command, ) - print(f">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> {await setup_teardown_container.stdout()}") + logging.info(f"Teardown stdout: {await setup_teardown_container.stdout()}") @pytest.fixture(name="previous_connector_image_name") diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py index 961e2f156c16..a5c6c44e365d 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py @@ -4,9 +4,12 @@ import json import re +from logging import Logger from pathlib import Path from typing import Any, Callable, Dict, List, Mapping, MutableMapping, Optional, Tuple, Union +from uuid import uuid4 +import dagger import pytest from airbyte_protocol.models import ( AirbyteMessage, @@ -19,7 +22,7 @@ Type, ) from connector_acceptance_test import BaseTest -from connector_acceptance_test.config import Config, EmptyStreamConfiguration, IncrementalConfig +from connector_acceptance_test.config import ClientContainerConfig, Config, EmptyStreamConfiguration, IncrementalConfig from connector_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, incremental_only_catalog from connector_acceptance_test.utils.timeouts import TWENTY_MINUTES from deepdiff import DeepDiff @@ -99,6 +102,10 @@ def is_per_stream_state(message: AirbyteMessage) -> bool: return message.state and isinstance(message.state, AirbyteStateMessage) and message.state.type == AirbyteStateType.STREAM +def is_global_state(message: AirbyteMessage) -> bool: + return message.state and isinstance(message.state, AirbyteStateMessage) and message.state.type == AirbyteStateType.GLOBAL + + def construct_latest_state_from_messages(messages: List[AirbyteMessage]) -> Dict[str, Mapping[str, Any]]: """ Because connectors that have migrated to per-stream state only emit state messages with the new state value for a single @@ -132,6 +139,9 @@ async def test_two_sequential_reads( connector_config: SecretDict, configured_catalog_for_incremental: ConfiguredAirbyteCatalog, docker_runner: ConnectorRunner, + client_container: Optional[dagger.Container], + client_container_config: Optional[ClientContainerConfig], + detailed_logger: Logger, ): """ This test makes two calls to the read method and verifies that the records returned are different. @@ -154,8 +164,7 @@ async def test_two_sequential_reads( # For legacy state format, the final state message contains the final state of all streams. For per-stream state format, # the complete final state of streams must be assembled by going through all prior state messages received - is_per_stream = is_per_stream_state(states_1[-1]) - if is_per_stream: + if is_per_stream_state(states_1[-1]): latest_state = construct_latest_state_from_messages(states_1) state_input = [] for stream_name, stream_state in latest_state.items(): @@ -168,10 +177,19 @@ async def test_two_sequential_reads( "stream": {"stream_descriptor": stream_descriptor, "stream_state": stream_state}, } ) + elif is_global_state(states_1[-1]): + # TODO: DB sources to fill out this case + state_input = states_1[-1].state.data else: state_input = states_1[-1].state.data # READ #2 + if client_container and client_container_config.between_syncs_command: + detailed_logger.info( + await client_container.with_env_variable("CACHEBUSTER", str(uuid4())) + .with_exec(client_container_config.between_syncs_command, skip_entrypoint=True) + .stdout() + ) output_2 = await docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) records_2 = filter_output(output_2, type_=Type.RECORD) @@ -182,7 +200,14 @@ async def test_two_sequential_reads( ), f"Records should change between reads but did not.\n\n records_1: {records_1} \n\n state: {state_input} \n\n records_2: {records_2} \n\n diff: {diff}" async def test_read_sequential_slices( - self, inputs: IncrementalConfig, connector_config, configured_catalog_for_incremental, docker_runner: ConnectorRunner + self, + inputs: IncrementalConfig, + connector_config, + configured_catalog_for_incremental, + docker_runner: ConnectorRunner, + client_container: Optional[dagger.Container], + client_container_config: Optional[ClientContainerConfig], + detailed_logger: Logger, ): """ Incremental test that makes calls to the read method without a state checkpoint. Then we partition the results by stream and @@ -227,6 +252,13 @@ async def test_read_sequential_slices( state_message, mutating_stream_name_to_per_stream_state ) + if client_container and client_container_config.between_syncs_command: + detailed_logger.info( + await client_container.with_env_variable("CACHEBUSTER", str(uuid4())) + .with_exec(client_container_config.between_syncs_command, skip_entrypoint=True) + .stdout() + ) + output_N = await docker_runner.call_read_with_state( connector_config, configured_catalog_for_incremental_per_stream, state=state_input ) @@ -262,6 +294,11 @@ async def test_state_with_abnormally_large_values( ), f"The sync should produce no records when run with the state with abnormally large values {records[0].record.stream}" assert states, "The sync should produce at least one STATE message" + if states and is_global_state(states[0]): + # TODO: DB sources to fill out this case. Also, can we assume all states will be global if the first one is? + pass + + # TODO: else: cursor_fields_per_stream = { stream.stream.name: self._get_cursor_field(stream) for stream in configured_catalog.streams @@ -351,6 +388,9 @@ def get_next_state_input( stream_name_to_per_stream_state[per_stream.stream_descriptor.name] = ( per_stream.stream_state.dict() if per_stream.stream_state else {} ) + elif current_state and current_state.type == AirbyteStateType.GLOBAL: + # TODO: DB Sources to fill in this case + pass state_input = [ {"type": "STREAM", "stream": {"stream_descriptor": {"name": stream_name}, "stream_state": stream_state}} for stream_name, stream_state in stream_name_to_per_stream_state.items() diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py new file mode 100644 index 000000000000..0b0f8c3fdf39 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import os +from glob import glob +from pathlib import Path +from typing import List + +import dagger +from connector_acceptance_test.utils import SecretDict + +IN_CONTAINER_CONFIG_PATH = Path("/tmp/config.json") +IN_CONTAINER_OUTPUT_PATH = Path("/tmp/output.txt") + + +async def _build_container(dagger_client: dagger.Client, dockerfile_path: Path) -> dagger.Container: + workspace = ( + dagger_client.container() + .with_mounted_directory("/tmp/setup_teardown_context", dagger_client.host().directory(os.path.dirname(dockerfile_path))) + .directory("/tmp/setup_teardown_context") + ) + return await dagger_client.container().build(context=workspace, dockerfile="Dockerfile.cat_setup_teardown") + + +async def _build_client_container(dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path) -> dagger.Container: + container = await _build_container(dagger_client, dockerfile_path) + return container.with_mounted_directory( + "/connector", dagger_client.host().directory(str(connector_path), exclude=get_default_excluded_files()) + ) + + +def get_default_excluded_files() -> List[str]: + return ( + [".git"] + + glob("**/build", recursive=True) + + glob("**/.venv", recursive=True) + + glob("**/__pycache__", recursive=True) + + glob("**/*.egg-info", recursive=True) + + glob("**/.vscode", recursive=True) + + glob("**/.pytest_cache", recursive=True) + + glob("**/.eggs", recursive=True) + + glob("**/.mypy_cache", recursive=True) + + glob("**/.DS_Store", recursive=True) + + glob("**/.gradle", recursive=True) + ) + + +async def _run_with_config(container: dagger.Container, command: List[str], config: SecretDict) -> dagger.Container: + container = container.with_new_file(str(IN_CONTAINER_CONFIG_PATH), contents=json.dumps(dict(config))) + return await _run(container, command) + + +async def _run(container: dagger.Container, command: List[str]) -> dagger.Container: + return await container.with_exec(command, skip_entrypoint=True) + + +async def get_client_container(dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path): + return await _build_client_container(dagger_client, connector_path, dockerfile_path) + + +async def do_setup(container: dagger.Container, command: List[str], connector_config: SecretDict): + return await _run_with_config(container, command, connector_config) + + +async def do_teardown(container: dagger.Container, command: List[str]): + return await _run(container, command) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/setup_and_teardown_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/setup_and_teardown_runner.py deleted file mode 100644 index 56523b0ed419..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/setup_and_teardown_runner.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -import os -from pathlib import Path -from typing import List - -import dagger -from connector_acceptance_test.utils import SecretDict - -IN_CONTAINER_CONFIG_PATH = Path("/tmp/config.json") -IN_CONTAINER_OUTPUT_PATH = Path("/tmp/output.txt") - - -async def _build_container(dagger_client: dagger.Client, dockerfile_path: Path) -> dagger.Container: - workspace = ( - dagger_client.container() - .with_directory("/tmp", dagger_client.host().directory(os.path.dirname(dockerfile_path))) - .with_workdir("/tmp") - .with_file("/tmp/Dockerfile.cat_setup_teardown", dagger_client.host().file(str(dockerfile_path.expanduser()))) - .directory("/tmp") - ) - return await dagger_client.container().build(context=workspace, dockerfile="Dockerfile.cat_setup_teardown") - - -async def _build_setup_container(dagger_client: dagger.Client, dockerfile_path: Path) -> dagger.Container: - return await _build_container(dagger_client, dockerfile_path) - - -async def _run_with_config(container: dagger.Container, command: List[str], config: SecretDict) -> dagger.Container: - container = container.with_new_file(str(IN_CONTAINER_CONFIG_PATH), contents=json.dumps(dict(config))) - return await _run(container, command) - - -async def _run(container: dagger.Container, command: List[str]) -> dagger.Container: - return await container.with_exec(command, skip_entrypoint=True) - - -async def do_setup(dagger_client: dagger.Client, dockerfile_path: Path, command: List[str], connector_config: SecretDict): - return await _run_with_config(await _build_setup_container(dagger_client, dockerfile_path), command, connector_config) - - -async def do_teardown(container: dagger.Container, command: List[str]): - return await _run(container, command) diff --git a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml index 8a11d0642728..4cc9d34809ea 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml +++ b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector-acceptance-test" -version = "3.7.0" +version = "3.8.3" description = "Contains acceptance tests for connectors." authors = ["Airbyte "] license = "MIT" diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py index e2e89af4debf..05724361dab0 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py @@ -5,6 +5,7 @@ import json import operator from contextlib import nullcontext as does_not_raise +from logging import Logger from pathlib import Path from typing import Any, Optional from unittest.mock import MagicMock, patch @@ -186,6 +187,9 @@ async def test_incremental_two_sequential_reads( connector_config=MagicMock(), configured_catalog_for_incremental=catalog, docker_runner=docker_runner_mock, + client_container=None, + client_container_config=None, + detailed_logger=Logger("test"), ) @@ -399,6 +403,9 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse configured_catalog_for_incremental=catalog, docker_runner=docker_runner_mock, inputs=inputs, + client_container=None, + client_container_config=None, + detailed_logger=Logger("test"), ) @@ -493,6 +500,9 @@ async def test_config_skip_test(mocker): ] ), docker_runner=docker_runner_mock, + client_container=None, + client_container_config=None, + detailed_logger=Logger("test"), ) # This is guaranteed to fail when the test gets executed diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 975ce629bfcd..2da285d20c34 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.35.16' + cdkVersionRequired = '0.38.3' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 67233c3c4f0f..dd69ea37cd1a 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 3.1.1 + dockerImageTag: 3.2.0 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/RedshiftDestination.kt b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/RedshiftDestination.kt index 52f803456194..3792ae154601 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/RedshiftDestination.kt +++ b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/RedshiftDestination.kt @@ -35,7 +35,6 @@ import io.airbyte.cdk.integrations.destination.async.model.PartialAirbyteRecordM import io.airbyte.cdk.integrations.destination.async.state.FlushFailure import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination.Companion.DISABLE_TYPE_DEDUPE import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination.Companion.RAW_SCHEMA_OVERRIDE -import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler import io.airbyte.cdk.integrations.destination.s3.AesCbcEnvelopeEncryption import io.airbyte.cdk.integrations.destination.s3.EncryptionConfig import io.airbyte.cdk.integrations.destination.s3.EncryptionConfig.Companion.fromJson @@ -61,6 +60,7 @@ import io.airbyte.integrations.destination.redshift.constants.RedshiftDestinatio import io.airbyte.integrations.destination.redshift.operation.RedshiftStagingStorageOperation import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftDV2Migration import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftDestinationHandler +import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftGenerationIdMigration import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftRawTableAirbyteMetaMigration import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGenerator import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftState @@ -69,6 +69,7 @@ import io.airbyte.integrations.destination.redshift.util.RedshiftUtil import io.airbyte.protocol.models.v0.AirbyteConnectionStatus import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.AirbyteRecordMessageMeta +import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage.AirbyteStreamStatus import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog import io.airbyte.protocol.models.v0.ConnectorSpecification import io.airbyte.protocol.models.v0.DestinationSyncMode @@ -81,7 +82,6 @@ import java.util.concurrent.Executors import java.util.function.Consumer import javax.sql.DataSource import org.apache.commons.lang3.NotImplementedException -import org.apache.commons.lang3.StringUtils import org.jetbrains.annotations.VisibleForTesting import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -187,7 +187,11 @@ class RedshiftDestination : BaseConnector(), Destination { isSchemaMismatch = true, isFinalTableEmpty = true, destinationState = - RedshiftState(needsSoftReset = false, isAirbyteMetaPresentInRaw = true), + RedshiftState( + needsSoftReset = false, + isAirbyteMetaPresentInRaw = true, + isGenerationIdPresent = true, + ), ), FileUploadFormat.CSV, destinationColumns, @@ -211,7 +215,7 @@ class RedshiftDestination : BaseConnector(), Destination { ) streamOperation.finalizeTable( streamConfig, - StreamSyncSummary(recordsWritten = Optional.of(1)), + StreamSyncSummary(recordsWritten = 1, AirbyteStreamStatus.COMPLETE), ) // And now that we have a table, simulate the next sync startup. @@ -315,14 +319,6 @@ class RedshiftDestination : BaseConnector(), Destination { return RedshiftSqlGenerator(namingResolver, config) } - private fun getDestinationHandler( - databaseName: String, - database: JdbcDatabase, - rawTableSchema: String - ): JdbcDestinationHandler { - return RedshiftDestinationHandler(databaseName, database, rawTableSchema) - } - private fun getMigrations( database: JdbcDatabase, databaseName: String, @@ -336,6 +332,7 @@ class RedshiftDestination : BaseConnector(), Destination { sqlGenerator, ), RedshiftRawTableAirbyteMetaMigration(database, databaseName), + RedshiftGenerationIdMigration(database, databaseName) ) } @@ -372,13 +369,7 @@ class RedshiftDestination : BaseConnector(), Destination { else NoEncryption() val s3Options = RedshiftUtil.findS3Options(config) val s3Config: S3DestinationConfig = S3DestinationConfig.getS3DestinationConfig(s3Options) - val defaultNamespace = config["schema"].asText() - for (stream in catalog.streams) { - if (StringUtils.isEmpty(stream.stream.namespace)) { - stream.stream.namespace = defaultNamespace - } - } val sqlGenerator = RedshiftSqlGenerator(namingResolver, config) val parsedCatalog: ParsedCatalog @@ -388,10 +379,10 @@ class RedshiftDestination : BaseConnector(), Destination { val rawNamespace: String if (getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).isPresent) { rawNamespace = getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).get() - catalogParser = CatalogParser(sqlGenerator, rawNamespace) + catalogParser = CatalogParser(sqlGenerator, defaultNamespace, rawNamespace) } else { rawNamespace = JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE - catalogParser = CatalogParser(sqlGenerator, rawNamespace) + catalogParser = CatalogParser(sqlGenerator, defaultNamespace, rawNamespace) } val redshiftDestinationHandler = RedshiftDestinationHandler(databaseName, database, rawNamespace) @@ -436,8 +427,7 @@ class RedshiftDestination : BaseConnector(), Destination { }, onFlush = DefaultFlush(OPTIMAL_FLUSH_BATCH_SIZE, syncOperation), catalog, - BufferManager(bufferMemoryLimit), - Optional.ofNullable(defaultNamespace), + BufferManager(defaultNamespace, bufferMemoryLimit), FlushFailure(), Executors.newFixedThreadPool(5), AirbyteMessageDeserializer(getDataTransformer(parsedCatalog, defaultNamespace)), @@ -463,7 +453,7 @@ class RedshiftDestination : BaseConnector(), Destination { "com.amazon.redshift.ssl.NonValidatingFactory" ) - private val destinationColumns = JavaBaseConstants.DestinationColumns.V2_WITH_META + private val destinationColumns = JavaBaseConstants.DestinationColumns.V2_WITH_GENERATION private const val OPTIMAL_FLUSH_BATCH_SIZE: Long = 50 * 1024 * 1024 private val bufferMemoryLimit: Long = (Runtime.getRuntime().maxMemory() * 0.5).toLong() diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/operation/RedshiftStagingStorageOperation.kt b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/operation/RedshiftStagingStorageOperation.kt index fd1d80f78ff9..611fa189dea5 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/operation/RedshiftStagingStorageOperation.kt +++ b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/operation/RedshiftStagingStorageOperation.kt @@ -51,7 +51,8 @@ class RedshiftStagingStorageOperation( s3StorageOperations.createBucketIfNotExists() } - override fun writeToStage(streamId: StreamId, data: SerializableBuffer) { + override fun writeToStage(streamConfig: StreamConfig, data: SerializableBuffer) { + val streamId = streamConfig.id val objectPath: String = getStagingPath(streamId) log.info { "Uploading records to for ${streamId.rawNamespace}.${streamId.rawName} to path $objectPath" @@ -201,7 +202,8 @@ class RedshiftStagingStorageOperation( ${JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT} TIMESTAMPTZ DEFAULT GETDATE(), ${JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT} TIMESTAMPTZ, ${JavaBaseConstants.COLUMN_NAME_DATA} SUPER NOT NULL, - ${JavaBaseConstants.COLUMN_NAME_AB_META} SUPER NULL + ${JavaBaseConstants.COLUMN_NAME_AB_META} SUPER NULL, + ${JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID} BIGINT NULL ) """.trimIndent() } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDV2Migration.kt b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDV2Migration.kt index 1c1c48806b11..f2d315080315 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDV2Migration.kt +++ b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDV2Migration.kt @@ -34,6 +34,7 @@ class RedshiftDV2Migration( RedshiftState( needsSoftReset = false, isAirbyteMetaPresentInRaw = false, + isGenerationIdPresent = false ), true, ) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.kt b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.kt index 9c584b20599e..32194aa06321 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.kt +++ b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.kt @@ -90,12 +90,9 @@ class RedshiftDestinationHandler( for (transaction in transactions) { val transactionId = UUID.randomUUID() if (logStatements) { - log.info( - "Executing sql {}-{}: {}", - queryId, - transactionId, - java.lang.String.join("\n", transaction) - ) + log.info { + "Executing sql $queryId-$transactionId: ${transaction.joinToString("\n")}" + } } val startTime = System.currentTimeMillis() @@ -113,7 +110,7 @@ class RedshiftDestinationHandler( logStatements = logStatements ) } catch (e: SQLException) { - log.error("Sql {}-{} failed", queryId, transactionId, e) + log.error(e) { "Sql $queryId-$transactionId failed" } // This is a big hammer for something that should be much more targetted, only when // executing the // DROP TABLE command. @@ -129,12 +126,9 @@ class RedshiftDestinationHandler( throw e } - log.info( - "Sql {}-{} completed in {} ms", - queryId, - transactionId, - System.currentTimeMillis() - startTime - ) + log.info { + "Sql $queryId-$transactionId completed in ${System.currentTimeMillis() - startTime} ms" + } } } @@ -156,7 +150,8 @@ class RedshiftDestinationHandler( return RedshiftState( json.hasNonNull("needsSoftReset") && json["needsSoftReset"].asBoolean(), json.hasNonNull("isAirbyteMetaPresentInRaw") && - json["isAirbyteMetaPresentInRaw"].asBoolean() + json["isAirbyteMetaPresentInRaw"].asBoolean(), + json.hasNonNull("isGenerationIdPresent") && json["isGenerationIdPresent"].asBoolean(), ) } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftGenerationIdMigration.kt b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftGenerationIdMigration.kt new file mode 100644 index 000000000000..3566b36f2130 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftGenerationIdMigration.kt @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping + +import com.fasterxml.jackson.databind.JsonNode +import io.airbyte.cdk.db.jdbc.JdbcDatabase +import io.airbyte.cdk.integrations.base.JavaBaseConstants +import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialStatus +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig +import io.airbyte.integrations.base.destination.typing_deduping.migrators.Migration +import io.github.oshai.kotlinlogging.KotlinLogging + +private val logger = KotlinLogging.logger {} + +class RedshiftGenerationIdMigration( + private val database: JdbcDatabase, + private val databaseName: String, +) : Migration { + override fun migrateIfNecessary( + destinationHandler: DestinationHandler, + stream: StreamConfig, + state: DestinationInitialStatus + ): Migration.MigrationResult { + if (state.destinationState.isGenerationIdPresent) { + logger.info { + "Skipping generation_id migration for ${stream.id.originalNamespace}.${stream.id.originalName} because our state says it's already done" + } + return Migration.MigrationResult(state.destinationState, invalidateInitialState = false) + } + + if (!state.initialRawTableStatus.rawTableExists) { + // The raw table doesn't exist. No migration necessary. Update the state. + logger.info { + "Skipping generation_id migration for ${stream.id.originalNamespace}.${stream.id.originalName} because the raw table doesn't exist" + } + return Migration.MigrationResult( + state.destinationState.copy(isGenerationIdPresent = true), + invalidateInitialState = false + ) + } + + // Add generation_id to the raw table if necessary + val rawTableDefinitionQueryResult: List = + database.queryJsons( + """ + SHOW COLUMNS + FROM TABLE "$databaseName"."${stream.id.rawNamespace}"."${stream.id.rawName}" + LIKE '${JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID}' + """.trimIndent() + ) + if (rawTableDefinitionQueryResult.isNotEmpty()) { + logger.info { + "${stream.id.originalNamespace}.${stream.id.originalName}: Skipping generation_id migration for raw table because it already has the generation_id column" + } + } else { + logger.info { + "Migrating generation_id for table ${stream.id.rawNamespace}.${stream.id.rawName}" + } + // Quote for raw table columns + val alterRawTableSql = + """ + ALTER TABLE "${stream.id.rawNamespace}"."${stream.id.rawName}" + ADD COLUMN "${JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID}" BIGINT; + """.trimIndent() + database.execute(alterRawTableSql) + } + + // Add generation_id to the final table if necessary + // As a slight optimization, only do this if we previously detected that the final table + // schema is wrong + if (state.isFinalTablePresent && state.isSchemaMismatch) { + val finalTableColumnQueryResult: List = + database.queryJsons( + """ + SHOW COLUMNS + FROM TABLE "$databaseName"."${stream.id.finalNamespace}"."${stream.id.finalName}" + LIKE '${JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID}' + """.trimIndent() + ) + if (finalTableColumnQueryResult.isNotEmpty()) { + logger.info { + "${stream.id.originalNamespace}.${stream.id.originalName}: Skipping generation_id migration for final table because it already has the generation_id column" + } + } else { + logger.info { + "Migrating generation_id for table ${stream.id.finalNamespace}.${stream.id.finalName}" + } + database.execute( + """ + ALTER TABLE "${stream.id.finalNamespace}"."${stream.id.finalName}" + ADD COLUMN "${JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID}" BIGINT NULL; + """.trimIndent() + ) + } + } else { + logger.info { + "${stream.id.originalNamespace}.${stream.id.originalName}: Skipping generation_id migration for final table. Final table exists: ${state.isFinalTablePresent}; final table schema is incorrect: ${state.isSchemaMismatch}" + } + } + + return Migration.MigrationResult( + state.destinationState.copy(isGenerationIdPresent = true), + invalidateInitialState = true + ) + } +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.kt b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.kt index d4b77e2174e2..811ad7386c07 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.kt +++ b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.kt @@ -17,7 +17,7 @@ import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf import io.airbyte.integrations.destination.redshift.constants.RedshiftDestinationConstants import io.airbyte.protocol.models.AirbyteRecordMessageMetaChange import java.sql.Timestamp -import java.util.* +import java.util.Optional import java.util.stream.Collectors import org.jooq.Condition import org.jooq.DataType @@ -146,7 +146,7 @@ open class RedshiftSqlGenerator( * @param arrays * @return */ - fun arrayConcatStmt(arrays: List?>): Field<*>? { + private fun arrayConcatStmt(arrays: List?>): Field<*>? { if (arrays.isEmpty()) { return DSL.field("ARRAY()") // Return an empty string if the list is empty } @@ -165,7 +165,7 @@ open class RedshiftSqlGenerator( return result } - fun toCastingErrorCaseStmt(column: ColumnId, type: AirbyteType): Field<*> { + private fun toCastingErrorCaseStmt(column: ColumnId, type: AirbyteType): Field<*> { val field: Field<*> = DSL.field(DSL.quotedName(JavaBaseConstants.COLUMN_NAME_DATA, column.originalName)) // Just checks if data is not null but casted data is null. This also accounts for @@ -260,7 +260,14 @@ open class RedshiftSqlGenerator( "OBJECT", superType, DSL.`val`(AIRBYTE_META_COLUMN_CHANGES_KEY), - airbyteMetaChangesArray + airbyteMetaChangesArray, + DSL.`val`(JavaBaseConstants.AIRBYTE_META_SYNC_ID_KEY), + DSL.field( + DSL.quotedName( + JavaBaseConstants.COLUMN_NAME_AB_META, + JavaBaseConstants.AIRBYTE_META_SYNC_ID_KEY + ) + ), ) .`as`(JavaBaseConstants.COLUMN_NAME_AB_META) } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt index 056b3323046e..0b0b845c16ea 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt +++ b/airbyte-integrations/connectors/destination-redshift/src/main/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt @@ -6,8 +6,11 @@ package io.airbyte.integrations.destination.redshift.typing_deduping import io.airbyte.integrations.base.destination.typing_deduping.migrators.MinimumDestinationState -data class RedshiftState(val needsSoftReset: Boolean, val isAirbyteMetaPresentInRaw: Boolean) : - MinimumDestinationState { +data class RedshiftState( + val needsSoftReset: Boolean, + val isAirbyteMetaPresentInRaw: Boolean, + val isGenerationIdPresent: Boolean, +) : MinimumDestinationState { override fun needsSoftReset(): Boolean { return needsSoftReset } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.kt b/airbyte-integrations/connectors/destination-redshift/src/test-integration/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.kt index e9d8c18e1c63..6f745e418f79 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.kt +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/kotlin/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.kt @@ -7,8 +7,11 @@ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.db.JdbcCompatibleSourceOperations +import io.airbyte.cdk.integrations.base.JavaBaseConstants import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcTypingDedupingTest import io.airbyte.commons.json.Jsons.deserialize +import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest +import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest.Companion import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator import io.airbyte.integrations.destination.redshift.RedshiftDestination import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer @@ -24,6 +27,7 @@ import javax.sql.DataSource import org.jooq.DSLContext import org.jooq.conf.Settings import org.jooq.impl.DSL +import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test @@ -170,6 +174,9 @@ abstract class AbstractRedshiftTypingDedupingTest : JdbcTypingDedupingTest() { .withStreams( List.of( ConfiguredAirbyteStream() + .withSyncId(42) + .withGenerationId(43) + .withMinimumGenerationId(0) .withSyncMode(SyncMode.FULL_REFRESH) .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) .withStream( @@ -201,6 +208,117 @@ abstract class AbstractRedshiftTypingDedupingTest : JdbcTypingDedupingTest() { verifySyncResult(expectedRawRecords, expectedFinalRecords, disableFinalTableComparison()) } + @Test + @Throws(Exception::class) + fun testGenerationIdMigrationForAppend() { + val catalog = + ConfiguredAirbyteCatalog() + .withStreams( + listOf( + ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withSyncId(42L) + .withGenerationId(43L) + .withMinimumGenerationId(0L) + .withStream( + AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA) + ) + ) + ) + + // First sync + val messages1 = readMessages("dat/sync1_messages.jsonl") + runSync( + catalog, + messages1, + "airbyte/destination-redshift:3.1.1", + ) + + // Second sync + val messages2 = readMessages("dat/sync2_messages.jsonl") + runSync(catalog, messages2) + + // The first 5 records in these files were written by the old version, + // which does not write _airbyte_generation_id to the raw/final tables, + // and does not write sync_id to _airbyte_meta. + // So modify the expected records to reflect those differences. + val expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl") + for (i in 0..4) { + val record = expectedRawRecords2[i] as ObjectNode + record.remove(JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID) + (record.get(JavaBaseConstants.COLUMN_NAME_AB_META) as ObjectNode).remove( + JavaBaseConstants.AIRBYTE_META_SYNC_ID_KEY + ) + } + val expectedFinalRecords2 = + readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl") + for (i in 0..4) { + val record = expectedFinalRecords2[i] as ObjectNode + record.remove(JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID) + (record.get(JavaBaseConstants.COLUMN_NAME_AB_META) as ObjectNode).remove( + JavaBaseConstants.AIRBYTE_META_SYNC_ID_KEY + ) + } + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()) + + // Verify that we didn't trigger a soft reset. + // There should be two unique loaded_at values in the raw table. + // (only do this if T+D is enabled to begin with; otherwise loaded_at will just be null) + if (!disableFinalTableComparison()) { + val actualRawRecords2 = dumpRawTableRecords(streamNamespace, streamName) + val loadedAtValues = + actualRawRecords2 + .map { record: JsonNode -> + record.get(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT) + } + .toSet() + Assertions.assertEquals( + 2, + loadedAtValues.size, + "Expected two different values for loaded_at. If there is only 1 value, then we incorrectly triggered a soft reset. If there are more than 2, then something weird happened?" + ) + } + } + + @Test + fun testGenerationIdMigrationForOverwrite() { + val catalog = + ConfiguredAirbyteCatalog() + .withStreams( + listOf( + ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withSyncId(42L) + .withGenerationId(43L) + .withMinimumGenerationId(0L) + .withStream( + AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(BaseTypingDedupingTest.Companion.SCHEMA), + ), + ), + ) + + // First sync + val messages1 = readMessages("dat/sync1_messages.jsonl") + runSync(catalog, messages1, "airbyte/destination-redshift:3.1.1") + + // Second sync + val messages2 = readMessages("dat/sync2_messages.jsonl") + runSync(catalog, messages2) + + val expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_overwrite_raw.jsonl") + val expectedFinalRecords2 = + readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl") + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()) + } + protected fun generateRandomString(totalLength: Int): String { return RANDOM.ints('a'.code, 'z'.code + 1) .limit(totalLength.toLong()) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl index 89913b828215..f62fbeb08acf 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl index 9eb65ecf771a..2f97b055c2ba 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -1,4 +1,4 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl index 387f7776cebf..7ba6be8e4bb7 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -1,5 +1,5 @@ // Keep the Alice record with more recent updated_at -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl index 1b29b504aadd..7b5f7a1facef 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl @@ -1 +1 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00.000000Z", "name": "Someone completely different"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00.000000Z", "name": "Someone completely different"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl index 8a9002dad5f6..c2c780a9d49b 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -1,6 +1,6 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} // Invalid columns are nulled out (i.e. SQL null, not JSON null) -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl index 442ba0192225..d6a32df916aa 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -1,6 +1,6 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} // Invalid data is still allowed in the raw table. -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}]}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_meta": {"changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}]}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl index abbb44d6df67..6a13529ec2e3 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl @@ -1 +1 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}, "_airbyte_meta": {"changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_recordnull_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_recordnull_expectedrecords_final.jsonl index 4b2592985d1f..98a036b92574 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_recordnull_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_recordnull_expectedrecords_final.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "PLACE_HOLDER", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "PLACE_HOLDER", "address": {"city": "San Francisco", "state": "CA"}} // name is SQL null after nulling the record before persisting it. -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"$.name","change":"NULLED","reason":"DESTINATION_FIELD_SIZE_LIMITATION"}]}, "id1": 2, "id2": 201, "updated_at": "2000-01-01T00:00:00.000000Z", "address": {"city": "New York", "state": "NY"}} \ No newline at end of file +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"$.name","change":"NULLED","reason":"DESTINATION_FIELD_SIZE_LIMITATION"}]}, "id1": 2, "id2": 201, "updated_at": "2000-01-01T00:00:00.000000Z", "address": {"city": "New York", "state": "NY"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_recordnull_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_recordnull_expectedrecords_raw.jsonl index b6746d3f906e..20178d571416 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_recordnull_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_recordnull_expectedrecords_raw.jsonl @@ -1,2 +1,2 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "PLACE_HOLDER", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 201, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": null, "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta":{"changes":[{"field":"$.name","change":"NULLED","reason":"DESTINATION_FIELD_SIZE_LIMITATION"}]}} \ No newline at end of file +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "PLACE_HOLDER", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 201, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": null, "address": {"city": "New York", "state": "NY"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"$.name","change":"NULLED","reason":"DESTINATION_FIELD_SIZE_LIMITATION"}]}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl index 31af3c2967a8..6520a1c79fda 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} // Charlie wasn't re-emitted with updated_at, so it still has a null cursor -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 2, "id2": 200, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 2, "id2": 200, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl index dbf989c5b83a..7dc1f622d446 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -1,7 +1,7 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl index 0d83b1e66369..3f733de63c0b 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -1,9 +1,9 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_mixed_meta_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_mixed_meta_final.jsonl index b040bc169783..907cb3c5983d 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_mixed_meta_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_mixed_meta_final.jsonl @@ -4,7 +4,7 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "address": {"city": "San Francisco", "state": "CA"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl index 0a4deced5cef..9c14b92152b4 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl index fbf2611fe68e..bcfa3c5e8c56 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl index c259b5206fb9..65bb7a58fd88 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -1,4 +1,4 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} // Delete Bob, keep Charlie -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl index 69eeec6bab90..a2ea1317e982 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl @@ -1 +1 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00.000000Z", "name": "Someone completely different v2"} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00.000000Z", "name": "Someone completely different v2"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_meta_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_meta_final.jsonl index 3f1e127f3646..92e93d490750 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_meta_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_meta_final.jsonl @@ -1,5 +1,5 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} // Delete Bob, updated Charlie -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}, "id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "address": {"city": "San Francisco", "state": "CA"}} // Record before meta in raw table will continue to have errors. {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_mixed_meta_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_mixed_meta_raw.jsonl index 0203c7194f40..0dc68689f0af 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_mixed_meta_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_mixed_meta_raw.jsonl @@ -5,7 +5,7 @@ {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} // And append the records from the second sync -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00Z", "name":"Charlie", "age":42, "registration_date":"2023-12-23", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta":{"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00Z", "name":"Charlie", "age":42, "registration_date":"2023-12-23", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_overwrite_raw.jsonl new file mode 100644 index 000000000000..144920cb7796 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_overwrite_raw.jsonl @@ -0,0 +1,4 @@ +// Only sync2 messages present in overwrite mode +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl index 128a2d15bade..2fb3c2e010ad 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl @@ -1,10 +1,10 @@ // We keep the records from the first sync -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_meta": {"changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}]}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} // And append the records from the second sync -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z", "_airbyte_meta": {"changes": []}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z", "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl index 6ae7bc9030ad..d9b153184fc1 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl @@ -1,2 +1,2 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}, "_airbyte_meta": {"changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}, "_airbyte_generation_id": 43, "_airbyte_meta": {"sync_id": 42, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl index 698c33c88769..89492e9429ce 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -1,7 +1,7 @@ -{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": [{"field": "string", "change": "NULLED", "reason": "SOURCE_SERIALIZATION_ERROR"}]}} -{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": [{"field":"struct","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"array","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"number","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"}]}} +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_generation_id": 42, "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": 42, "changes": [{"field": "string", "change": "NULLED", "reason": "SOURCE_SERIALIZATION_ERROR"}]}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": [{"field":"struct","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"array","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"number","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"}]}} // Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. // But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). -{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl index 73351b7972a9..7d014b4c0a9c 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -1,5 +1,5 @@ -{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} -{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} -{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}, "_airbyte_meta": {"changes": [{"field": "string", "change": "NULLED", "reason": "SOURCE_SERIALIZATION_ERROR"}]}} -{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} -{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}, "_airbyte_meta": {"sync_id": 42, "changes": [{"field": "string", "change": "NULLED", "reason": "SOURCE_SERIALIZATION_ERROR"}]}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_generation_id": 42, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_v1v2_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_v1v2_expectedrecords_final.jsonl index 0ea024ad6441..ecdbd9e9bac4 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_v1v2_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_v1v2_expectedrecords_final.jsonl @@ -1,8 +1,8 @@ // Same as alltypes_expected but the meta didn't exist in v1 raw tables, so that information is not resurrected to the final. -{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": [{"field":"struct","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"array","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"number","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"integer","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"boolean","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"timestamp_with_timezone","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"timestamp_without_timezone","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"time_with_timezone","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"time_without_timezone","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"date","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"}]}} +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": [{"field":"struct","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"array","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"number","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"integer","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"boolean","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"timestamp_with_timezone","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"timestamp_without_timezone","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"time_with_timezone","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"time_without_timezone","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"},{"field":"date","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"}]}} // Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. // But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). -{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_v1v2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_v1v2_expectedrecords_raw.jsonl index a341d911fbbc..cff6ccdb7837 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_v1v2_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_v1v2_expectedrecords_raw.jsonl @@ -1,5 +1,5 @@ -{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} -{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} -{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} -{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} -{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_generation_id": 0, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl index 47da4b3bceef..2424bebe7b6a 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl @@ -1,2 +1,2 @@ -{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84} -{"_airbyte_raw_id": "b9ac9f01-abc1-4e7c-89e5-eac9223d5726", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": [{"field":"integer","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"}]}, "id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:01.000000Z", "string": "Bob"} +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84} +{"_airbyte_raw_id": "b9ac9f01-abc1-4e7c-89e5-eac9223d5726", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": [{"field":"integer","change":"NULLED","reason":"DESTINATION_TYPECAST_ERROR"}]}, "id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:01.000000Z", "string": "Bob"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl index e015923deeb7..24c68af6eaff 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl @@ -1,5 +1,5 @@ -{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "[\"I\",\"am\",\"an\",\"array\"]", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "{\"I\":\"am\",\"an\":\"object\"}", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "true", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "3.14", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} -{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "[\"I\",\"am\",\"an\",\"array\"]", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "{\"I\":\"am\",\"an\":\"object\"}", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "true", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "3.14", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/mixedcasecolumnname_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/mixedcasecolumnname_expectedrecords_final.jsonl index 3b494b78e4c4..d6b4bb964629 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/mixedcasecolumnname_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/mixedcasecolumnname_expectedrecords_final.jsonl @@ -1 +1 @@ -{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "iamacasesensitivecolumnname": "Case senstive value"} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "iamacasesensitivecolumnname": "Case senstive value"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl index d14bcddf132f..0f204d84fdc9 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl @@ -1 +1 @@ -{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}} +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl index 8ffcc0c73bdc..b7e7144fe2b2 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl @@ -1 +1 @@ -{"_airbyte_raw_id":"b2e0efc4-38a8-47ba-970c-8103f09f08d5","_airbyte_extracted_at":"2023-01-01T00:00:00.000000Z","_airbyte_meta":{"changes":[]}, "current_date": "foo", "join": "bar"} +{"_airbyte_raw_id":"b2e0efc4-38a8-47ba-970c-8103f09f08d5", "_airbyte_extracted_at":"2023-01-01T00:00:00.000000Z", "_airbyte_meta":{"sync_id": null, "changes":[]}, "current_date": "foo", "join": "bar"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl index 33a87e1f4748..102066ffffc7 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl @@ -2,15 +2,15 @@ // TIME, TIMETZ, TIMESTAMP, TIMESTAMPTZ values are UTC in user tables. // Note that redshift stores precision to microseconds. Java deserialization in tests preserves them only for non-zero values // except for timestamp with time zone where Z is required at end for even zero values -{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "time_with_timezone": "12:34:56Z"} -{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} -{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} -{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} -{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} -{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} -{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} -{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.123000Z", "time_with_timezone": "12:34:56.123Z"} +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "time_with_timezone": "12:34:56Z"} +{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} +{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} +{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} +{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} +{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} +{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} +{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.123000Z", "time_with_timezone": "12:34:56.123Z"} -{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_without_timezone": "2023-01-23T12:34:56", "time_without_timezone": "12:34:56", "date": "2023-01-23"} +{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_without_timezone": "2023-01-23T12:34:56", "time_without_timezone": "12:34:56", "date": "2023-01-23"} // Bigquery returns 6 decimal places if there are any decimal places... but not for timestamp_with_timezone -{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.123", "time_without_timezone": "12:34:56.123"} +{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.123", "time_without_timezone": "12:34:56.123"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl index 9d73b0601264..540dc40d3235 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl @@ -6,4 +6,4 @@ // * includes$$doubledollar -> includes__doubledollar // * includes.period -> includes_period // * endswithbackslash\ -> endswithbackslash_ -{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"changes": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "_starts_with_dollar_sign": "foo", "includes_doublequote": "foo", "includes_singlequote": "foo", "includes_backtick": "foo", "includes_period": "foo", "includes__doubledollar": "foo", "endswithbackslash_": "foo"} +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"sync_id": null, "changes": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "_starts_with_dollar_sign": "foo", "includes_doublequote": "foo", "includes_singlequote": "foo", "includes_backtick": "foo", "includes_period": "foo", "includes__doubledollar": "foo", "endswithbackslash_": "foo"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql b/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql index e224ff77babe..591b75a1e617 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql +++ b/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql @@ -18,6 +18,7 @@ insert into "test_schema"."users_finalunittest" ( "_ab_cdc_deleted_at", "_airbyte_raw_id", "_airbyte_extracted_at", + "_airbyte_generation_id", "_airbyte_meta" ) with @@ -44,6 +45,7 @@ with cast("_airbyte_data"."_ab_cdc_deleted_at" as timestamp with time zone) as "_ab_cdc_deleted_at", "_airbyte_raw_id", "_airbyte_extracted_at", + "_airbyte_generation_id", OBJECT( 'changes', ARRAY_CONCAT( @@ -147,7 +149,9 @@ with and "_airbyte_meta"."changes" is not null and IS_ARRAY("_airbyte_meta"."changes") ) THEN "_airbyte_meta"."changes" ELSE ARRAY() END - ) + ), + 'sync_id', + "_airbyte_meta"."sync_id" ) as "_airbyte_meta" from "test_schema"."users_raw" where ( @@ -191,6 +195,7 @@ select "_ab_cdc_deleted_at", "_airbyte_raw_id", "_airbyte_extracted_at", + "_airbyte_generation_id", "_airbyte_meta" from "numbered_rows" where "row_number" = 1; diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index bbf62f964dd7..0ff098315a74 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -3,7 +3,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.37.1' + cdkVersionRequired = '0.40.9' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index d32c083e1882..8e61b9b175fa 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.10.1 + dockerImageTag: 3.11.0 dockerRepository: airbyte/destination-snowflake documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake githubIssueLabel: destination-snowflake @@ -34,6 +34,7 @@ data: memory_request: 2Gi supportLevel: certified supportsDbt: true + supportsRefreshes: true tags: - language:java connectorTestSuitesOptions: diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.kt b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.kt index 20519034b395..b57f8aad1486 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.kt @@ -122,6 +122,12 @@ constructor( hasUnprocessedRecords = true, maxProcessedTimestamp = Optional.empty() ), + initialTempRawTableStatus = + InitialRawTableStatus( + rawTableExists = false, + hasUnprocessedRecords = true, + maxProcessedTimestamp = Optional.empty() + ), isSchemaMismatch = true, isFinalTableEmpty = true, destinationState = diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/migrations/SnowflakeAbMetaAndGenIdMigration.kt b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/migrations/SnowflakeAbMetaAndGenIdMigration.kt index f67b038f16fe..56c68043b866 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/migrations/SnowflakeAbMetaAndGenIdMigration.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/migrations/SnowflakeAbMetaAndGenIdMigration.kt @@ -122,6 +122,14 @@ class SnowflakeAbMetaAndGenIdMigration(private val database: JdbcDatabase) : state.destinationState.copy(isAirbyteMetaPresentInRaw = true), true ) + } else if (!state.isFinalTablePresent) { + log.info { + "skipping migration of generation_id for table ${stream.id.finalNamespace}.${stream.id.finalName} because final table doesn't exist" + } + } else { + log.info { + "skipping migration of generation_id for table ${stream.id.finalNamespace}.${stream.id.finalName} because schemas match" + } } // Final table is untouched, so we don't need to fetch the initial status diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStagingClient.kt b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStagingClient.kt index c9cb6d74fe9e..3e6acf9ebff5 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStagingClient.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStagingClient.kt @@ -169,11 +169,12 @@ class SnowflakeStagingClient(private val database: JdbcDatabase) { stageName: String, stagingPath: String, stagedFiles: List, - streamId: StreamId + streamId: StreamId, + suffix: String = "" ) { try { val queryId = UUID.randomUUID() - val query = getCopyQuery(stageName, stagingPath, stagedFiles, streamId) + val query = getCopyQuery(stageName, stagingPath, stagedFiles, streamId, suffix) log.info { "query $queryId, $query" } // queryJsons is intentionally used here to get the error message in case of failure // instead of execute @@ -252,12 +253,13 @@ class SnowflakeStagingClient(private val database: JdbcDatabase) { stageName: String, stagingPath: String, stagedFiles: List, - streamId: StreamId + streamId: StreamId, + suffix: String ): String { return String.format( COPY_QUERY_1S1T + generateFilesList(stagedFiles) + ";", streamId.rawNamespace, - streamId.rawName, + streamId.rawName + suffix, stageName, stagingPath ) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperation.kt b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperation.kt index a0a2e60329a2..d873df31890f 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperation.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperation.kt @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.snowflake.operation +import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.base.JavaBaseConstants import io.airbyte.cdk.integrations.destination.StandardNameTransformer import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer @@ -15,7 +16,6 @@ import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduperUtil import io.airbyte.integrations.destination.snowflake.SnowflakeSQLNameTransformer import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeDestinationHandler import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeSqlGenerator -import io.airbyte.protocol.models.v0.DestinationSyncMode import io.github.oshai.kotlinlogging.KotlinLogging import java.time.Instant import java.time.ZoneOffset @@ -35,19 +35,77 @@ class SnowflakeStorageOperation( private val connectionId = UUID.randomUUID() private val syncDateTime = Instant.now() - override fun prepareStage(streamId: StreamId, destinationSyncMode: DestinationSyncMode) { + override fun prepareStage(streamId: StreamId, suffix: String, replace: Boolean) { // create raw table - destinationHandler.execute(Sql.of(createTableQuery(streamId))) - if (destinationSyncMode == DestinationSyncMode.OVERWRITE) { - destinationHandler.execute(Sql.of(truncateTableQuery(streamId))) + destinationHandler.execute(Sql.of(createTableQuery(streamId, suffix))) + if (replace) { + destinationHandler.execute(Sql.of(truncateTableQuery(streamId, suffix))) } // create stage staging.createStageIfNotExists(getStageName(streamId)) } - internal fun createTableQuery(streamId: StreamId): String { + override fun overwriteStage(streamId: StreamId, suffix: String) { + if (suffix.isBlank()) { + throw IllegalArgumentException("Cannot overwrite raw table with empty suffix") + } + // Something weird happening with SWAP WITH in truncateRefresh tests, + // so using DROP AND ALTER RENAME instead + destinationHandler.execute( + Sql.of("DROP TABLE IF EXISTS \"${streamId.rawNamespace}\".\"${streamId.rawName}\"") + ) + val swapQuery = + """ + | ALTER TABLE "${streamId.rawNamespace}"."${streamId.rawName+suffix}" RENAME TO "${streamId.rawNamespace}"."${streamId.rawName}"; + """.trimMargin() + destinationHandler.execute(Sql.of(swapQuery)) + } + + override fun transferFromTempStage(streamId: StreamId, suffix: String) { + if (suffix.isBlank()) { + throw IllegalArgumentException( + "Cannot transfer records from temp raw table with empty suffix" + ) + } + destinationHandler.execute( + Sql.of( + """ + INSERT INTO "${streamId.rawNamespace}"."${streamId.rawName}" + SELECT * FROM "${streamId.rawNamespace}"."${streamId.rawName + suffix}" + """.trimIndent() + ) + ) + destinationHandler.execute( + Sql.of( + """ + DROP TABLE "${streamId.rawNamespace}"."${streamId.rawName + suffix}" + """.trimIndent() + ) + ) + } + + override fun getStageGeneration(streamId: StreamId, suffix: String): Long? { + val results = + destinationHandler.query( + """ + SELECT "${JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID}" FROM "${streamId.rawNamespace}"."${streamId.rawName + suffix}" LIMIT 1 + """.trimIndent() + ) + if (results.isEmpty()) return null + var generationIdNode: JsonNode? = + results.first().get(JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID) + if (generationIdNode == null) { + // This is the dance where QUOTED_IDENTIFIERS_IGNORE_CASE will return uppercase column + // as result, so check for fallback. + generationIdNode = + results.first().get(JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID.uppercase()) + } + return generationIdNode?.asLong() + } + + internal fun createTableQuery(streamId: StreamId, suffix: String): String { return """ - |CREATE TABLE IF NOT EXISTS "${streamId.rawNamespace}"."${streamId.rawName}"( + |CREATE TABLE IF NOT EXISTS "${streamId.rawNamespace}"."${streamId.rawName + suffix}"( | "${JavaBaseConstants.COLUMN_NAME_AB_RAW_ID}" VARCHAR PRIMARY KEY, | "${JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT}" TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp(), | "${JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT}" TIMESTAMP WITH TIME ZONE DEFAULT NULL, @@ -58,11 +116,15 @@ class SnowflakeStorageOperation( """.trimMargin() } - internal fun truncateTableQuery(streamId: StreamId): String { - return "TRUNCATE TABLE \"${streamId.rawNamespace}\".\"${streamId.rawName}\";\n" + internal fun truncateTableQuery(streamId: StreamId, suffix: String): String { + return "TRUNCATE TABLE \"${streamId.rawNamespace}\".\"${streamId.rawName + suffix}\";\n" } - override fun writeToStage(streamConfig: StreamConfig, data: SerializableBuffer) { + override fun writeToStage( + streamConfig: StreamConfig, + suffix: String, + data: SerializableBuffer + ) { val stageName = getStageName(streamConfig.id) val stagingPath = getStagingPath() val stagedFileName = staging.uploadRecordsToStage(data, stageName, stagingPath) @@ -70,7 +132,8 @@ class SnowflakeStorageOperation( stageName, stagingPath, listOf(stagedFileName), - streamConfig.id + streamConfig.id, + suffix ) } override fun cleanupStage(streamId: StreamId) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.kt b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.kt index b67205dd4f0a..8f0435f5a429 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.kt @@ -11,6 +11,7 @@ import io.airbyte.cdk.integrations.destination.jdbc.ColumnDefinition import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler import io.airbyte.commons.json.Jsons.emptyObject +import io.airbyte.integrations.base.destination.operation.AbstractStreamOperation import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType import io.airbyte.integrations.base.destination.typing_deduping.Array @@ -85,14 +86,24 @@ class SnowflakeDestinationHandler( @Throws(Exception::class) private fun getInitialRawTableState( id: StreamId, + suffix: String, ): InitialRawTableStatus { - // Short-circuit for overwrite, table will be truncated anyway + val rawTableName = id.rawName + suffix val tableExists = database.executeMetadataQuery { databaseMetaData: DatabaseMetaData -> - LOGGER.info("Retrieving table from Db metadata: {} {}", id.rawNamespace, id.rawName) + LOGGER.info( + "Retrieving table from Db metadata: {} {}", + id.rawNamespace, + rawTableName + ) try { val rs = - databaseMetaData.getTables(databaseName, id.rawNamespace, id.rawName, null) + databaseMetaData.getTables( + databaseName, + id.rawNamespace, + rawTableName, + null + ) // When QUOTED_IDENTIFIERS_IGNORE_CASE is set to true, the raw table is // interpreted as uppercase // in db metadata calls. check for both @@ -100,7 +111,7 @@ class SnowflakeDestinationHandler( databaseMetaData.getTables( databaseName, id.rawNamespace.uppercase(), - id.rawName.uppercase(), + rawTableName.uppercase(), null ) rs.next() || rsUppercase.next() @@ -130,7 +141,7 @@ class SnowflakeDestinationHandler( StringSubstitutor( java.util.Map.of( "raw_table", - id.rawTableId(SnowflakeSqlGenerator.QUOTE) + id.rawTableId(SnowflakeSqlGenerator.QUOTE, suffix) ) ) .replace( @@ -186,7 +197,7 @@ class SnowflakeDestinationHandler( StringSubstitutor( java.util.Map.of( "raw_table", - id.rawTableId(SnowflakeSqlGenerator.QUOTE) + id.rawTableId(SnowflakeSqlGenerator.QUOTE, suffix) ) ) .replace( @@ -286,7 +297,7 @@ class SnowflakeDestinationHandler( "VARIANT" == existingTable.columns[abMetaColumnName]!!.type } - fun isAirbyteGenerationIdColumnMatch(existingTable: TableDefinition): Boolean { + private fun isAirbyteGenerationIdColumnMatch(existingTable: TableDefinition): Boolean { val abGenerationIdColumnName: String = JavaBaseConstants.COLUMN_NAME_AB_GENERATION_ID.uppercase(Locale.getDefault()) return existingTable.columns.containsKey(abGenerationIdColumnName) && @@ -388,7 +399,12 @@ class SnowflakeDestinationHandler( !existingSchemaMatchesStreamConfig(streamConfig, existingTable!!) isFinalTableEmpty = hasRowCount && tableRowCounts[namespace]!![name] == 0 } - val initialRawTableState = getInitialRawTableState(streamConfig.id) + val initialRawTableState = getInitialRawTableState(streamConfig.id, "") + val tempRawTableState = + getInitialRawTableState( + streamConfig.id, + AbstractStreamOperation.TMP_TABLE_SUFFIX + ) val destinationState = destinationStates.getOrDefault( streamConfig.id.asPair(), @@ -398,6 +414,7 @@ class SnowflakeDestinationHandler( streamConfig, isFinalTablePresent, initialRawTableState, + tempRawTableState, isSchemaMismatch, isFinalTableEmpty, destinationState @@ -466,6 +483,10 @@ class SnowflakeDestinationHandler( } } + fun query(sql: String): List { + return database.queryJsons(sql) + } + companion object { private val LOGGER: Logger = LoggerFactory.getLogger(SnowflakeDestinationHandler::class.java) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.kt b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.kt index 673f8198aa45..f31c3cfc0d3e 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.kt @@ -11,7 +11,6 @@ import java.sql.Connection import java.sql.ResultSet import java.sql.SQLException import java.util.* -import java.util.Map import java.util.stream.Collectors import kotlin.collections.List import org.apache.commons.text.StringSubstitutor @@ -100,14 +99,12 @@ object SnowflakeTestUtils { .createStatement() .executeQuery( StringSubstitutor( - Map.of( - "columns", - columns.stream().collect(Collectors.joining(",")), - "table", - tableIdentifier, - "extracted_at", - if (upcaseExtractedAt) "_AIRBYTE_EXTRACTED_AT" - else "\"_airbyte_extracted_at\"" + mapOf( + "columns" to columns.stream().collect(Collectors.joining(",")), + "table" to tableIdentifier, + "extracted_at" to + if (upcaseExtractedAt) "_AIRBYTE_EXTRACTED_AT" + else "\"_airbyte_extracted_at\"" ) ) .replace( diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperationIntegrationTest.kt b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperationIntegrationTest.kt new file mode 100644 index 000000000000..c30718de52de --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperationIntegrationTest.kt @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.operation + +import com.fasterxml.jackson.databind.JsonNode +import io.airbyte.cdk.db.jdbc.JdbcDatabase +import io.airbyte.cdk.db.jdbc.JdbcUtils +import io.airbyte.cdk.integrations.base.JavaBaseConstants +import io.airbyte.cdk.integrations.destination.async.model.PartialAirbyteMessage +import io.airbyte.cdk.integrations.destination.async.model.PartialAirbyteRecordMessage +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer +import io.airbyte.cdk.integrations.destination.s3.FileUploadFormat +import io.airbyte.cdk.integrations.destination.staging.StagingSerializedBufferFactory +import io.airbyte.commons.json.Jsons +import io.airbyte.commons.string.Strings +import io.airbyte.integrations.base.destination.operation.AbstractStreamOperation +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig +import io.airbyte.integrations.base.destination.typing_deduping.StreamId +import io.airbyte.integrations.destination.snowflake.OssCloudEnvVarConsts +import io.airbyte.integrations.destination.snowflake.SnowflakeDatabaseUtils +import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeDestinationHandler +import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeSqlGenerator +import io.airbyte.protocol.models.v0.AirbyteMessage +import io.airbyte.protocol.models.v0.AirbyteRecordMessageMeta +import io.airbyte.protocol.models.v0.DestinationSyncMode +import java.nio.file.Files +import java.nio.file.Paths +import java.util.* +import net.snowflake.client.jdbc.SnowflakeSQLException +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.Mockito.mock + +class SnowflakeStorageOperationIntegrationTest { + + private var streamId: StreamId = mock() + private var streamConfig: StreamConfig = mock() + @BeforeEach + fun setup() { + val randomString = Strings.addRandomSuffix("", "", 10) + streamId = + StreamId( + finalNamespace = "final_namespace_$randomString", + finalName = "final_name_$randomString", + rawNamespace = "raw_namespace_$randomString", + rawName = "raw_name_$randomString", + originalNamespace = "original_namespace_$randomString", + originalName = "original_name_$randomString", + ) + streamConfig = + StreamConfig( + streamId, + DestinationSyncMode.APPEND, + emptyList(), + Optional.empty(), + LinkedHashMap(), + GENERATION_ID, + 0, + SYNC_ID, + ) + database.execute( + """ + CREATE SCHEMA "${streamId.rawNamespace}" + """.trimIndent() + ) + } + + @AfterEach + fun teardown() { + database.execute("DROP SCHEMA IF EXISTS \"${streamId.rawNamespace}\" CASCADE") + } + + private fun record(recordNumber: Int): PartialAirbyteMessage { + val serializedData = """{"record_number": $recordNumber}""" + return PartialAirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withSerialized(serializedData) + .withRecord( + PartialAirbyteRecordMessage() + .withNamespace(streamId.originalNamespace) + .withStream(streamId.originalName) + .withEmittedAt(10_000) + .withMeta( + AirbyteRecordMessageMeta() + .withChanges(emptyList()) + .withAdditionalProperty( + JavaBaseConstants.AIRBYTE_META_SYNC_ID_KEY, + SYNC_ID, + ), + ) + .withData(Jsons.deserialize(serializedData)), + ) + } + + private fun buffer( + partialAirbyteMessage: PartialAirbyteMessage, + callback: (buffer: SerializableBuffer) -> Unit + ) { + val csvBuffer = + StagingSerializedBufferFactory.initializeBuffer( + FileUploadFormat.CSV, + JavaBaseConstants.DestinationColumns.V2_WITH_GENERATION + ) + csvBuffer.use { + it.accept( + partialAirbyteMessage.serialized!!, + Jsons.serialize(partialAirbyteMessage.record!!.meta), + streamConfig.generationId, + partialAirbyteMessage.record!!.emittedAt + ) + it.flush() + callback(csvBuffer) + } + } + + private fun dumpRawRecords(suffix: String): List { + val query = + """ + SELECT * FROM ${streamId.rawTableId("\"", suffix)} + """.trimIndent() + return database.queryJsons(query) + } + + @Test + fun testTransferStage() { + storageOperation.prepareStage(streamId, "") + storageOperation.prepareStage(streamId, AbstractStreamOperation.TMP_TABLE_SUFFIX) + // Table is currently empty, so expect null generation. + assertNull( + storageOperation.getStageGeneration(streamId, AbstractStreamOperation.TMP_TABLE_SUFFIX) + ) + // Write one record to the real raw table + buffer(record(1)) { + storageOperation.writeToStage( + streamConfig, + "", + it, + ) + } + println(dumpRawRecords("")) + assertEquals( + listOf(Jsons.deserialize("""{"record_number":1}""")), + dumpRawRecords("").map { it["_airbyte_data"] }, + ) + // And write one record to the temp final table + buffer(record(2)) { + storageOperation.writeToStage( + streamConfig, + AbstractStreamOperation.TMP_TABLE_SUFFIX, + it, + ) + } + assertEquals( + listOf(Jsons.deserialize("""{"record_number": 2}""")), + dumpRawRecords(AbstractStreamOperation.TMP_TABLE_SUFFIX).map { it["_airbyte_data"] }, + ) + assertEquals( + GENERATION_ID, + storageOperation.getStageGeneration(streamId, AbstractStreamOperation.TMP_TABLE_SUFFIX) + ) + // If we transfer the records, we should end up with 2 records in the real raw table. + storageOperation.transferFromTempStage(streamId, AbstractStreamOperation.TMP_TABLE_SUFFIX) + assertEquals( + listOf( + Jsons.deserialize("""{"record_number": 1}"""), + Jsons.deserialize("""{"record_number": 2}"""), + ), + dumpRawRecords("") + .map { it["_airbyte_data"] } + .sortedBy { it.get("record_number").asLong() }, + ) + // After transferring the records to the real table, the temp table should no longer exist. + assertThrows(SnowflakeSQLException::class.java) { + dumpRawRecords(AbstractStreamOperation.TMP_TABLE_SUFFIX) + } + } + + @Test + fun testOverwriteStage() { + // If we then create another temp raw table and _overwrite_ the real raw table, + // we should end up with a single raw record. + storageOperation.prepareStage(streamId, "") + storageOperation.prepareStage(streamId, AbstractStreamOperation.TMP_TABLE_SUFFIX) + buffer(record(3)) { + storageOperation.writeToStage( + streamConfig, + "", + it, + ) + } + buffer(record(4)) { + storageOperation.writeToStage( + streamConfig, + AbstractStreamOperation.TMP_TABLE_SUFFIX, + it, + ) + } + storageOperation.overwriteStage(streamId, AbstractStreamOperation.TMP_TABLE_SUFFIX) + assertEquals( + listOf(Jsons.deserialize("""{"record_number": 4}""")), + dumpRawRecords("").map { it["_airbyte_data"] }, + ) + assertThrows(SnowflakeSQLException::class.java) { + dumpRawRecords(AbstractStreamOperation.TMP_TABLE_SUFFIX) + } + } + + companion object { + private val config = + Jsons.deserialize( + Files.readString(Paths.get("secrets/1s1t_internal_staging_config.json")) + ) + private val datasource = + SnowflakeDatabaseUtils.createDataSource(config, OssCloudEnvVarConsts.AIRBYTE_OSS) + private val database: JdbcDatabase = SnowflakeDatabaseUtils.getDatabase(datasource) + private val storageOperation: SnowflakeStorageOperation = + SnowflakeStorageOperation( + SnowflakeSqlGenerator(0), + SnowflakeDestinationHandler( + config[JdbcUtils.DATABASE_KEY].asText(), + database, + config[JdbcUtils.SCHEMA_KEY].asText(), + ), + 0, + SnowflakeStagingClient(database), + ) + private const val SYNC_ID = 12L + private const val GENERATION_ID = 42L + } +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.kt b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.kt index b178ffe73001..b9f4af56186b 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.kt @@ -18,6 +18,7 @@ import io.airbyte.integrations.base.destination.typing_deduping.StreamId import io.airbyte.integrations.destination.snowflake.* import io.airbyte.protocol.models.v0.* import io.airbyte.workers.exception.TestHarnessException +import io.github.oshai.kotlinlogging.KotlinLogging import java.nio.file.Path import java.sql.SQLException import java.util.* @@ -28,6 +29,8 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Test +private val LOGGER = KotlinLogging.logger {} + abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { private var databaseName: String? = null private var database: JdbcDatabase? = null @@ -132,6 +135,9 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { .withSyncMode(SyncMode.INCREMENTAL) .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) .withPrimaryKey(java.util.List.of(listOf("id1"), listOf("id2"))) + .withGenerationId(0) + .withMinimumGenerationId(0) + .withSyncId(0) .withStream( AirbyteStream() .withNamespace(streamNamespace) @@ -191,20 +197,21 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { catalog, messages1, "airbyte/destination-snowflake:2.1.7", - ) { config: JsonNode? -> - // Defensive to avoid weird behaviors or test failures if the original config is being - // altered by - // another thread, thanks jackson for a mutable JsonNode - val copiedConfig = Jsons.clone(config!!) - if (config is ObjectNode) { - // Opt out of T+D to run old V1 sync - (copiedConfig as ObjectNode?)!!.put( - "use_1s1t_format", - false, - ) - } - copiedConfig - } + { config: JsonNode? -> + // Defensive to avoid weird behaviors or test failures if the original config is + // being altered by another thread, thanks jackson for a mutable JsonNode + val copiedConfig = Jsons.clone(config!!) + if (config is ObjectNode) { + // Opt out of T+D to run old V1 sync + (copiedConfig as ObjectNode?)!!.put( + "use_1s1t_format", + false, + ) + } + copiedConfig + }, + streamStatus = null + ) // The record differ code is already adapted to V2 columns format, use the post V2 sync // to verify that append mode preserved all the raw records and final records. @@ -225,13 +232,16 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { @Test @Throws(Exception::class) - fun testExtractedAtUtcTimezoneMigration() { + open fun testExtractedAtUtcTimezoneMigration() { val catalog = ConfiguredAirbyteCatalog() .withStreams( java.util.List.of( ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) + .withGenerationId(0) + .withSyncId(0) + .withMinimumGenerationId(0) .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) .withPrimaryKey(java.util.List.of(listOf("id1"), listOf("id2"))) .withCursorField(listOf("updated_at")) @@ -246,7 +256,7 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { // First sync val messages1 = readMessages("dat/sync1_messages.jsonl") - runSync(catalog, messages1, "airbyte/destination-snowflake:3.5.11") + runSync(catalog, messages1, "airbyte/destination-snowflake:3.5.11", streamStatus = null) // The dumpRawTable code already accounts for Meta and GenID columns, so we cannot use it // to verify expected records. We will rely on the second sync to verify raw and final @@ -264,7 +274,7 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { } @Test - fun testAirbyteMetaAndGenerationIdMigration() { + open fun testAirbyteMetaAndGenerationIdMigration() { val catalog = ConfiguredAirbyteCatalog() .withStreams( @@ -286,7 +296,7 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { // First sync val messages1 = readMessages("dat/sync1_messages.jsonl") - runSync(catalog, messages1, "airbyte/destination-snowflake:3.9.1") + runSync(catalog, messages1, "airbyte/destination-snowflake:3.9.1", streamStatus = null) // Second sync val messages2 = readMessages("dat/sync2_messages.jsonl") @@ -338,42 +348,7 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { } @Test - fun testAirbyteMetaAndGenerationIdMigrationForOverwrite() { - val catalog = - ConfiguredAirbyteCatalog() - .withStreams( - listOf( - ConfiguredAirbyteStream() - .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) - .withSyncId(42L) - .withGenerationId(43L) - .withMinimumGenerationId(0L) - .withStream( - AirbyteStream() - .withNamespace(streamNamespace) - .withName(streamName) - .withJsonSchema(BaseTypingDedupingTest.Companion.SCHEMA), - ), - ), - ) - - // First sync - val messages1 = readMessages("dat/sync1_messages.jsonl") - runSync(catalog, messages1, "airbyte/destination-snowflake:3.9.1") - - // Second sync - val messages2 = readMessages("dat/sync2_messages.jsonl") - runSync(catalog, messages2) - - val expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_overwrite_raw.jsonl") - val expectedFinalRecords2 = - readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl") - verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()) - } - - @Test - fun testAirbyteMetaAndGenerationIdMigrationForOverwrite310Broken() { + open fun testAirbyteMetaAndGenerationIdMigrationForOverwrite() { val catalog = ConfiguredAirbyteCatalog() .withStreams( @@ -383,7 +358,7 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) .withSyncId(42L) .withGenerationId(43L) - .withMinimumGenerationId(0L) + .withMinimumGenerationId(43L) .withStream( AirbyteStream() .withNamespace(streamNamespace) @@ -395,15 +370,9 @@ abstract class AbstractSnowflakeTypingDedupingTest : BaseTypingDedupingTest() { // First sync val messages1 = readMessages("dat/sync1_messages.jsonl") - runSync(catalog, messages1, "airbyte/destination-snowflake:3.9.1") + runSync(catalog, messages1, "airbyte/destination-snowflake:3.9.1", streamStatus = null) // Second sync - // This throws exception due to a broken migration in connector - assertThrows(TestHarnessException::class.java) { - runSync(catalog, messages1, "airbyte/destination-snowflake:3.10.0") - } - - // Third sync val messages2 = readMessages("dat/sync2_messages.jsonl") runSync(catalog, messages2) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_with_new_gen_id_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_with_new_gen_id_final.jsonl new file mode 100644 index 000000000000..0f34e9450361 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_with_new_gen_id_final.jsonl @@ -0,0 +1,9 @@ +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 43, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "San Francisco", "state": "CA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 43, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 43, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 43, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie", "AGE": 42, "REGISTRATION_DATE": "2023-12-23"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 43, "ID1": 3, "ID2": 200, "UPDATED_AT": "2000-01-01T00:04:00.000000000Z", "NAME": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} + +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 44, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 44, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "New York", "state": "NY"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 44, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:01:00.000000000Z", "_AB_CDC_DELETED_AT": "1970-01-01T00:00:00.000000000Z"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_final.jsonl new file mode 100644 index 000000000000..df3ccb1b8131 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_final.jsonl @@ -0,0 +1,3 @@ +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 44, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 44, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "New York", "state": "NY"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000Z", "_AIRBYTE_META": {"changes":[],"sync_id":42}, "_AIRBYTE_GENERATION_ID": 44, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:01:00.000000000Z", "_AB_CDC_DELETED_AT": "1970-01-01T00:00:00.000000000Z"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_raw.jsonl new file mode 100644 index 000000000000..9deb338f67d5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_with_new_gen_id_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_with_new_gen_id_raw.jsonl new file mode 100644 index 000000000000..9e00d3955b08 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_with_new_gen_id_raw.jsonl @@ -0,0 +1,10 @@ +// We keep the records from the first sync +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} +// And append the records from the second sync +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStagingClientTest.kt b/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStagingClientTest.kt index edc649ccefe2..dd51dc310537 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStagingClientTest.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStagingClientTest.kt @@ -100,7 +100,7 @@ class SnowflakeStagingClientTest { inOrder .verify(database) .queryJsons( - stagingClient.getCopyQuery(stageName, stagingPath, stagedFiles, streamId) + stagingClient.getCopyQuery(stageName, stagingPath, stagedFiles, streamId, "") ) verifyNoMoreInteractions(database) } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperationTest.kt b/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperationTest.kt index 4c5e89623b7c..b80c0a6ce628 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperationTest.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/operation/SnowflakeStorageOperationTest.kt @@ -42,10 +42,10 @@ class SnowflakeStorageOperationTest { @Test fun verifyPrepareStageCreatesTableAndStage() { val inOrder = inOrder(destinationHandler, stagingClient) - storageOperation.prepareStage(streamId, DestinationSyncMode.APPEND) + storageOperation.prepareStage(streamId, "", false) inOrder .verify(destinationHandler) - .execute(Sql.of(storageOperation.createTableQuery(streamId))) + .execute(Sql.of(storageOperation.createTableQuery(streamId, ""))) inOrder .verify(stagingClient) .createStageIfNotExists(storageOperation.getStageName(streamId)) @@ -55,13 +55,13 @@ class SnowflakeStorageOperationTest { @Test fun verifyPrepareStageOverwriteTruncatesTable() { val inOrder = inOrder(destinationHandler, stagingClient) - storageOperation.prepareStage(streamId, DestinationSyncMode.OVERWRITE) + storageOperation.prepareStage(streamId, "", true) inOrder .verify(destinationHandler) - .execute(Sql.of(storageOperation.createTableQuery(streamId))) + .execute(Sql.of(storageOperation.createTableQuery(streamId, ""))) inOrder .verify(destinationHandler) - .execute(Sql.of(storageOperation.truncateTableQuery(streamId))) + .execute(Sql.of(storageOperation.truncateTableQuery(streamId, ""))) inOrder .verify(stagingClient) .createStageIfNotExists(storageOperation.getStageName(streamId)) @@ -80,12 +80,18 @@ class SnowflakeStorageOperationTest { val storageOperation = SnowflakeStorageOperation(sqlGenerator, destinationHandler, 1, stagingClient) - storageOperation.writeToStage(streamConfig, data) + storageOperation.writeToStage(streamConfig, "", data) val inOrder = inOrder(stagingClient) inOrder.verify(stagingClient).uploadRecordsToStage(any(), eq(stageName), any()) inOrder .verify(stagingClient) - .copyIntoTableFromStage(eq(stageName), any(), eq(listOf(mockTmpFileName)), eq(streamId)) + .copyIntoTableFromStage( + eq(stageName), + any(), + eq(listOf(mockTmpFileName)), + eq(streamId), + eq("") + ) verifyNoMoreInteractions(stagingClient) } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.kt b/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.kt index 0394820b4898..7613f371f5e4 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.kt +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/kotlin/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.kt @@ -92,6 +92,9 @@ class SnowflakeSqlGeneratorTest { ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withGenerationId(0) + .withMinimumGenerationId(0) + .withSyncId(0) .withStream( AirbyteStream() .withName("foo") diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec_oss.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec_oss.json index 46da9aef832b..ef65f9dd15fd 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec_oss.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec_oss.json @@ -90,7 +90,7 @@ }, "replication_start_date": { "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If start date is not provided, the date 2 years ago from today will be used.", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If start date is not provided or older than 2 years ago from today, the date 2 years ago from today will be used.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], "order": 7, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index 4539c36c6b36..f286d3d20be3 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -15,7 +15,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 4.3.5 + dockerImageTag: 4.3.6 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml b/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml index 8604809ce265..a742ef544be3 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/pyproject.toml @@ -3,7 +3,7 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.3.5" +version = "4.3.6" name = "source-amazon-seller-partner" description = "Source implementation for Amazon Seller Partner." authors = ["Airbyte "] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index aea291617c8d..77c8b71baf80 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -73,6 +73,9 @@ ) from source_amazon_seller_partner.utils import AmazonConfigException +# given the retention period: 730 +DEFAULT_RETENTION_PERIOD_IN_DAYS = 730 + class SourceAmazonSellerPartner(AbstractSource): @staticmethod @@ -86,18 +89,26 @@ def _get_stream_kwargs(config: Mapping[str, Any]) -> Mapping[str, Any]: host=endpoint.replace("https://", ""), refresh_access_token_headers={"Content-Type": "application/x-www-form-urlencoded"}, ) - start_date = ( - config.get("replication_start_date") - if config.get("replication_start_date") - else pendulum.now("utc").subtract(years=2).strftime("%Y-%m-%dT%H:%M:%SZ") + + start_date = config.get("replication_start_date") + use_default_start_date = ( + not start_date or (pendulum.now("utc") - pendulum.parse(start_date)).days > DEFAULT_RETENTION_PERIOD_IN_DAYS ) + if use_default_start_date: + start_date = pendulum.now("utc").subtract(days=DEFAULT_RETENTION_PERIOD_IN_DAYS).strftime("%Y-%m-%dT%H:%M:%SZ") + + end_date = config.get("replication_end_date") + use_default_end_date = not end_date or end_date < start_date + if use_default_end_date: + end_date = None # None to sync all data + stream_kwargs = { "url_base": endpoint, "authenticator": auth, "replication_start_date": start_date, "marketplace_id": marketplace_id, "period_in_days": config.get("period_in_days", 30), - "replication_end_date": config.get("replication_end_date"), + "replication_end_date": end_date, } return stream_kwargs diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json index 01a8e84956aa..beb8003a3301 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -90,7 +90,7 @@ }, "replication_start_date": { "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If start date is not provided, the date 2 years ago from today will be used.", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If start date is not provided or older than 2 years ago from today, the date 2 years ago from today will be used.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], "order": 7, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index 9dbb06d9d5be..ee256f8ff26f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -409,6 +409,15 @@ def read_records( "Try to grant required permissions/scopes or re-authenticate." ) return [] + + errors = " ".join([er.get("message", "") for er in e.response.json().get("errors", [])]) + if "does not support account ID of type class com.amazon.partner.account.id.VendorGroupId." in errors: + logger.warning( + f"The endpoint {e.response.url} returned {e.response.status_code}: {errors}. " + "This is most likely due to account type (Vendor) on the credentials in use. " + "Try to re-authenticate with Seller account type and sync again." + ) + return [] raise e # create and retrieve the report @@ -448,7 +457,8 @@ def read_records( if stream_slice and "dataStartTime" in stream_slice: exception_message += ( f" for period {stream_slice['dataStartTime']}-{stream_slice['dataEndTime']}. " - f"This will be read during the next sync. Error: {error_response}" + f"This will be read during the next sync. Report ID: {report_id}." + f" Error: {error_response}" ) raise AirbyteTracedException(internal_message=exception_message) elif processing_status == ReportProcessingStatus.CANCELLED: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py index b270db46edd9..eae025abbbce 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py @@ -393,7 +393,7 @@ def test_given_report_status_fatal_when_read_then_exception_raised( if stream_name == "GET_VENDOR_TRAFFIC_REPORT": config_end_date = VENDOR_TRAFFIC_REPORT_CONFIG_END_DATE assert ( - f"Failed to retrieve the report '{stream_name}' for period {CONFIG_START_DATE}-{config_end_date}. This will be read during the next sync. Error: {{'errorDetails': 'Error in report request: This report type requires the reportPeriod, distributorView, sellingProgram reportOption to be specified. Please review the document for this report type on GitHub, provide a value for this reportOption in your request, and try again.'}}" + f"Failed to retrieve the report '{stream_name}' for period {CONFIG_START_DATE}-{config_end_date}. This will be read during the next sync. Report ID: 6789087632. Error: {{'errorDetails': 'Error in report request: This report type requires the reportPeriod, distributorView, sellingProgram reportOption to be specified. Please review the document for this report type on GitHub, provide a value for this reportOption in your request, and try again.'}}" ) in output.errors[-1].trace.error.message @pytest.mark.parametrize( @@ -443,6 +443,35 @@ def test_given_http_error_500_on_create_report_when_read_then_no_records_and_err assert_message_in_log_output(message_on_backoff_exception, output) assert len(output.records) == 0 + @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) + @HttpMocker() + def test_given_http_error_not_support_account_id_of_type_vendor_when_read_then_no_records_and_error_logged( + self, stream_name: str, data_format: str, http_mocker: HttpMocker + ): + mock_auth(http_mocker) + response_body = { + "errors": [ + {"code": "InvalidInput", + "message": "Report type 301 does not support account ID of type class com.amazon.partner.account.id.VendorGroupId.", + "details": ""} + ] + } + http_mocker.post( + _create_report_request(stream_name).build(), + response_with_status(status_code=HTTPStatus.BAD_REQUEST, body=response_body), + ) + + warning_message = ( + "The endpoint https://sellingpartnerapi-na.amazon.com/reports/2021-06-30/reports returned 400: " + "Report type 301 does not support account ID of type class com.amazon.partner.account.id.VendorGroupId.." + " This is most likely due to account type (Vendor) on the credentials in use." + " Try to re-authenticate with Seller account type and sync again." + ) + + output = self._read(stream_name, config()) + assert_message_in_log_output(warning_message, output) + assert len(output.records) == 0 + @freezegun.freeze_time(NOW.isoformat()) class TestIncremental: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py index 429704349743..9744d3f99d50 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py @@ -138,7 +138,7 @@ def test_read_records_retrieve_fatal(self, report_init_kwargs, mocker, requests_ ) assert e.value.internal_message == ( f"Failed to retrieve the report 'GET_TEST_REPORT' for period {stream_start}-{stream_end}. " - "This will be read during the next sync. Error: Failed to retrieve the report result document." + "This will be read during the next sync. Report ID: some_report_id. Error: Failed to retrieve the report result document." ) def test_read_records_retrieve_cancelled(self, report_init_kwargs, mocker, requests_mock, caplog): diff --git a/airbyte-integrations/connectors/source-clazar/README.md b/airbyte-integrations/connectors/source-clazar/README.md new file mode 100644 index 000000000000..370be69c0844 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/README.md @@ -0,0 +1,103 @@ +# Clazar Source + +This is the repository for the Clazar configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/clazar). + +## Local development + +### Prerequisites + +* Python (`^3.9`) +* Poetry (`^1.7`) - installation instructions [here](https://python-poetry.org/docs/#installation) + + + +### Installing the connector + +From this connector directory, run: +```bash +poetry install --with dev +``` + + +### Create credentials + +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/clazar) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `src/source_clazar/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector + +``` +poetry run source-clazar spec +poetry run source-clazar check --config secrets/config.json +poetry run source-clazar discover --config secrets/config.json +poetry run source-clazar read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Running tests + +To run tests locally, from the connector directory run: + +``` +poetry run pytest tests +``` + +### Building the docker image + +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-clazar build +``` + +An image will be available on your host with the tag `airbyte/source-clazar:dev`. + + +### Running as a docker container + +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-clazar:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clazar:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clazar:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-clazar:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +### Running our CI test suite + +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-clazar test +``` + +### Customizing acceptance Tests + +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +### Dependency Management + +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector + +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-clazar test` +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` +3. Make sure the `metadata.yaml` content is up to date. +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/clazar.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-clazar/__init__.py b/airbyte-integrations/connectors/source-clazar/__init__.py new file mode 100644 index 000000000000..66f6de8cb2bb --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-clazar/icon.svg b/airbyte-integrations/connectors/source-clazar/icon.svg new file mode 100644 index 000000000000..e7c2546fccd4 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-integrations/connectors/source-clazar/integration_tests/__init__.py b/airbyte-integrations/connectors/source-clazar/integration_tests/__init__.py new file mode 100644 index 000000000000..66f6de8cb2bb --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py new file mode 100644 index 000000000000..efc25f08ce82 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-clazar/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-clazar/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..9cc43d1255d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/integration_tests/configured_catalog.json @@ -0,0 +1,204 @@ +{ + "streams": [ + { + "stream": { + "name": "listings", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "buyers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contracts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "opportunities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "private_offers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_aws_marketplace_disbursements", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_aws_marketplace_revenue", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_aws_cosell_opportunities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_azure_marketplace_revenue", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_azure_marketplace_customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_azure_marketplace_orders", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_azure_marketplace_metered_usage", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_azure_cosell_opportunities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_gcp_marketplace_disbursements", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_gcp_marketplace_disbursements_summary", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_gcp_marketplace_charges_and_usage", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_gcp_marketplace_daily_insights", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_gcp_marketplace_incremental_daily_insights", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_gcp_marketplace_monthly_insights", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytics_gcp_marketplace_incremental_monthly_insights", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-clazar/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-clazar/integration_tests/invalid_config.json new file mode 100644 index 000000000000..dded178c3f19 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "client_id": "", + "client_secret": "" +} diff --git a/airbyte-integrations/connectors/source-clazar/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-clazar/integration_tests/sample_config.json new file mode 100644 index 000000000000..dded178c3f19 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "client_id": "", + "client_secret": "" +} diff --git a/airbyte-integrations/connectors/source-clazar/main.py b/airbyte-integrations/connectors/source-clazar/main.py new file mode 100644 index 000000000000..82736763be59 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_clazar import SourceClazar + +if __name__ == "__main__": + source = SourceClazar() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-clazar/metadata.yaml b/airbyte-integrations/connectors/source-clazar/metadata.yaml new file mode 100644 index 000000000000..71b823c05009 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/metadata.yaml @@ -0,0 +1,32 @@ +data: + allowedHosts: + hosts: + - api.clazar.io + registries: + oss: + enabled: true + cloud: + enabled: true + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-clazar + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.2@sha256:57703de3b4c4204bd68a7b13c9300f8e03c0189bffddaffc796f1da25d2dbea0 + connectorSubtype: api + connectorType: source + definitionId: d7df7b64-6266-45b5-ad83-e1515578f371 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-clazar + githubIssueLabel: source-clazar + icon: clazar.svg + license: MIT + name: Clazar + releaseDate: 2024-06-27 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/clazar + tags: + - language:python + - cdk:low-code +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-clazar/poetry.lock b/airbyte-integrations/connectors/source-clazar/poetry.lock new file mode 100644 index 000000000000..06b1660e1004 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/poetry.lock @@ -0,0 +1,1380 @@ +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "2.0.0" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "airbyte_cdk-2.0.0-py3-none-any.whl", hash = "sha256:59821cbc5dc37a5fefa09ed0bd37a539be8cd9d4b2d97ceba355591256e6ecb7"}, + {file = "airbyte_cdk-2.0.0.tar.gz", hash = "sha256:8bd4fb25a13051a0d5d99373f73d4b2dbae2dfdc05b56b3ec0c68e31f8a0b519"}, +] + +[package.dependencies] +airbyte-protocol-models-pdv2 = ">=0.12.2,<0.13.0" +backoff = "*" +cachetools = "*" +cryptography = ">=42.0.5,<43.0.0" +Deprecated = ">=1.2,<1.3" +dpath = ">=2.1.6,<3.0.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<0.3" +jsonschema = ">=3.2.0,<3.3.0" +langchain_core = "0.1.42" +pendulum = "<3.0.0" +pydantic = ">=2.7,<3.0" +pyjwt = ">=2.8.0,<3.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +pytz = "2024.1" +PyYAML = ">=6.0.1,<7.0.0" +requests = "*" +requests_cache = "*" +wcmatch = "8.4" + +[package.extras] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models-pdv2" +version = "0.12.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models_pdv2-0.12.2-py3-none-any.whl", hash = "sha256:8b3f9d0388928547cdf2e9134c0d589e4bcaa6f63bf71a21299f6824bfb7ad0e"}, + {file = "airbyte_protocol_models_pdv2-0.12.2.tar.gz", hash = "sha256:130c9ab289f3f53749ce63ff1abbfb67a44b7e5bd2794865315a2976138b672b"}, +] + +[package.dependencies] +pydantic = ">=2.7.2,<3.0.0" + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.3" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.6.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "42.0.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.2.0" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576"}, + {file = "dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.4" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpointer" +version = "3.0.0" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"}, + {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"}, +] + +[[package]] +name = "jsonref" +version = "0.2" +description = "An implementation of JSON Reference for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonref-0.2-py3-none-any.whl", hash = "sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f"}, + {file = "jsonref-0.2.tar.gz", hash = "sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "langchain-core" +version = "0.1.42" +description = "Building applications with LLMs through composability" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langchain_core-0.1.42-py3-none-any.whl", hash = "sha256:c5653ffa08a44f740295c157a24c0def4a753333f6a2c41f76bf431cd00be8b5"}, + {file = "langchain_core-0.1.42.tar.gz", hash = "sha256:40751bf60ea5d8e2b2efe65290db434717ee3834870c002e40e2811f09d814e6"}, +] + +[package.dependencies] +jsonpatch = ">=1.33,<2.0" +langsmith = ">=0.1.0,<0.2.0" +packaging = ">=23.2,<24.0" +pydantic = ">=1,<3" +PyYAML = ">=5.3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +extended-testing = ["jinja2 (>=3,<4)"] + +[[package]] +name = "langsmith" +version = "0.1.83" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.83-py3-none-any.whl", hash = "sha256:f54d8cd8479b648b6339f3f735d19292c3516d080f680933ecdca3eab4b67ed3"}, + {file = "langsmith-0.1.83.tar.gz", hash = "sha256:5cdd947212c8ad19adb992c06471c860185a777daa6859bb47150f90daf64bf3"}, +] + +[package.dependencies] +orjson = ">=3.9.14,<4.0.0" +pydantic = {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""} +requests = ">=2,<3" + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "orjson" +version = "3.10.5" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b"}, + {file = "orjson-3.10.5-cp310-none-win32.whl", hash = "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2"}, + {file = "orjson-3.10.5-cp310-none-win_amd64.whl", hash = "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228"}, + {file = "orjson-3.10.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa"}, + {file = "orjson-3.10.5-cp311-none-win32.whl", hash = "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04"}, + {file = "orjson-3.10.5-cp311-none-win_amd64.whl", hash = "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c"}, + {file = "orjson-3.10.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3"}, + {file = "orjson-3.10.5-cp312-none-win32.whl", hash = "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2"}, + {file = "orjson-3.10.5-cp312-none-win_amd64.whl", hash = "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5"}, + {file = "orjson-3.10.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b"}, + {file = "orjson-3.10.5-cp38-none-win32.whl", hash = "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4"}, + {file = "orjson-3.10.5-cp38-none-win_amd64.whl", hash = "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09"}, + {file = "orjson-3.10.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86"}, + {file = "orjson-3.10.5-cp39-none-win32.whl", hash = "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47"}, + {file = "orjson-3.10.5-cp39-none-win_amd64.whl", hash = "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7"}, + {file = "orjson-3.10.5.tar.gz", hash = "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + +[[package]] +name = "pydantic" +version = "2.8.0" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.8.0-py3-none-any.whl", hash = "sha256:ead4f3a1e92386a734ca1411cb25d94147cf8778ed5be6b56749047676d6364e"}, + {file = "pydantic-2.8.0.tar.gz", hash = "sha256:d970ffb9d030b710795878940bd0489842c638e7252fc4a19c3ae2f7da4d6141"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.20.0" +typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.20.0" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:e9dcd7fb34f7bfb239b5fa420033642fff0ad676b765559c3737b91f664d4fa9"}, + {file = "pydantic_core-2.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:649a764d9b0da29816889424697b2a3746963ad36d3e0968784ceed6e40c6355"}, + {file = "pydantic_core-2.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7701df088d0b05f3460f7ba15aec81ac8b0fb5690367dfd072a6c38cf5b7fdb5"}, + {file = "pydantic_core-2.20.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ab760f17c3e792225cdaef31ca23c0aea45c14ce80d8eff62503f86a5ab76bff"}, + {file = "pydantic_core-2.20.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cb1ad5b4d73cde784cf64580166568074f5ccd2548d765e690546cff3d80937d"}, + {file = "pydantic_core-2.20.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b81ec2efc04fc1dbf400647d4357d64fb25543bae38d2d19787d69360aad21c9"}, + {file = "pydantic_core-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4a9732a5cad764ba37f3aa873dccb41b584f69c347a57323eda0930deec8e10"}, + {file = "pydantic_core-2.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6dc85b9e10cc21d9c1055f15684f76fa4facadddcb6cd63abab702eb93c98943"}, + {file = "pydantic_core-2.20.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:21d9f7e24f63fdc7118e6cc49defaab8c1d27570782f7e5256169d77498cf7c7"}, + {file = "pydantic_core-2.20.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8b315685832ab9287e6124b5d74fc12dda31e6421d7f6b08525791452844bc2d"}, + {file = "pydantic_core-2.20.0-cp310-none-win32.whl", hash = "sha256:c3dc8ec8b87c7ad534c75b8855168a08a7036fdb9deeeed5705ba9410721c84d"}, + {file = "pydantic_core-2.20.0-cp310-none-win_amd64.whl", hash = "sha256:85770b4b37bb36ef93a6122601795231225641003e0318d23c6233c59b424279"}, + {file = "pydantic_core-2.20.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:58e251bb5a5998f7226dc90b0b753eeffa720bd66664eba51927c2a7a2d5f32c"}, + {file = "pydantic_core-2.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:78d584caac52c24240ef9ecd75de64c760bbd0e20dbf6973631815e3ef16ef8b"}, + {file = "pydantic_core-2.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5084ec9721f82bef5ff7c4d1ee65e1626783abb585f8c0993833490b63fe1792"}, + {file = "pydantic_core-2.20.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6d0f52684868db7c218437d260e14d37948b094493f2646f22d3dda7229bbe3f"}, + {file = "pydantic_core-2.20.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1def125d59a87fe451212a72ab9ed34c118ff771e5473fef4f2f95d8ede26d75"}, + {file = "pydantic_core-2.20.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b34480fd6778ab356abf1e9086a4ced95002a1e195e8d2fd182b0def9d944d11"}, + {file = "pydantic_core-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d42669d319db366cb567c3b444f43caa7ffb779bf9530692c6f244fc635a41eb"}, + {file = "pydantic_core-2.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:53b06aea7a48919a254b32107647be9128c066aaa6ee6d5d08222325f25ef175"}, + {file = "pydantic_core-2.20.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1f038156b696a1c39d763b2080aeefa87ddb4162c10aa9fabfefffc3dd8180fa"}, + {file = "pydantic_core-2.20.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3f0f3a4a23717280a5ee3ac4fb1f81d6fde604c9ec5100f7f6f987716bb8c137"}, + {file = "pydantic_core-2.20.0-cp311-none-win32.whl", hash = "sha256:316fe7c3fec017affd916a0c83d6f1ec697cbbbdf1124769fa73328e7907cc2e"}, + {file = "pydantic_core-2.20.0-cp311-none-win_amd64.whl", hash = "sha256:2d06a7fa437f93782e3f32d739c3ec189f82fca74336c08255f9e20cea1ed378"}, + {file = "pydantic_core-2.20.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:d6f8c49657f3eb7720ed4c9b26624063da14937fc94d1812f1e04a2204db3e17"}, + {file = "pydantic_core-2.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad1bd2f377f56fec11d5cfd0977c30061cd19f4fa199bf138b200ec0d5e27eeb"}, + {file = "pydantic_core-2.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed741183719a5271f97d93bbcc45ed64619fa38068aaa6e90027d1d17e30dc8d"}, + {file = "pydantic_core-2.20.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d82e5ed3a05f2dcb89c6ead2fd0dbff7ac09bc02c1b4028ece2d3a3854d049ce"}, + {file = "pydantic_core-2.20.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2ba34a099576234671f2e4274e5bc6813b22e28778c216d680eabd0db3f7dad"}, + {file = "pydantic_core-2.20.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:879ae6bb08a063b3e1b7ac8c860096d8fd6b48dd9b2690b7f2738b8c835e744b"}, + {file = "pydantic_core-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b0eefc7633a04c0694340aad91fbfd1986fe1a1e0c63a22793ba40a18fcbdc8"}, + {file = "pydantic_core-2.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73deadd6fd8a23e2f40b412b3ac617a112143c8989a4fe265050fd91ba5c0608"}, + {file = "pydantic_core-2.20.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:35681445dc85446fb105943d81ae7569aa7e89de80d1ca4ac3229e05c311bdb1"}, + {file = "pydantic_core-2.20.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0f6dd3612a3b9f91f2e63924ea18a4476656c6d01843ca20a4c09e00422195af"}, + {file = "pydantic_core-2.20.0-cp312-none-win32.whl", hash = "sha256:7e37b6bb6e90c2b8412b06373c6978d9d81e7199a40e24a6ef480e8acdeaf918"}, + {file = "pydantic_core-2.20.0-cp312-none-win_amd64.whl", hash = "sha256:7d4df13d1c55e84351fab51383520b84f490740a9f1fec905362aa64590b7a5d"}, + {file = "pydantic_core-2.20.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:d43e7ab3b65e4dc35a7612cfff7b0fd62dce5bc11a7cd198310b57f39847fd6c"}, + {file = "pydantic_core-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b6a24d7b5893392f2b8e3b7a0031ae3b14c6c1942a4615f0d8794fdeeefb08b"}, + {file = "pydantic_core-2.20.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b2f13c3e955a087c3ec86f97661d9f72a76e221281b2262956af381224cfc243"}, + {file = "pydantic_core-2.20.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:72432fd6e868c8d0a6849869e004b8bcae233a3c56383954c228316694920b38"}, + {file = "pydantic_core-2.20.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d70a8ff2d4953afb4cbe6211f17268ad29c0b47e73d3372f40e7775904bc28fc"}, + {file = "pydantic_core-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e49524917b8d3c2f42cd0d2df61178e08e50f5f029f9af1f402b3ee64574392"}, + {file = "pydantic_core-2.20.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4f0f71653b1c1bad0350bc0b4cc057ab87b438ff18fa6392533811ebd01439c"}, + {file = "pydantic_core-2.20.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:16197e6f4fdecb9892ed2436e507e44f0a1aa2cff3b9306d1c879ea2f9200997"}, + {file = "pydantic_core-2.20.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:763602504bf640b3ded3bba3f8ed8a1cc2fc6a87b8d55c1c5689f428c49c947e"}, + {file = "pydantic_core-2.20.0-cp313-none-win32.whl", hash = "sha256:a3f243f318bd9523277fa123b3163f4c005a3e8619d4b867064de02f287a564d"}, + {file = "pydantic_core-2.20.0-cp313-none-win_amd64.whl", hash = "sha256:03aceaf6a5adaad3bec2233edc5a7905026553916615888e53154807e404545c"}, + {file = "pydantic_core-2.20.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d6f2d8b8da1f03f577243b07bbdd3412eee3d37d1f2fd71d1513cbc76a8c1239"}, + {file = "pydantic_core-2.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a272785a226869416c6b3c1b7e450506152d3844207331f02f27173562c917e0"}, + {file = "pydantic_core-2.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efbb412d55a4ffe73963fed95c09ccb83647ec63b711c4b3752be10a56f0090b"}, + {file = "pydantic_core-2.20.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e4f46189d8740561b43655263a41aac75ff0388febcb2c9ec4f1b60a0ec12f3"}, + {file = "pydantic_core-2.20.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87d3df115f4a3c8c5e4d5acf067d399c6466d7e604fc9ee9acbe6f0c88a0c3cf"}, + {file = "pydantic_core-2.20.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a340d2bdebe819d08f605e9705ed551c3feb97e4fd71822d7147c1e4bdbb9508"}, + {file = "pydantic_core-2.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:616b9c2f882393d422ba11b40e72382fe975e806ad693095e9a3b67c59ea6150"}, + {file = "pydantic_core-2.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:25c46bb2ff6084859bbcfdf4f1a63004b98e88b6d04053e8bf324e115398e9e7"}, + {file = "pydantic_core-2.20.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:23425eccef8f2c342f78d3a238c824623836c6c874d93c726673dbf7e56c78c0"}, + {file = "pydantic_core-2.20.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:52527e8f223ba29608d999d65b204676398009725007c9336651c2ec2d93cffc"}, + {file = "pydantic_core-2.20.0-cp38-none-win32.whl", hash = "sha256:1c3c5b7f70dd19a6845292b0775295ea81c61540f68671ae06bfe4421b3222c2"}, + {file = "pydantic_core-2.20.0-cp38-none-win_amd64.whl", hash = "sha256:8093473d7b9e908af1cef30025609afc8f5fd2a16ff07f97440fd911421e4432"}, + {file = "pydantic_core-2.20.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:ee7785938e407418795e4399b2bf5b5f3cf6cf728077a7f26973220d58d885cf"}, + {file = "pydantic_core-2.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0e75794883d635071cf6b4ed2a5d7a1e50672ab7a051454c76446ef1ebcdcc91"}, + {file = "pydantic_core-2.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:344e352c96e53b4f56b53d24728217c69399b8129c16789f70236083c6ceb2ac"}, + {file = "pydantic_core-2.20.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:978d4123ad1e605daf1ba5e01d4f235bcf7b6e340ef07e7122e8e9cfe3eb61ab"}, + {file = "pydantic_core-2.20.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c05eaf6c863781eb834ab41f5963604ab92855822a2062897958089d1335dad"}, + {file = "pydantic_core-2.20.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc7e43b4a528ffca8c9151b6a2ca34482c2fdc05e6aa24a84b7f475c896fc51d"}, + {file = "pydantic_core-2.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:658287a29351166510ebbe0a75c373600cc4367a3d9337b964dada8d38bcc0f4"}, + {file = "pydantic_core-2.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1dacf660d6de692fe351e8c806e7efccf09ee5184865893afbe8e59be4920b4a"}, + {file = "pydantic_core-2.20.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3e147fc6e27b9a487320d78515c5f29798b539179f7777018cedf51b7749e4f4"}, + {file = "pydantic_core-2.20.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c867230d715a3dd1d962c8d9bef0d3168994ed663e21bf748b6e3a529a129aab"}, + {file = "pydantic_core-2.20.0-cp39-none-win32.whl", hash = "sha256:22b813baf0dbf612752d8143a2dbf8e33ccb850656b7850e009bad2e101fc377"}, + {file = "pydantic_core-2.20.0-cp39-none-win_amd64.whl", hash = "sha256:3a7235b46c1bbe201f09b6f0f5e6c36b16bad3d0532a10493742f91fbdc8035f"}, + {file = "pydantic_core-2.20.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cafde15a6f7feaec2f570646e2ffc5b73412295d29134a29067e70740ec6ee20"}, + {file = "pydantic_core-2.20.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:2aec8eeea0b08fd6bc2213d8e86811a07491849fd3d79955b62d83e32fa2ad5f"}, + {file = "pydantic_core-2.20.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:840200827984f1c4e114008abc2f5ede362d6e11ed0b5931681884dd41852ff1"}, + {file = "pydantic_core-2.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8ea1d8b7df522e5ced34993c423c3bf3735c53df8b2a15688a2f03a7d678800"}, + {file = "pydantic_core-2.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d5b8376a867047bf08910573deb95d3c8dfb976eb014ee24f3b5a61ccc5bee1b"}, + {file = "pydantic_core-2.20.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d08264b4460326cefacc179fc1411304d5af388a79910832835e6f641512358b"}, + {file = "pydantic_core-2.20.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7a3639011c2e8a9628466f616ed7fb413f30032b891898e10895a0a8b5857d6c"}, + {file = "pydantic_core-2.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:05e83ce2f7eba29e627dd8066aa6c4c0269b2d4f889c0eba157233a353053cea"}, + {file = "pydantic_core-2.20.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:603a843fea76a595c8f661cd4da4d2281dff1e38c4a836a928eac1a2f8fe88e4"}, + {file = "pydantic_core-2.20.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac76f30d5d3454f4c28826d891fe74d25121a346c69523c9810ebba43f3b1cec"}, + {file = "pydantic_core-2.20.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e3b1d4b1b3f6082849f9b28427ef147a5b46a6132a3dbaf9ca1baa40c88609"}, + {file = "pydantic_core-2.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2761f71faed820e25ec62eacba670d1b5c2709bb131a19fcdbfbb09884593e5a"}, + {file = "pydantic_core-2.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a0586cddbf4380e24569b8a05f234e7305717cc8323f50114dfb2051fcbce2a3"}, + {file = "pydantic_core-2.20.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:b8c46a8cf53e849eea7090f331ae2202cd0f1ceb090b00f5902c423bd1e11805"}, + {file = "pydantic_core-2.20.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b4a085bd04af7245e140d1b95619fe8abb445a3d7fdf219b3f80c940853268ef"}, + {file = "pydantic_core-2.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:116b326ac82c8b315e7348390f6d30bcfe6e688a7d3f1de50ff7bcc2042a23c2"}, + {file = "pydantic_core-2.20.0.tar.gz", hash = "sha256:366be8e64e0cb63d87cf79b4e1765c0703dd6313c729b22e7b9e378db6b96877"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "8.2.2" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"}, + {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "requests-mock" +version = "1.12.1" +description = "Mock out responses from the requests package" +optional = false +python-versions = ">=3.5" +files = [ + {file = "requests-mock-1.12.1.tar.gz", hash = "sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401"}, + {file = "requests_mock-1.12.1-py2.py3-none-any.whl", hash = "sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563"}, +] + +[package.dependencies] +requests = ">=2.22,<3" + +[package.extras] +fixture = ["fixtures"] + +[[package]] +name = "setuptools" +version = "70.2.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-70.2.0-py3-none-any.whl", hash = "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05"}, + {file = "setuptools-70.2.0.tar.gz", hash = "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1"}, +] + +[package.extras] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "tenacity" +version = "8.4.2" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"}, + {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "97ce1f66d04dbb8985f8429b11898cebe7680f4cbe9a8ecb40f79bfdfef26deb" diff --git a/airbyte-integrations/connectors/source-clazar/pyproject.toml b/airbyte-integrations/connectors/source-clazar/pyproject.toml new file mode 100644 index 000000000000..e6f2b79c3879 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.1.0" +name = "source-clazar" +description = "Source implementation for clazar." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/clazar" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +packages = [ { include = "source_clazar" }, {include = "main.py" } ] + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "< 3.0.0" + +[tool.poetry.scripts] +source-clazar = "source_clazar.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "*" +pytest-mock = "*" +pytest = "*" diff --git a/airbyte-integrations/connectors/source-clazar/source_clazar/__init__.py b/airbyte-integrations/connectors/source-clazar/source_clazar/__init__.py new file mode 100644 index 000000000000..d89aafb4582a --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/source_clazar/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceClazar + +__all__ = ["SourceClazar"] diff --git a/airbyte-integrations/connectors/source-clazar/source_clazar/manifest.yaml b/airbyte-integrations/connectors/source-clazar/source_clazar/manifest.yaml new file mode 100644 index 000000000000..75eb03d109d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/source_clazar/manifest.yaml @@ -0,0 +1,4497 @@ +version: 2.0.0 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - listings + - analytics_aws_marketplace_disbursements + +definitions: + streams: + buyers: + type: DeclarativeStream + name: buyers + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 100 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /buyers + http_method: GET + request_parameters: + response_format: common + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/buyers" + listings: + type: DeclarativeStream + name: listings + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 100 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /listings + http_method: GET + request_parameters: + response_format: common + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/listings" + contracts: + type: DeclarativeStream + name: contracts + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 100 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /contracts + http_method: GET + request_parameters: + response_format: common + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/contracts" + opportunities: + type: DeclarativeStream + name: opportunities + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 100 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /opportunities + http_method: GET + request_parameters: + response_format: common + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/opportunities" + private_offers: + type: DeclarativeStream + name: private_offers + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 100 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /private_offers + http_method: GET + request_parameters: + response_format: common + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/private_offers" + analytics_aws_marketplace_revenue: + type: DeclarativeStream + name: analytics_aws_marketplace_revenue + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/aws_marketplace_revenue + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_aws_marketplace_revenue" + analytics_aws_cosell_opportunities: + type: DeclarativeStream + name: analytics_aws_cosell_opportunities + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/aws_cosell_opportunities + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_aws_cosell_opportunities" + analytics_azure_marketplace_orders: + type: DeclarativeStream + name: analytics_azure_marketplace_orders + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/azure_marketplace_orders + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_azure_marketplace_orders" + analytics_azure_marketplace_revenue: + type: DeclarativeStream + name: analytics_azure_marketplace_revenue + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/azure_marketplace_revenue + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_azure_marketplace_revenue" + analytics_azure_cosell_opportunities: + type: DeclarativeStream + name: analytics_azure_cosell_opportunities + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/azure_cosell_opportunities + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_azure_cosell_opportunities" + analytics_azure_marketplace_customers: + type: DeclarativeStream + name: analytics_azure_marketplace_customers + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/azure_marketplace_customers + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_azure_marketplace_customers" + analytics_aws_marketplace_disbursements: + type: DeclarativeStream + name: analytics_aws_marketplace_disbursements + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/aws_marketplace_disbursements + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_aws_marketplace_disbursements" + analytics_gcp_marketplace_disbursements: + type: DeclarativeStream + name: analytics_gcp_marketplace_disbursements + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/gcp_marketplace_disbursements + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_gcp_marketplace_disbursements" + analytics_gcp_marketplace_daily_insights: + type: DeclarativeStream + name: analytics_gcp_marketplace_daily_insights + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/gcp_marketplace_daily_insights + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_gcp_marketplace_daily_insights" + analytics_azure_marketplace_metered_usage: + type: DeclarativeStream + name: analytics_azure_marketplace_metered_usage + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/azure_marketplace_metered_usage + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_azure_marketplace_metered_usage" + analytics_gcp_marketplace_monthly_insights: + type: DeclarativeStream + name: analytics_gcp_marketplace_monthly_insights + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/gcp_marketplace_monthly_insights + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_gcp_marketplace_monthly_insights" + analytics_gcp_marketplace_charges_and_usage: + type: DeclarativeStream + name: analytics_gcp_marketplace_charges_and_usage + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/gcp_marketplace_charges_and_usage + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_gcp_marketplace_charges_and_usage" + analytics_gcp_marketplace_disbursements_summary: + type: DeclarativeStream + name: analytics_gcp_marketplace_disbursements_summary + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/gcp_marketplace_disbursements_summary + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_gcp_marketplace_disbursements_summary" + analytics_gcp_marketplace_incremental_daily_insights: + type: DeclarativeStream + name: analytics_gcp_marketplace_incremental_daily_insights + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/gcp_marketplace_incremental_daily_insights + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_gcp_marketplace_incremental_daily_insights" + analytics_gcp_marketplace_incremental_monthly_insights: + type: DeclarativeStream + name: analytics_gcp_marketplace_incremental_monthly_insights + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 5000 + start_from_page: 1 + inject_on_first_request: true + requester: + $ref: "#/definitions/base_requester" + path: /analytics/datasets/gcp_marketplace_incremental_monthly_insights + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/analytics_gcp_marketplace_incremental_monthly_insights" + base_requester: + type: HttpRequester + url_base: https://api.clazar.io + authenticator: + type: OAuthAuthenticator + client_id: '{{ config["client_id"] }}' + grant_type: client_credentials + client_secret: '{{ config["client_secret"] }}' + refresh_request_body: {} + token_refresh_endpoint: https://api.clazar.io/authenticate/ + +streams: + - $ref: "#/definitions/streams/listings" + - $ref: "#/definitions/streams/buyers" + - $ref: "#/definitions/streams/contracts" + - $ref: "#/definitions/streams/opportunities" + - $ref: "#/definitions/streams/private_offers" + - $ref: "#/definitions/streams/analytics_aws_marketplace_disbursements" + - $ref: "#/definitions/streams/analytics_aws_marketplace_revenue" + - $ref: "#/definitions/streams/analytics_aws_cosell_opportunities" + - $ref: "#/definitions/streams/analytics_azure_marketplace_revenue" + - $ref: "#/definitions/streams/analytics_azure_marketplace_customers" + - $ref: "#/definitions/streams/analytics_azure_marketplace_orders" + - $ref: "#/definitions/streams/analytics_azure_marketplace_metered_usage" + - $ref: "#/definitions/streams/analytics_azure_cosell_opportunities" + - $ref: "#/definitions/streams/analytics_gcp_marketplace_disbursements" + - $ref: "#/definitions/streams/analytics_gcp_marketplace_disbursements_summary" + - $ref: "#/definitions/streams/analytics_gcp_marketplace_charges_and_usage" + - $ref: "#/definitions/streams/analytics_gcp_marketplace_daily_insights" + - $ref: "#/definitions/streams/analytics_gcp_marketplace_incremental_daily_insights" + - $ref: "#/definitions/streams/analytics_gcp_marketplace_monthly_insights" + - $ref: >- + #/definitions/streams/analytics_gcp_marketplace_incremental_monthly_insights + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - client_id + - client_secret + properties: + client_id: + type: string + order: 0 + title: Client ID + airbyte_secret: true + client_secret: + type: string + order: 1 + title: Client secret + airbyte_secret: true + additionalProperties: true + +metadata: + autoImportSchema: + buyers: true + listings: true + contracts: true + opportunities: true + private_offers: true + analytics_aws_marketplace_revenue: true + analytics_aws_cosell_opportunities: true + analytics_azure_marketplace_orders: true + analytics_azure_marketplace_revenue: true + analytics_azure_cosell_opportunities: true + analytics_azure_marketplace_customers: true + analytics_aws_marketplace_disbursements: true + analytics_gcp_marketplace_disbursements: true + analytics_gcp_marketplace_daily_insights: true + analytics_azure_marketplace_metered_usage: true + analytics_gcp_marketplace_monthly_insights: true + analytics_gcp_marketplace_charges_and_usage: true + analytics_gcp_marketplace_disbursements_summary: true + analytics_gcp_marketplace_incremental_daily_insights: true + analytics_gcp_marketplace_incremental_monthly_insights: true + +schemas: + buyers: + type: object + $schema: http://json-schema.org/schema# + required: + - id + properties: + metadata: + type: + - object + - "null" + id: + type: string + name: + type: + - string + - "null" + cloud: + type: + - string + - "null" + domain: + type: + - string + - "null" + status: + type: + - string + - "null" + listing_id: + type: + - string + - "null" + cloud_account_id: + type: + - string + - "null" + cloud_identifiers: + type: + - object + - "null" + properties: + tenant_id: + type: + - string + - "null" + account_id: + type: + - string + - "null" + product_code: + type: + - string + - "null" + aws_account_id: + type: + - string + - "null" + customer_identifier: + type: + - string + - "null" + custom_properties: + type: + - object + - "null" + latest_contract_id: + type: + - string + - "null" + registration_details: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + field: + type: + - string + - "null" + value: + type: + - string + - "null" + external_object_associations: + type: + - object + - "null" + properties: + orb: + type: + - object + - "null" + properties: + mapped_object: + type: + - object + - "null" + salesforce: + type: + - object + - "null" + properties: + writeback_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + url: + type: + - string + - "null" + object_type: + type: + - string + - "null" + clazar_created: + type: + - boolean + - "null" + last_writeback_at: + type: + - string + - "null" + additionalProperties: true + listings: + type: object + $schema: http://json-schema.org/schema# + required: + - id + properties: + metadata: + type: + - object + - "null" + id: + type: string + cloud: + type: + - string + - "null" + title: + type: + - string + - "null" + status: + type: + - string + - "null" + cloud_id: + type: + - string + - "null" + eula_url: + type: + - string + - "null" + cloud_url: + type: + - string + - "null" + eula_type: + type: + - string + - "null" + long_description: + type: + - string + - "null" + short_description: + type: + - string + - "null" + additionalProperties: true + contracts: + type: object + $schema: http://json-schema.org/schema# + required: + - id + properties: + metadata: + type: + - object + - "null" + id: + type: string + cloud: + type: + - string + - "null" + end_at: + type: + - string + - "null" + status: + type: + - string + - "null" + buyer_id: + type: + - string + - "null" + cloud_id: + type: + - string + - "null" + duration: + type: + - string + - "null" + start_at: + type: + - string + - "null" + cloud_url: + type: + - string + - "null" + auto_renew: + type: + - boolean + - "null" + listing_id: + type: + - string + - "null" + accepted_at: + type: + - string + - "null" + latest_offer_id: + type: + - string + - "null" + cloud_identifiers: + type: + - object + - "null" + properties: + product_code: + type: + - string + - "null" + aws_account_id: + type: + - string + - "null" + entitlement_id: + type: + - string + - "null" + customer_identifier: + type: + - string + - "null" + saas_subscription_id: + type: + - string + - "null" + custom_properties: + type: + - object + - "null" + external_object_associations: + type: + - object + - "null" + properties: + orb: + type: + - object + - "null" + properties: + mapped_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + salesforce: + type: + - object + - "null" + properties: + writeback_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + url: + type: + - string + - "null" + object_type: + type: + - string + - "null" + clazar_created: + type: + - boolean + - "null" + last_writeback_at: + type: + - string + - "null" + additionalProperties: true + opportunities: + type: object + $schema: http://json-schema.org/schema# + required: + - id + properties: + type: + type: + - string + - "null" + metadata: + type: + - object + - "null" + id: + type: string + cloud: + type: + - string + - "null" + stage: + type: + - string + - "null" + title: + type: + - string + - "null" + status: + type: + - string + - "null" + cloud_id: + type: + - string + - "null" + accept_by: + type: + - string + - "null" + cloud_url: + type: + - string + - "null" + published_at: + type: + - string + - "null" + customer_company: + type: + - string + - "null" + customer_website: + type: + - string + - "null" + last_modified_at: + type: + - string + - "null" + custom_properties: + type: + - object + - "null" + target_close_date: + type: + - string + - "null" + external_object_associations: + type: + - object + - "null" + properties: + hubspot: + type: + - object + - "null" + properties: + mapped_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + url: + type: + - string + - "null" + name: + type: + - string + - "null" + object_type: + type: + - string + - "null" + writeback_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + url: + type: + - string + - "null" + object_type: + type: + - string + - "null" + clazar_created: + type: + - boolean + - "null" + last_writeback_at: + type: + - string + - "null" + salesforce: + type: + - object + - "null" + properties: + mapped_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + url: + type: + - string + - "null" + name: + type: + - string + - "null" + object_type: + type: + - string + - "null" + writeback_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + url: + type: + - string + - "null" + object_type: + type: + - string + - "null" + clazar_created: + type: + - boolean + - "null" + last_writeback_at: + type: + - string + - "null" + additionalProperties: true + private_offers: + type: object + $schema: http://json-schema.org/schema# + required: + - id + properties: + metadata: + type: + - object + - "null" + id: + type: string + name: + type: + - string + - "null" + cloud: + type: + - string + - "null" + status: + type: + - string + - "null" + archived: + type: + - string + - "null" + cloud_id: + type: + - string + - "null" + duration: + type: + - string + - "null" + eula_url: + type: + - string + - "null" + cloud_url: + type: + - string + - "null" + eula_type: + type: + - string + - "null" + listing_id: + type: + - string + - "null" + offer_type: + type: + - string + - "null" + accepted_at: + type: + - string + - "null" + published_at: + type: + - string + - "null" + expiration_at: + type: + - string + - "null" + custom_properties: + type: + - object + - "null" + external_object_associations: + type: + - object + - "null" + properties: + hubspot: + type: + - object + - "null" + properties: + mapped_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + url: + type: + - string + - "null" + name: + type: + - string + - "null" + object_type: + type: + - string + - "null" + writeback_object: + type: + - object + - "null" + salesforce: + type: + - object + - "null" + properties: + mapped_object: + type: + - object + - "null" + writeback_object: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + url: + type: + - string + - "null" + object_type: + type: + - string + - "null" + clazar_created: + type: + - boolean + - "null" + last_writeback_at: + type: + - string + - "null" + additionalProperties: true + analytics_aws_marketplace_revenue: + type: object + $schema: http://json-schema.org/schema# + properties: + currency: + type: + - string + - "null" + offer_id: + type: + - string + - "null" + seller_id: + type: + - string + - "null" + invoice_id: + type: + - string + - "null" + offer_name: + type: + - string + - "null" + payer_city: + type: + - string + - "null" + product_id: + type: + - string + - "null" + listing_fee: + type: + - number + - "null" + agreement_id: + type: + - string + - "null" + gross_refund: + type: + - number + - "null" + invoice_date: + type: + - string + - "null" + product_code: + type: + - string + - "null" + aws_tax_share: + type: + - number + - "null" + gross_revenue: + type: + - number + - "null" + payer_country: + type: + - string + - "null" + payment_terms: + type: + - string + - "null" + product_title: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + isv_account_id: + type: + - string + - "null" + wholesale_cost: + type: + - number + - "null" + subscriber_city: + type: + - string + - "null" + isv_company_name: + type: + - string + - "null" + offer_visibility: + type: + - string + - "null" + payer_address_id: + type: + - string + - "null" + payment_due_date: + type: + - string + - "null" + seller_tax_share: + type: + - number + - "null" + disbursement_date: + type: + - string + - "null" + hubspot_object_id: + type: + - string + - "null" + payer_postal_code: + type: + - string + - "null" + agreement_end_date: + type: + - string + - "null" + hubspot_object_url: + type: + - string + - "null" + listing_fee_refund: + type: + - number + - "null" + payer_company_name: + type: + - string + - "null" + payer_email_domain: + type: + - string + - "null" + seller_net_revenue: + type: + - number + - "null" + subscriber_country: + type: + - string + - "null" + disbursement_status: + type: + - string + - "null" + agreement_start_date: + type: + - string + - "null" + aws_seller_of_record: + type: + - string + - "null" + aws_tax_share_refund: + type: + - number + - "null" + payer_aws_account_id: + type: + - string + - "null" + customer_reference_id: + type: + - string + - "null" + payer_state_or_region: + type: + - string + - "null" + reseller_company_name: + type: + - string + - "null" + subscriber_address_id: + type: + - string + - "null" + usage_period_end_date: + type: + - string + - "null" + wholesale_cost_refund: + type: + - number + - "null" + disburse_bank_trace_id: + type: + - string + - "null" + listing_fee_invoice_id: + type: + - string + - "null" + listing_fee_percentage: + type: + - number + - "null" + subscriber_postal_code: + type: + - string + - "null" + resale_authorization_id: + type: + - string + - "null" + reseller_aws_account_id: + type: + - string + - "null" + seller_tax_share_refund: + type: + - number + - "null" + subscriber_company_name: + type: + - string + - "null" + subscriber_email_domain: + type: + - string + - "null" + usage_period_start_date: + type: + - string + - "null" + transaction_reference_id: + type: + - string + - "null" + agreement_acceptance_date: + type: + - string + - "null" + aws_tax_share_listing_fee: + type: + - number + - "null" + resale_authorization_name: + type: + - string + - "null" + subscriber_aws_account_id: + type: + - string + - "null" + subscriber_state_or_region: + type: + - string + - "null" + aws_tax_share_refund_listing_fee: + type: + - number + - "null" + resale_authorization_description: + type: + - string + - "null" + additionalProperties: true + analytics_aws_cosell_opportunities: + type: object + $schema: http://json-schema.org/schema# + properties: + age: + type: + - number + - "null" + geo: + type: + - string + - "null" + stage: + type: + - string + - "null" + state: + type: + - string + - "null" + region: + type: + - string + - "null" + status: + type: + - string + - "null" + country: + type: + - string + - "null" + segment: + type: + - string + - "null" + industry: + type: + - string + - "null" + use_case: + type: + - string + - "null" + aws_stage: + type: + - string + - "null" + next_step: + type: + - string + - "null" + seller_id: + type: + - string + - "null" + created_date: + type: + - string + - "null" + business_unit: + type: + - string + - "null" + is_greenfield: + type: + - boolean + - "null" + seller_domain: + type: + - string + - "null" + opportunity_id: + type: + - string + - "null" + opportunity_type: + type: + - string + - "null" + estimated_aws_mrr: + type: + - number + - "null" + target_close_date: + type: + - string + - "null" + aws_sales_rep_name: + type: + - string + - "null" + closed_lost_reason: + type: + - string + - "null" + last_activity_date: + type: + - string + - "null" + customer_company_name: + type: + - string + - "null" + opportunity_ownership: + type: + - string + - "null" + is_this_for_marketplace: + type: + - boolean + - "null" + primary_campaign_source: + type: + - string + - "null" + partner_primary_need_from_aws: + type: + - array + - "null" + items: + type: + - string + - "null" + is_opp_from_marketing_activity: + type: + - boolean + - "null" + is_marketing_development_funded: + type: + - boolean + - "null" + is_net_new_business_for_company: + type: + - boolean + - "null" + aws_marketplace_engagement_score: + type: + - string + - "null" + additionalProperties: true + analytics_azure_marketplace_orders: + type: object + $schema: http://json-schema.org/schema# + properties: + version: + type: + - number + - "null" + sku: + type: + - string + - "null" + plan_id: + type: + - string + - "null" + asset_id: + type: + - string + - "null" + currency: + type: + - string + - "null" + is_trial: + type: + - number + - "null" + offer_id: + type: + - string + - "null" + quantity: + type: + - number + - "null" + has_trial: + type: + - number + - "null" + seller_id: + type: + - string + - "null" + auto_renew: + type: + - boolean + - "null" + billing_id: + type: + - string + - "null" + offer_name: + type: + - string + - "null" + offer_type: + type: + - string + - "null" + customer_id: + type: + - string + - "null" + multi_party: + type: + - number + - "null" + billing_plan: + type: + - string + - "null" + billing_term: + type: + - string + - "null" + order_action: + type: + - string + - "null" + order_status: + type: + - string + - "null" + reference_id: + type: + - string + - "null" + dataset_month: + type: + - string + - "null" + order_version: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + term_end_date: + type: + - string + - "null" + billed_revenue: + type: + - number + - "null" + is_preview_sku: + type: + - number + - "null" + list_price_usd: + type: + - string + - "null" + event_timestamp: + type: + - string + - "null" + is_new_customer: + type: + - number + - "null" + is_private_plan: + type: + - number + - "null" + plan_visibility: + type: + - string + - "null" + term_start_date: + type: + - string + - "null" + customer_country: + type: + - string + - "null" + is_private_offer: + type: + - number + - "null" + month_start_date: + type: + - string + - "null" + offer_visibility: + type: + - string + - "null" + private_offer_id: + type: + - string + - "null" + quantity_changed: + type: + - number + - "null" + order_cancel_date: + type: + - string + - "null" + azure_license_type: + type: + - string + - "null" + billing_account_id: + type: + - string + - "null" + discount_price_usd: + type: + - number + - "null" + private_offer_name: + type: + - string + - "null" + purchase_record_id: + type: + - string + - "null" + cloud_instance_name: + type: + - string + - "null" + order_purchase_date: + type: + - string + - "null" + customer_company_name: + type: + - string + - "null" + marketplace_license_type: + type: + - string + - "null" + marketplace_subscription_id: + type: + - string + - "null" + purchase_record_line_item_id: + type: + - string + - "null" + additionalProperties: true + analytics_azure_marketplace_revenue: + type: object + $schema: http://json-schema.org/schema# + properties: + sku: + type: + - string + - "null" + units: + type: + - string + - "null" + plan_id: + type: + - string + - "null" + revenue: + type: + - number + - "null" + asset_id: + type: + - string + - "null" + offer_id: + type: + - string + - "null" + quantity: + type: + - number + - "null" + seller_id: + type: + - string + - "null" + billing_id: + type: + - string + - "null" + earning_id: + type: + - string + - "null" + invoice_id: + type: + - string + - "null" + offer_name: + type: + - string + - "null" + offer_type: + type: + - string + - "null" + customer_id: + type: + - string + - "null" + is_invoiced: + type: + - number + - "null" + multi_party: + type: + - number + - "null" + invoice_date: + type: + - string + - "null" + line_item_id: + type: + - string + - "null" + billing_model: + type: + - string + - "null" + customer_city: + type: + - string + - "null" + customer_name: + type: + - string + - "null" + dataset_month: + type: + - string + - "null" + payout_status: + type: + - string + - "null" + sales_channel: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + term_end_date: + type: + - string + - "null" + customer_email: + type: + - string + - "null" + customer_state: + type: + - string + - "null" + incentive_rate: + type: + - number + - "null" + list_price_usd: + type: + - number + - "null" + adjustment_type: + type: + - string + - "null" + is_private_plan: + type: + - number + - "null" + payout_currency: + type: + - string + - "null" + plan_visibility: + type: + - string + - "null" + term_start_date: + type: + - string + - "null" + customer_country: + type: + - string + - "null" + days_uncollected: + type: + - string + - "null" + exchange_rate_pc: + type: + - number + - "null" + invoice_due_date: + type: + - string + - "null" + is_private_offer: + type: + - number + - "null" + month_start_date: + type: + - string + - "null" + offer_visibility: + type: + - string + - "null" + private_offer_id: + type: + - string + - "null" + trial_deployment: + type: + - number + - "null" + earning_amount_cc: + type: + - number + - "null" + earning_amount_pc: + type: + - number + - "null" + exchange_rate_usd: + type: + - number + - "null" + payment_sent_date: + type: + - string + - "null" + billing_account_id: + type: + - string + - "null" + earning_amount_usd: + type: + - number + - "null" + exchange_rate_date: + type: + - string + - "null" + private_offer_name: + type: + - string + - "null" + purchase_record_id: + type: + - string + - "null" + customer_postal_code: + type: + - string + - "null" + discounted_price_usd: + type: + - number + - "null" + estimated_revenue_pc: + type: + - number + - "null" + transaction_currency: + type: + - string + - "null" + customer_company_name: + type: + - string + - "null" + earning_currency_code: + type: + - string + - "null" + estimated_revenue_usd: + type: + - number + - "null" + transaction_amount_cc: + type: + - number + - "null" + transaction_amount_pc: + type: + - number + - "null" + estimated_payout_month: + type: + - string + - "null" + transaction_amount_usd: + type: + - number + - "null" + payment_instrument_type: + type: + - string + - "null" + earning_exchange_rate_pc: + type: + - number + - "null" + additionalProperties: true + analytics_azure_cosell_opportunities: + type: object + $schema: http://json-schema.org/schema# + properties: + metadata: + type: + - object + - "null" + id: + type: + - number + - "null" + city: + type: + - string + - "null" + tags: + type: + - array + - "null" + items: + type: + - string + - "null" + uuid: + type: + - string + - "null" + cloud: + type: + - string + - "null" + notes: + type: + - string + - "null" + state: + type: + - string + - "null" + status: + type: + - string + - "null" + target: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + type: + type: + - string + - "null" + id: + type: + - string + - "null" + name: + type: + - string + - "null" + address: + type: + - string + - "null" + country: + type: + - string + - "null" + is_spam: + type: + - boolean + - "null" + currency: + type: + - string + - "null" + deal_name: + type: + - string + - "null" + deal_type: + type: + - string + - "null" + direction: + type: + - string + - "null" + solutions: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + id: + type: + - string + - "null" + name: + type: + - string + - "null" + created_at: + type: + - string + - "null" + deal_value: + type: + - number + - "null" + deleted_at: + type: + - string + - "null" + row_number: + type: + - number + - "null" + sub_status: + type: + - string + - "null" + sync_error: + type: + - object + - "null" + properties: + error: + type: + - string + - "null" + updated_at: + type: + - string + - "null" + external_id: + type: + - string + - "null" + postal_code: + type: + - string + - "null" + referral_id: + type: + - string + - "null" + sales_stage: + type: + - string + - "null" + cloud_status: + type: + - string + - "null" + created_by_id: + type: + - number + - "null" + engagement_id: + type: + - string + - "null" + qualification: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + status_reason: + type: + - string + - "null" + assistance_type: + type: + - string + - "null" + azure_seller_id: + type: + - number + - "null" + external_status: + type: + - string + - "null" + organization_id: + type: + - string + - "null" + referral_source: + type: + - string + - "null" + closed_date_time: + type: + - string + - "null" + cloud_created_at: + type: + - string + - "null" + customer_website: + type: + - string + - "null" + is_macc_eligible: + type: + - boolean + - "null" + partner_contacts: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + email: + type: + - string + - "null" + last_name: + type: + - string + - "null" + first_name: + type: + - string + - "null" + phone_number: + type: + - string + - "null" + closing_date_time: + type: + - string + - "null" + created_date_time: + type: + - string + - "null" + customer_contacts: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + email: + type: + - string + - "null" + last_name: + type: + - string + - "null" + first_name: + type: + - string + - "null" + phone_number: + type: + - string + - "null" + organization_name: + type: + - string + - "null" + seller_created_at: + type: + - string + - "null" + updated_date_time: + type: + - string + - "null" + accepted_date_time: + type: + - string + - "null" + consent_to_contact: + type: + - boolean + - "null" + customer_duns_number: + type: + - string + - "null" + domain_info_category: + type: + - string + - "null" + expiration_date_time: + type: + - string + - "null" + is_matching_complete: + type: + - boolean + - "null" + microsoft_account_id: + type: + - string + - "null" + microsoft_sales_reps: + type: + - array + - "null" + customer_company_name: + type: + - string + - "null" + shared_referral_metadata: + type: + - object + - "null" + properties: + status: + type: + - string + - "null" + sub_status: + type: + - string + - "null" + referral_id: + type: + - string + - "null" + status_reason: + type: + - string + - "null" + assistance_type: + type: + - string + - "null" + closed_date_time: + type: + - string + - "null" + created_date_time: + type: + - string + - "null" + updated_date_time: + type: + - string + - "null" + expiration_date_time: + type: + - string + - "null" + consent_to_to_share_info_with_others: + type: + - boolean + - "null" + consent_to_share_referral_with_microsoft_sellers: + type: + - boolean + - "null" + additionalProperties: true + analytics_azure_marketplace_customers: + type: object + $schema: http://json-schema.org/schema# + properties: + sku: + type: + - string + - "null" + email: + type: + - string + - "null" + plan_id: + type: + - string + - "null" + offer_id: + type: + - string + - "null" + date_lost: + type: + - string + - "null" + is_active: + type: + - number + - "null" + last_name: + type: + - string + - "null" + seller_id: + type: + - string + - "null" + billing_id: + type: + - string + - "null" + first_name: + type: + - string + - "null" + offer_name: + type: + - string + - "null" + offer_type: + type: + - string + - "null" + customer_id: + type: + - string + - "null" + multi_party: + type: + - number + - "null" + reference_id: + type: + - string + - "null" + customer_city: + type: + - string + - "null" + customer_type: + type: + - string + - "null" + dataset_month: + type: + - string + - "null" + date_acquired: + type: + - string + - "null" + provider_name: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + address_line_1: + type: + - string + - "null" + customer_state: + type: + - string + - "null" + provider_email: + type: + - string + - "null" + is_private_plan: + type: + - number + - "null" + plan_visibility: + type: + - string + - "null" + purchaser_email: + type: + - string + - "null" + last_modified_at: + type: + - string + - "null" + private_offer_id: + type: + - string + - "null" + azure_license_type: + type: + - string + - "null" + billing_account_id: + type: + - string + - "null" + private_offer_name: + type: + - string + - "null" + customer_postal_code: + type: + - string + - "null" + customer_company_name: + type: + - string + - "null" + promotional_customers: + type: + - number + - "null" + commerce_root_customer: + type: + - string + - "null" + customer_country_region: + type: + - string + - "null" + marketplace_subscription_id: + type: + - string + - "null" + additionalProperties: true + analytics_aws_marketplace_disbursements: + type: object + $schema: http://json-schema.org/schema# + properties: + currency: + type: + - string + - "null" + offer_id: + type: + - string + - "null" + seller_id: + type: + - string + - "null" + invoice_id: + type: + - string + - "null" + offer_name: + type: + - string + - "null" + payer_city: + type: + - string + - "null" + product_id: + type: + - string + - "null" + listing_fee: + type: + - number + - "null" + agreement_id: + type: + - string + - "null" + gross_refund: + type: + - number + - "null" + invoice_date: + type: + - string + - "null" + product_code: + type: + - string + - "null" + gross_revenue: + type: + - number + - "null" + payer_country: + type: + - string + - "null" + payment_terms: + type: + - string + - "null" + product_title: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + isv_account_id: + type: + - string + - "null" + wholesale_cost: + type: + - number + - "null" + subscriber_city: + type: + - string + - "null" + isv_company_name: + type: + - string + - "null" + offer_visibility: + type: + - string + - "null" + payer_address_id: + type: + - string + - "null" + payment_due_date: + type: + - string + - "null" + seller_tax_share: + type: + - number + - "null" + disbursement_date: + type: + - string + - "null" + hubspot_object_id: + type: + - string + - "null" + payer_postal_code: + type: + - string + - "null" + agreement_end_date: + type: + - string + - "null" + hubspot_object_url: + type: + - string + - "null" + listing_fee_refund: + type: + - number + - "null" + payer_company_name: + type: + - string + - "null" + payer_email_domain: + type: + - string + - "null" + seller_net_revenue: + type: + - number + - "null" + subscriber_country: + type: + - string + - "null" + disbursement_period: + type: + - string + - "null" + disbursement_status: + type: + - string + - "null" + agreement_start_date: + type: + - string + - "null" + aws_seller_of_record: + type: + - string + - "null" + payer_aws_account_id: + type: + - string + - "null" + salesforce_object_id: + type: + - string + - "null" + customer_reference_id: + type: + - string + - "null" + disbursed_net_revenue: + type: + - number + - "null" + payer_state_or_region: + type: + - string + - "null" + reseller_company_name: + type: + - string + - "null" + salesforce_object_url: + type: + - string + - "null" + subscriber_address_id: + type: + - string + - "null" + usage_period_end_date: + type: + - string + - "null" + wholesale_cost_refund: + type: + - number + - "null" + disburse_bank_trace_id: + type: + - string + - "null" + listing_fee_invoice_id: + type: + - string + - "null" + listing_fee_percentage: + type: + - number + - "null" + subscriber_postal_code: + type: + - string + - "null" + resale_authorization_id: + type: + - string + - "null" + reseller_aws_account_id: + type: + - string + - "null" + seller_tax_share_refund: + type: + - number + - "null" + subscriber_company_name: + type: + - string + - "null" + subscriber_email_domain: + type: + - string + - "null" + undisbursed_net_revenue: + type: + - number + - "null" + usage_period_start_date: + type: + - string + - "null" + transaction_reference_id: + type: + - string + - "null" + agreement_acceptance_date: + type: + - string + - "null" + aws_tax_share_listing_fee: + type: + - number + - "null" + disbursement_time_in_days: + type: + - number + - "null" + resale_authorization_name: + type: + - string + - "null" + subscriber_aws_account_id: + type: + - string + - "null" + subscriber_state_or_region: + type: + - string + - "null" + aws_tax_share_refund_listing_fee: + type: + - number + - "null" + resale_authorization_description: + type: + - string + - "null" + additionalProperties: true + analytics_gcp_marketplace_disbursements: + type: object + $schema: http://json-schema.org/schema# + properties: + ds: + type: + - string + - "null" + domain: + type: + - string + - "null" + sku_id: + type: + - string + - "null" + channel: + type: + - string + - "null" + country: + type: + - string + - "null" + seller_id: + type: + - string + - "null" + account_id: + type: + - string + - "null" + usage_unit: + type: + - string + - "null" + offer_title: + type: + - string + - "null" + postal_code: + type: + - string + - "null" + solution_id: + type: + - string + - "null" + cust_charges: + type: + - number + - "null" + cust_identity: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + solution_name: + type: + - string + - "null" + c_u_account_id: + type: + - string + - "null" + entitlement_id: + type: + - string + - "null" + order_end_date: + type: + - string + - "null" + usage_quantity: + type: + - number + - "null" + organization_id: + type: + - string + - "null" + released_amount: + type: + - number + - "null" + report_timezone: + type: + - number + - "null" + sku_description: + type: + - string + - "null" + withheld_amount: + type: + - number + - "null" + abandoned_amount: + type: + - number + - "null" + cud_credits_used: + type: + - number + - "null" + discount_offered: + type: + - number + - "null" + order_start_date: + type: + - string + - "null" + payment_schedule: + type: + - string + - "null" + private_offer_id: + type: + - string + - "null" + total_deductions: + type: + - number + - "null" + transaction_type: + type: + - string + - "null" + aggregated_payout: + type: + - number + - "null" + billing_frequency: + type: + - string + - "null" + cust_organization: + type: + - string + - "null" + installment_count: + type: + - number + - "null" + reseller_discount: + type: + - number + - "null" + state_or_province: + type: + - string + - "null" + wholesale_charges: + type: + - number + - "null" + trial_credits_used: + type: + - number + - "null" + disbursement_entity: + type: + - string + - "null" + insights_account_id: + type: + - string + - "null" + entitlement_end_date: + type: + - string + - "null" + max_usage_start_time: + type: + - string + - "null" + min_usage_start_time: + type: + - string + - "null" + report_creation_date: + type: + - string + - "null" + total_contract_value: + type: + - number + - "null" + private_offer_creator: + type: + - string + - "null" + entitlement_start_date: + type: + - string + - "null" + final_installment_date: + type: + - string + - "null" + first_installment_date: + type: + - string + - "null" + marketplace_fee_amount: + type: + - number + - "null" + partner_testing_credit: + type: + - number + - "null" + marketplace_fee_percent: + type: + - string + - "null" + replaced_private_offer_id: + type: + - string + - "null" + term_total_contract_value: + type: + - number + - "null" + private_offer_acceptance_date: + type: + - string + - "null" + additionalProperties: true + analytics_gcp_marketplace_daily_insights: + type: object + $schema: http://json-schema.org/schema# + properties: + ds: + type: + - string + - "null" + date: + type: + - string + - "null" + unit: + type: + - string + - "null" + usage: + type: + - number + - "null" + domain: + type: + - string + - "null" + sku_id: + type: + - string + - "null" + channel: + type: + - string + - "null" + charges: + type: + - number + - "null" + company: + type: + - string + - "null" + country: + type: + - string + - "null" + currency: + type: + - string + - "null" + end_date: + type: + - string + - "null" + quote_id: + type: + - string + - "null" + released: + type: + - number + - "null" + withheld: + type: + - number + - "null" + abandoned: + type: + - number + - "null" + seller_id: + type: + - string + - "null" + trial_use: + type: + - number + - "null" + account_id: + type: + - string + - "null" + due_vendor: + type: + - number + - "null" + start_date: + type: + - string + - "null" + offer_title: + type: + - string + - "null" + postal_code: + type: + - string + - "null" + payment_type: + type: + - string + - "null" + quote_creator: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + solution_name: + type: + - string + - "null" + entitlement_id: + type: + - string + - "null" + order_end_date: + type: + - string + - "null" + prepay_credits: + type: + - number + - "null" + postpay_credits: + type: + - number + - "null" + sku_description: + type: + - string + - "null" + order_start_date: + type: + - string + - "null" + payment_schedule: + type: + - string + - "null" + transaction_type: + type: + - string + - "null" + billing_frequency: + type: + - string + - "null" + installment_count: + type: + - number + - "null" + parent_account_id: + type: + - string + - "null" + reseller_discount: + type: + - number + - "null" + state_or_province: + type: + - string + - "null" + wholesale_charges: + type: + - number + - "null" + external_account_id: + type: + - string + - "null" + final_installment_date: + type: + - string + - "null" + first_installment_date: + type: + - string + - "null" + private_offer_acceptance_date: + type: + - string + - "null" + private_offer_total_contract_value: + type: + - number + - "null" + additionalProperties: true + analytics_azure_marketplace_metered_usage: + type: object + $schema: http://json-schema.org/schema# + properties: + sku: + type: + - string + - "null" + plan_id: + type: + - string + - "null" + asset_id: + type: + - string + - "null" + meter_id: + type: + - string + - "null" + offer_id: + type: + - string + - "null" + price_cc: + type: + - number + - "null" + core_size: + type: + - number + - "null" + raw_usage: + type: + - number + - "null" + seller_id: + type: + - string + - "null" + billing_id: + type: + - string + - "null" + offer_name: + type: + - string + - "null" + offer_type: + type: + - string + - "null" + usage_date: + type: + - string + - "null" + usage_unit: + type: + - string + - "null" + customer_id: + type: + - string + - "null" + multi_party: + type: + - number + - "null" + reference_id: + type: + - string + - "null" + customer_name: + type: + - string + - "null" + dataset_month: + type: + - string + - "null" + metered_usage: + type: + - number + - "null" + seller_domain: + type: + - string + - "null" + is_preview_sku: + type: + - number + - "null" + list_price_usd: + type: + - number + - "null" + vmsubscription: + type: + - number + - "null" + is_custom_meter: + type: + - number + - "null" + is_new_customer: + type: + - number + - "null" + is_private_plan: + type: + - number + - "null" + meter_dimension: + type: + - string + - "null" + plan_visibility: + type: + - string + - "null" + skubilling_type: + type: + - string + - "null" + usage_reference: + type: + - string + - "null" + customer_country: + type: + - string + - "null" + is_multisolution: + type: + - number + - "null" + is_private_offer: + type: + - number + - "null" + month_start_date: + type: + - string + - "null" + normalized_usage: + type: + - number + - "null" + offer_visibility: + type: + - string + - "null" + private_offer_id: + type: + - string + - "null" + deployment_method: + type: + - string + - "null" + azure_license_type: + type: + - string + - "null" + billing_account_id: + type: + - string + - "null" + estimated_price_pc: + type: + - number + - "null" + payout_currency_pc: + type: + - string + - "null" + private_offer_name: + type: + - string + - "null" + cloud_instance_name: + type: + - string + - "null" + customer_currency_cc: + type: + - string + - "null" + discounted_price_usd: + type: + - number + - "null" + customer_company_name: + type: + - string + - "null" + marketplace_license_type: + type: + - string + - "null" + marketplace_subscription_id: + type: + - string + - "null" + estimated_extended_charge_cc: + type: + - number + - "null" + estimated_extended_charge_pc: + type: + - number + - "null" + estimated_financial_impact_usd: + type: + - number + - "null" + partner_center_detected_anomaly: + type: + - number + - "null" + additionalProperties: true + analytics_gcp_marketplace_monthly_insights: + type: object + $schema: http://json-schema.org/schema# + properties: + ds: + type: + - string + - "null" + date: + type: + - string + - "null" + unit: + type: + - string + - "null" + usage: + type: + - number + - "null" + domain: + type: + - string + - "null" + sku_id: + type: + - string + - "null" + channel: + type: + - string + - "null" + charges: + type: + - number + - "null" + company: + type: + - string + - "null" + country: + type: + - string + - "null" + currency: + type: + - string + - "null" + end_date: + type: + - string + - "null" + quote_id: + type: + - string + - "null" + released: + type: + - number + - "null" + withheld: + type: + - number + - "null" + abandoned: + type: + - number + - "null" + seller_id: + type: + - string + - "null" + trial_use: + type: + - number + - "null" + account_id: + type: + - string + - "null" + due_vendor: + type: + - number + - "null" + start_date: + type: + - string + - "null" + offer_title: + type: + - string + - "null" + postal_code: + type: + - string + - "null" + payment_type: + type: + - string + - "null" + quote_creator: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + solution_name: + type: + - string + - "null" + entitlement_id: + type: + - string + - "null" + order_end_date: + type: + - string + - "null" + prepay_credits: + type: + - number + - "null" + postpay_credits: + type: + - number + - "null" + sku_description: + type: + - string + - "null" + order_start_date: + type: + - string + - "null" + payment_schedule: + type: + - string + - "null" + transaction_type: + type: + - string + - "null" + billing_frequency: + type: + - string + - "null" + installment_count: + type: + - number + - "null" + parent_account_id: + type: + - string + - "null" + reseller_discount: + type: + - number + - "null" + state_or_province: + type: + - string + - "null" + wholesale_charges: + type: + - number + - "null" + external_account_id: + type: + - string + - "null" + final_installment_date: + type: + - string + - "null" + first_installment_date: + type: + - string + - "null" + private_offer_acceptance_date: + type: + - string + - "null" + private_offer_total_contract_value: + type: + - number + - "null" + additionalProperties: true + analytics_gcp_marketplace_charges_and_usage: + type: object + $schema: http://json-schema.org/schema# + properties: + ds: + type: + - string + - "null" + sku: + type: + - string + - "null" + units: + type: + - string + - "null" + usage: + type: + - number + - "null" + charges: + type: + - number + - "null" + used_by: + type: + - string + - "null" + currency: + type: + - string + - "null" + location: + type: + - string + - "null" + released: + type: + - number + - "null" + resource: + type: + - string + - "null" + withheld: + type: + - number + - "null" + abandoned: + type: + - number + - "null" + seller_id: + type: + - string + - "null" + trial_use: + type: + - number + - "null" + account_id: + type: + - string + - "null" + due_partner: + type: + - number + - "null" + payment_type: + type: + - string + - "null" + google_entity: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + prepay_credits: + type: + - number + - "null" + postpay_credits: + type: + - number + - "null" + payment_schedule: + type: + - string + - "null" + billing_frequency: + type: + - string + - "null" + insights_account_id: + type: + - string + - "null" + refund_balance_outstanding: + type: + - number + - "null" + refund_balance_deducted_this_month: + type: + - number + - "null" + additionalProperties: true + analytics_gcp_marketplace_disbursements_summary: + type: object + $schema: http://json-schema.org/schema# + properties: + ds: + type: + - string + - "null" + total: + type: + - number + - "null" + amount: + type: + - number + - "null" + entity: + type: + - string + - "null" + currency: + type: + - string + - "null" + seller_id: + type: + - string + - "null" + withholding: + type: + - string + - "null" + payment_type: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + customer_refund: + type: + - number + - "null" + additionalProperties: true + analytics_gcp_marketplace_incremental_daily_insights: + type: object + $schema: http://json-schema.org/schema# + properties: + ds: + type: + - string + - "null" + date: + type: + - string + - "null" + unit: + type: + - string + - "null" + usage: + type: + - number + - "null" + domain: + type: + - string + - "null" + sku_id: + type: + - string + - "null" + channel: + type: + - string + - "null" + charges: + type: + - number + - "null" + company: + type: + - string + - "null" + country: + type: + - string + - "null" + currency: + type: + - string + - "null" + end_date: + type: + - string + - "null" + quote_id: + type: + - string + - "null" + released: + type: + - number + - "null" + withheld: + type: + - number + - "null" + abandoned: + type: + - number + - "null" + seller_id: + type: + - string + - "null" + trial_use: + type: + - number + - "null" + account_id: + type: + - string + - "null" + due_vendor: + type: + - number + - "null" + start_date: + type: + - string + - "null" + offer_title: + type: + - string + - "null" + postal_code: + type: + - string + - "null" + report_date: + type: + - string + - "null" + payment_type: + type: + - string + - "null" + quote_creator: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + solution_name: + type: + - string + - "null" + entitlement_id: + type: + - string + - "null" + order_end_date: + type: + - string + - "null" + prepay_credits: + type: + - number + - "null" + postpay_credits: + type: + - number + - "null" + sku_description: + type: + - string + - "null" + order_start_date: + type: + - string + - "null" + payment_schedule: + type: + - string + - "null" + transaction_type: + type: + - string + - "null" + billing_frequency: + type: + - string + - "null" + installment_count: + type: + - number + - "null" + parent_account_id: + type: + - string + - "null" + reseller_discount: + type: + - number + - "null" + state_or_province: + type: + - string + - "null" + wholesale_charges: + type: + - number + - "null" + external_account_id: + type: + - string + - "null" + final_installment_date: + type: + - string + - "null" + first_installment_date: + type: + - string + - "null" + private_offer_acceptance_date: + type: + - string + - "null" + private_offer_total_contract_value: + type: + - number + - "null" + additionalProperties: true + analytics_gcp_marketplace_incremental_monthly_insights: + type: object + $schema: http://json-schema.org/schema# + properties: + ds: + type: + - string + - "null" + date: + type: + - string + - "null" + unit: + type: + - string + - "null" + usage: + type: + - number + - "null" + domain: + type: + - string + - "null" + sku_id: + type: + - string + - "null" + channel: + type: + - string + - "null" + charges: + type: + - number + - "null" + company: + type: + - string + - "null" + country: + type: + - string + - "null" + currency: + type: + - string + - "null" + end_date: + type: + - string + - "null" + quote_id: + type: + - string + - "null" + released: + type: + - number + - "null" + withheld: + type: + - number + - "null" + abandoned: + type: + - number + - "null" + seller_id: + type: + - string + - "null" + trial_use: + type: + - number + - "null" + account_id: + type: + - string + - "null" + due_vendor: + type: + - number + - "null" + start_date: + type: + - string + - "null" + offer_title: + type: + - string + - "null" + postal_code: + type: + - string + - "null" + report_date: + type: + - string + - "null" + payment_type: + type: + - string + - "null" + quote_creator: + type: + - string + - "null" + seller_domain: + type: + - string + - "null" + solution_name: + type: + - string + - "null" + entitlement_id: + type: + - string + - "null" + order_end_date: + type: + - string + - "null" + prepay_credits: + type: + - number + - "null" + postpay_credits: + type: + - number + - "null" + sku_description: + type: + - string + - "null" + order_start_date: + type: + - string + - "null" + payment_schedule: + type: + - string + - "null" + transaction_type: + type: + - string + - "null" + billing_frequency: + type: + - string + - "null" + installment_count: + type: + - number + - "null" + parent_account_id: + type: + - string + - "null" + reseller_discount: + type: + - number + - "null" + state_or_province: + type: + - string + - "null" + wholesale_charges: + type: + - number + - "null" + external_account_id: + type: + - string + - "null" + final_installment_date: + type: + - string + - "null" + first_installment_date: + type: + - string + - "null" + private_offer_acceptance_date: + type: + - string + - "null" + private_offer_total_contract_value: + type: + - number + - "null" + additionalProperties: true diff --git a/airbyte-integrations/connectors/source-clazar/source_clazar/run.py b/airbyte-integrations/connectors/source-clazar/source_clazar/run.py new file mode 100644 index 000000000000..8e8e9cd79122 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/source_clazar/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceClazar + + +def run(): + source = SourceClazar() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-clazar/source_clazar/source.py b/airbyte-integrations/connectors/source-clazar/source_clazar/source.py new file mode 100644 index 000000000000..04bfc46c2cb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-clazar/source_clazar/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceClazar(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py index b91a0b49b694..4d84f48a074b 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py @@ -2,12 +2,25 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import sys -from airbyte_cdk.entrypoint import launch -from source_google_analytics_v4_service_account_only import SourceGoogleAnalyticsV4ServiceAccountOnly +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.exception_handler import init_uncaught_exception_handler +from airbyte_cdk.logger import init_logger +from airbyte_cdk.models import AirbyteMessage, ConnectorSpecification, FailureType, Type +from airbyte_cdk.utils import AirbyteTracedException if __name__ == "__main__": - source = SourceGoogleAnalyticsV4ServiceAccountOnly() - launch(source, sys.argv[1:]) + logger = init_logger("airbyte") + init_uncaught_exception_handler(logger) + + if AirbyteEntrypoint.parse_args(sys.argv[1:]).command == "spec": + message = AirbyteMessage(type=Type.SPEC, spec=ConnectorSpecification.parse_obj({"connectionSpecification": {}})) + print(message.json(exclude_unset=True)) + else: + error_message = "Google Analytics Universal Analytics Source Connector will be deprecated due to the deprecation of the Google Analytics Universal Analytics API by Google. This deprecation is scheduled by Google on July 1, 2024 (see Google’s Documentation for more details). Transition to the Google Analytics 4 (GA4) Source Connector by July 1, 2024, to continue accessing your analytics data." + raise AirbyteTracedException( + message=error_message, + internal_message=error_message, + failure_type=FailureType.config_error, + ) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/metadata.yaml index ad5328784295..ec9b5fd257b4 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: api connectorType: source definitionId: 9e28a926-8f3c-4911-982d-a2e1c378b59c - dockerImageTag: 0.0.2 + dockerImageTag: 0.1.0 dockerRepository: airbyte/source-google-analytics-v4-service-account-only documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4-service-account-only githubIssueLabel: source-google-analytics-v4-service-account-only @@ -22,11 +22,11 @@ data: name: Google Analytics (Universal Analytics) registries: cloud: - enabled: true + enabled: false oss: - enabled: true + enabled: false releaseStage: generally_available - supportLevel: community + supportLevel: archived tags: - language:python - cdk:python diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/main.py b/airbyte-integrations/connectors/source-google-analytics-v4/main.py index 3fd58bc1d5f6..4d84f48a074b 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/main.py @@ -2,7 +2,25 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_google_analytics_v4.run import run +import sys + +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.exception_handler import init_uncaught_exception_handler +from airbyte_cdk.logger import init_logger +from airbyte_cdk.models import AirbyteMessage, ConnectorSpecification, FailureType, Type +from airbyte_cdk.utils import AirbyteTracedException if __name__ == "__main__": - run() + logger = init_logger("airbyte") + init_uncaught_exception_handler(logger) + + if AirbyteEntrypoint.parse_args(sys.argv[1:]).command == "spec": + message = AirbyteMessage(type=Type.SPEC, spec=ConnectorSpecification.parse_obj({"connectionSpecification": {}})) + print(message.json(exclude_unset=True)) + else: + error_message = "Google Analytics Universal Analytics Source Connector will be deprecated due to the deprecation of the Google Analytics Universal Analytics API by Google. This deprecation is scheduled by Google on July 1, 2024 (see Google’s Documentation for more details). Transition to the Google Analytics 4 (GA4) Source Connector by July 1, 2024, to continue accessing your analytics data." + raise AirbyteTracedException( + message=error_message, + internal_message=error_message, + failure_type=FailureType.config_error, + ) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml index 5dd28fc7f094..536718c7bcfc 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: api connectorType: source definitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 - dockerImageTag: 0.3.3 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-google-analytics-v4 documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4 githubIssueLabel: source-google-analytics-v4 @@ -29,9 +29,9 @@ data: cloud: enabled: false oss: - enabled: true + enabled: false releaseStage: generally_available - supportLevel: certified + supportLevel: archived tags: - language:python - cdk:python diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/pyproject.toml b/airbyte-integrations/connectors/source-google-analytics-v4/pyproject.toml index 8738fb256f46..aef1cec59286 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/pyproject.toml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.3.3" +version = "0.4.0" name = "source-google-analytics-v4" description = "Source implementation for Google Analytics V4." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index 34660419209a..a559f2cbd10b 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 3.0.14 + dockerImageTag: 3.0.15 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg diff --git a/airbyte-integrations/connectors/source-instagram/pyproject.toml b/airbyte-integrations/connectors/source-instagram/pyproject.toml index 5217b54e55de..bfd3bd80e66e 100644 --- a/airbyte-integrations/connectors/source-instagram/pyproject.toml +++ b/airbyte-integrations/connectors/source-instagram/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "3.0.14" +version = "3.0.15" name = "source-instagram" description = "Source implementation for Instagram." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/components.py b/airbyte-integrations/connectors/source-instagram/source_instagram/components.py index 9463d2176ceb..74ee90ff8cf8 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/components.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/components.py @@ -1,10 +1,16 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +import logging from dataclasses import dataclass from datetime import datetime from typing import Any, Dict, MutableMapping, Optional import requests from airbyte_cdk.connector_builder.connector_builder_handler import resolve_manifest +from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import ( + ExponentialBackoffStrategy, +) +from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler +from airbyte_cdk.sources.declarative.requesters.http_requester import HttpClient, HttpMethod from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config from source_instagram import SourceInstagram @@ -14,16 +20,42 @@ GRAPH_URL = resolve_manifest(source=SourceInstagram()).record.data["manifest"]["definitions"]["base_requester"]["url_base"] -def get_http_response(path: str, request_params: Dict, config: Config) -> Optional[MutableMapping[str, Any]]: - url = f"{GRAPH_URL}/{path}" - token = config["access_token"] - headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} - params = { - **request_params, - } - response = requests.get(url, params=params, headers=headers) - if response.status_code == 200: - return response.json() +def get_http_response(name: str, path: str, request_params: Dict, config: Config) -> Optional[MutableMapping[str, Any]]: + http_logger = logging.getLogger(f"airbyte.HttpClient.{name}") + try: + url = f"{GRAPH_URL}/{path}" + token = config["access_token"] + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + params = { + **request_params, + } + factor = 5 + backoff_parameters = backoff_config = {"backoff": factor} + backoff_strategy = ExponentialBackoffStrategy(factor=factor, parameters=backoff_parameters, config=backoff_config) + error_handler = DefaultErrorHandler(config={}, parameters={}, backoff_strategies=[backoff_strategy]) + http_client = HttpClient( + name=name, + logger=http_logger, + use_cache=True, + error_handler=error_handler, + ) + _, response = http_client.send_request( + http_method=HttpMethod.GET.name, + url=url, + request_kwargs={}, + headers=headers, + params=params, + ) + response.raise_for_status() + except requests.HTTPError as http_err: + error = f"HTTP error occurred: {http_err.response.status_code} - {http_err.response.text}" + http_logger.error(f"Error getting children data: {error}") + raise Exception(error) + except Exception as err: + error = f"An error occurred: {err}" + http_logger.error(f"Error getting children data: {error}") + raise Exception(error) + return response.json() @dataclass @@ -93,7 +125,7 @@ def transform(self, record: MutableMapping[str, Any], config: Optional[Config] = if children: children_ids = [child.get("id") for child in children.get("data")] for children_id in children_ids: - media_data = get_http_response(children_id, {"fields": fields}, config=config) + media_data = get_http_response(f"MediaInsights.{children_id}", children_id, {"fields": fields}, config=config) media_data = InstagramClearUrlTransformation().transform(media_data) if media_data.get("timestamp"): dt = datetime.strptime(media_data["timestamp"], "%Y-%m-%dT%H:%M:%S%z") @@ -106,6 +138,55 @@ def transform(self, record: MutableMapping[str, Any], config: Optional[Config] = return record +@dataclass +class InstagramInsightsTransformation(RecordTransformation): + def transform(self, record: MutableMapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + """ + The transformation flattens the array of insights into a single object (dictionary). In such object, each key-value pair + in the resulting object represents a metric and and its corresponding value. + + Example Input: + { + "data": [ + { + "name": "comments", + "period": "lifetime", + "values": [ + { + "value": 7 + } + ], + "title": "title1", + "description": "Description1.", + "id": "insta_id/insights/comments/lifetime" + }, + { + "name": "ig_reels_avg_watch_time", + "period": "lifetime", + "values": [ + { + "value": 11900 + } + ], + "title": "2", + "description": "Description2.", + "id": "insta_id/insights/ig_reels_avg_watch_time/lifetime" + } + } + + Example Output: + { + "comments": 7, + "ig_reels_avg_watch_time": 11900 + } + """ + if record.get("data"): + insights_data = record.pop("data") + for insight in insights_data: + record[insight["name"]] = insight.get("values")[0]["value"] + return record + + @dataclass class InstagramBreakDownResultsTransformation(RecordTransformation): """ diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/manifest.yaml b/airbyte-integrations/connectors/source-instagram/source_instagram/manifest.yaml index 2b5ee67e8185..57b5117451af 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/manifest.yaml +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/manifest.yaml @@ -65,6 +65,17 @@ definitions: stream: $ref: "#/definitions/streams/Api" transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - media_insights_info + value: >- + {{ {"media_id": record['id'], "media_type": + record['media_type'], "media_product_type": + record['media_product_type'], "business_account_id": + stream_partition.account.business_account_id, "page_id": + stream_partition.account.page_id} }} - type: AddFields fields: - type: AddedFieldDefinition @@ -93,6 +104,110 @@ definitions: type: InlineSchemaLoader schema: $ref: "#/schemas/Media" + MediaInsights: + type: DeclarativeStream + name: media_insights + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: "{{ stream_partition.media_insights_info.media_id }}/insights" + http_method: GET + request_parameters: + metric: >- + {% if stream_partition.media_insights_info.media_product_type == + "REELS" %}{{ + 'comments,ig_reels_avg_watch_time,ig_reels_video_view_total_time,likes,plays,reach,saved,shares' + }}{% elif stream_partition.media_insights_info.media_type == + "VIDEO" and + stream_partition.media_insights_info.media_product_type == "FEED" + %}{{ 'impressions,reach,saved,video_views'}}{% elif + stream_partition.media_insights_info.media_type == "VIDEO" %}{{ + 'impressions,reach,saved,video_views,likes,comments,shares' + }}{%elif stream_partition.media_insights_info.media_type == + "CAROUSEL_ALBUM"%}{{ 'impressions,reach,saved,video_views' }}{% else %}{{ + 'impressions,reach,saved,video_views,likes,comments,shares' }}{% + endif %} + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + response_filters: + - type: HttpResponseFilter + action: IGNORE + predicate: >- + {{ 'error' in response and + response['error']['error_subcode'] == 2108006 }} + http_codes: [] + error_message: >- + Insights error for business_account_id: {{ + response['error']['message'] }} + - type: DefaultErrorHandler + max_retries: 5 + backoff_strategies: + - type: ExponentialBackoffStrategy + factor: 5 + response_filters: + - type: HttpResponseFilter + action: IGNORE + predicate: >- + {{ 'error' in response and ( (response['error']['code'] == + 100 and response['error']['error_subcode'] == 33) or + (response['error']['code'] == 10 and + response['error']['message'] == "(#10) Application does + not have permission for this action") ) }} + error_message: >- + Check provided permissions for: {{ + response['error']['message'] }} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: media_insights_info + partition_field: media_insights_info + stream: + $ref: "#/definitions/streams/Media" + transformations: + - type: CustomTransformation + class_name: source_instagram.components.InstagramInsightsTransformation + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - id + value: "{{ stream_partition.media_insights_info.media_id}}" + value_type: string + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - page_id + value: "{{ stream_partition.media_insights_info.page_id}}" + value_type: string + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - business_account_id + value: "{{ stream_partition.media_insights_info.business_account_id }}" + value_type: string + - type: CustomTransformation + class_name: source_instagram.components.InstagramClearUrlTransformation + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/MediaInsights" Users: type: DeclarativeStream name: users @@ -288,6 +403,15 @@ definitions: stream: $ref: "#/definitions/streams/Api" transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - story_insights_info + value: >- + {{ {"story_id": record['id'], "business_account_id": + stream_partition.account.business_account_id, "page_id": + stream_partition.account.page_id} }} - type: AddFields fields: - type: AddedFieldDefinition @@ -314,6 +438,75 @@ definitions: type: InlineSchemaLoader schema: $ref: "#/schemas/Stories" + StoryInsights: + type: DeclarativeStream + name: story_insights + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: "{{ stream_partition.story_insights_info.story_id }}/insights" + http_method: GET + request_parameters: + metric: impressions,reach,replies + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + backoff_strategies: + - type: ExponentialBackoffStrategy + factor: 5 + response_filters: + - type: HttpResponseFilter + action: IGNORE + predicate: >- + {{ 'error' in response and response['error']['code'] == 10 }} + error_message: >- + Insights error: {{ response['error']['message'] }} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: story_insights_info + partition_field: story_insights_info + stream: + $ref: "#/definitions/streams/Stories" + transformations: + - type: CustomTransformation + class_name: source_instagram.components.InstagramInsightsTransformation + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - id + value: "{{ stream_partition.story_insights_info.story_id }}" + value_type: string + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - page_id + value: "{{ stream_partition.story_insights_info.page_id }}" + value_type: string + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - business_account_id + value: "{{ stream_partition.story_insights_info.business_account_id }}" + value_type: string + - type: CustomTransformation + class_name: source_instagram.components.InstagramClearUrlTransformation + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/StoryInsights" Api: type: DeclarativeStream name: Api @@ -387,28 +580,36 @@ definitions: streams: - $ref: "#/definitions/streams/Media" + - $ref: "#/definitions/streams/MediaInsights" - $ref: "#/definitions/streams/Users" - $ref: "#/definitions/streams/UserLifetimeInsights" - $ref: "#/definitions/streams/Stories" + - $ref: "#/definitions/streams/StoryInsights" - $ref: "#/definitions/streams/Api" metadata: autoImportSchema: Media: true + MediaInsights: true Users: true UserLifetimeInsights: true Stories: true + StoryInsights: true Api: true yamlComponents: streams: Media: - transformations + MediaInsights: + - transformations Users: - transformations UserLifetimeInsights: - transformations Stories: - transformations + StoryInsights: + - transformations schemas: Media: @@ -578,6 +779,83 @@ schemas: - string required: - id + MediaInsights: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + business_account_id: + description: The unique identifier of the Instagram business account associated with the media. + type: + - "null" + - string + page_id: + description: The unique identifier of the Instagram page where the media is posted. + type: + - "null" + - string + id: + description: The unique identifier of the media. + type: + - "null" + - string + ig_reels_avg_watch_time: + description: The average watch time of Instagram Reels videos in seconds + type: + - "null" + - number + ig_reels_video_view_total_time: + description: The total watch time of Instagram Reels videos in seconds. + type: + - "null" + - number + impressions: + description: The number of times the media has been displayed to users. + type: + - "null" + - integer + reach: + description: The number of unique users who have seen the media. + type: + - "null" + - integer + saved: + description: The number of times users have saved the media. + type: + - "null" + - integer + video_views: + description: The total number of views on video media. + type: + - "null" + - integer + comments: + description: The number of comments received on the media. + type: + - "null" + - integer + likes: + description: The number of likes received on the media. + type: + - "null" + - integer + shares: + description: The number of times the media has been shared. + type: + - "null" + - integer + total_interactions: + description: The total number of interactions (likes, comments, shares) on the media. + type: + - "null" + - integer + plays: + description: The number of times the media has been played. + type: + - "null" + - integer + required: + - id Users: type: object $schema: http://json-schema.org/schema# @@ -763,6 +1041,41 @@ schemas: - string required: - id + StoryInsights: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + business_account_id: + description: The unique identifier of the business account associated with the story insights. + type: + - "null" + - string + page_id: + description: The unique identifier of the associated page where the story was posted. + type: + - "null" + - string + id: + description: The unique identifier of the story insights record. + type: + - "null" + - string + impressions: + description: The number of times the story was viewed. + type: + - "null" + - integer + reach: + description: The number of unique accounts that viewed the story. + type: + - "null" + - integer + replies: + description: The number of replies or interactions generated by the story. + type: + - "null" + - integer Api: type: object $schema: http://json-schema.org/schema# diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py index e1ef2962ba10..fe4a35309b95 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py @@ -7,7 +7,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams.core import Stream from source_instagram.api import InstagramAPI -from source_instagram.streams import MediaInsights, StoryInsights, UserInsights +from source_instagram.streams import UserInsights """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into @@ -53,9 +53,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: def get_non_low_code_streams(self, config: Mapping[str, Any]) -> List[Stream]: api = InstagramAPI(access_token=config["access_token"]) self._validate_start_date(config) - streams = [ - MediaInsights(api=api), - StoryInsights(api=api), - UserInsights(api=api, start_date=config["start_date"]), - ] + streams = [UserInsights(api=api, start_date=config["start_date"])] return streams diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index 2650e33c9656..a01fa00b8056 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -13,8 +13,6 @@ from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from cached_property import cached_property -from facebook_business.adobjects.igmedia import IGMedia -from facebook_business.exceptions import FacebookRequestError from source_instagram.api import InstagramAPI from .common import remove_params_from_url @@ -260,122 +258,3 @@ def _state_has_legacy_format(self, state: Mapping[str, Any]) -> bool: if not isinstance(value, Mapping): return True return False - - -class MediaInsights(DatetimeTransformerMixin, InstagramStream): - """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights""" - - MEDIA_METRICS = ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares"] - CAROUSEL_ALBUM_METRICS = ["impressions", "reach", "saved", "video_views"] - - REELS_METRICS = [ - "comments", - "ig_reels_avg_watch_time", - "ig_reels_video_view_total_time", - "likes", - "plays", - "reach", - "saved", - "shares", - ] - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - account = stream_slice["account"] - ig_account = account["instagram_business_account"] - media = ig_account.get_media(params=self.request_params(), fields=["media_type", "media_product_type"]) - for ig_media in media: - account_id = ig_account.get("id") - insights = self._get_insights(ig_media, account_id) - if insights is None: - continue - - insights["id"] = ig_media["id"] - insights["page_id"] = account["page_id"] - insights["business_account_id"] = ig_account["id"] - yield self.transform(insights) - - def _get_insights(self, item, account_id) -> Optional[MutableMapping[str, Any]]: - """Get insights for specific media""" - if item.get("media_product_type") == "REELS": - metrics = self.REELS_METRICS - elif item.get("media_type") == "VIDEO" and item.get("media_product_type") == "FEED": - metrics = ["impressions", "reach", "saved", "video_views", "video_views"] - elif item.get("media_type") == "VIDEO": - metrics = self.MEDIA_METRICS + ["video_views"] - elif item.get("media_type") == "CAROUSEL_ALBUM": - metrics = self.CAROUSEL_ALBUM_METRICS - - else: - metrics = self.MEDIA_METRICS - - try: - insights = item.get_insights(params={"metric": metrics}) - return {record.get("name"): record.get("values")[0]["value"] for record in insights} - except FacebookRequestError as error: - error_code = error.api_error_code() - error_subcode = error.api_error_subcode() - error_message = error.api_error_message() - - # An error might occur if the media was posted before the most recent time that - # the user's account was converted to a business account from a personal account - if error_subcode == 2108006: - details = error.body().get("error", {}).get("error_user_title") or error_message - self.logger.error(f"Insights error for business_account_id {account_id}: {details}") - # We receive all Media starting from the last one, and if on the next Media we get an Insight error, - # then no reason to make inquiries for each Media further, since they were published even earlier. - return None - elif ( - error_code == 100 - and error_subcode == 33 - or error_code == 10 - and error_message == "(#10) Application does not have permission for this action" - ): - self.logger.error(f"Check provided permissions for {account_id}: {error_message}") - return None - raise error - - -class StoryInsights(DatetimeTransformerMixin, InstagramStream): - """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights""" - - metrics = ["impressions", "reach", "replies"] - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - account = stream_slice["account"] - ig_account = account["instagram_business_account"] - stories = ig_account.get_stories(params=self.request_params(), fields=[]) - for ig_story in stories: - insights = self._get_insights(IGMedia(ig_story["id"])) - if not insights: - continue - - insights["id"] = ig_story["id"] - insights["page_id"] = account["page_id"] - insights["business_account_id"] = ig_account["id"] - yield self.transform(insights) - - def _get_insights(self, story: IGMedia) -> MutableMapping[str, Any]: - """Get insights for specific story""" - - # Story IG Media object metrics with values less than 5 will return an error code 10 with the message (#10) - # Not enough viewers for the media to show insights. - try: - insights = story.get_insights(params={"metric": self.metrics}) - return {record["name"]: record["values"][0]["value"] for record in insights} - except FacebookRequestError as error: - if error.api_error_code() == 10: - self.logger.error(f"Insights error: {error.api_error_message()}") - return {} - raise error diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py index 4cb254a4c573..7ac44159c282 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py @@ -3,7 +3,7 @@ # from __future__ import annotations -from typing import List, Optional +from typing import List, Optional, Union from airbyte_cdk.connector_builder.connector_builder_handler import resolve_manifest from airbyte_cdk.test.mock_http.request import HttpRequest @@ -27,6 +27,14 @@ def get_account_endpoint(cls) -> RequestBuilder: def get_media_endpoint(cls, item_id: str) -> RequestBuilder: return cls(resource="media").with_item_id(item_id) + @classmethod + def get_media_children_endpoint(cls, item_id: str) -> RequestBuilder: + return cls().with_item_id(item_id).with_item_id_is_sub_path(False) + + @classmethod + def get_media_insights_endpoint(cls, item_id: str) -> RequestBuilder: + return cls(resource="insights").with_item_id(item_id) + @classmethod def get_stories_endpoint(cls, item_id: str) -> RequestBuilder: return cls(resource="stories").with_item_id(item_id) @@ -66,7 +74,9 @@ def with_item_id_is_sub_path(self, is_sub_path: bool): self._item_id_is_sub_path = is_sub_path return self - def with_custom_param(self, param: str, value: str): + def with_custom_param(self, param: str, value: Union[str, List[str]], with_format=False): + if with_format and isinstance(value, List): + value = self._get_formatted_fields(value) self._query_params[param] = value return self diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py index 493ae91865be..429b75faa425 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py @@ -2,10 +2,11 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # +import json from unittest import TestCase from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput -from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import ( FieldPath, HttpResponseBuilder, @@ -40,6 +41,25 @@ "children" ] +_CHILDREN_FIELDS = [ + "id", + "ig_id", + "media_type", + "media_url", + "owner", + "permalink", + "shortcode", + "thumbnail_url", + "timestamp", + "username" + ] + +_CHILDREN_IDS = [ + "07608776690540123", + "52896800415362123", + "39559889460059123", + "17359925580923123" + ] _STREAM_NAME = "media" @@ -51,6 +71,13 @@ def _get_request() -> RequestBuilder: ) +def _get_children_request(media_id:str) -> RequestBuilder: + return( + RequestBuilder.get_media_children_endpoint(item_id=media_id) + .with_fields(_CHILDREN_FIELDS) + ) + + def _get_response() -> HttpResponseBuilder: return create_response_builder( response_template=find_template(_STREAM_NAME, __file__), @@ -127,3 +154,31 @@ def test_when_read_then_datetime_fields_transformed(self, http_mocker: HttpMocke output = self._read(config_=config()) assert len(output.records) == 1 assert output.records[0].record.data[created_time_field] == expected_datetime_value + + @HttpMocker() + def test_given_one_page_has_children_field(self, http_mocker: HttpMocker) -> None: + test = "children" + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_request().build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + for children_id in _CHILDREN_IDS: + http_mocker.get( + _get_children_request(children_id).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_children_for_{test}", __file__)), 200) + ) + + output = self._read(config_=config()) + assert len(output.records) == 1 + children = output.records[0].record.data["children"] + assert len(children) == 4 + for child in children: + assert "id" in child + assert "ig_id" in child + assert "media_type" in child + assert "owner" in child diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py new file mode 100644 index 000000000000..86c6db7c626b --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py @@ -0,0 +1,360 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# +import json +import unittest +from unittest import TestCase + +import pytest +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_protocol.models import SyncMode + +from .config import BUSINESS_ACCOUNT_ID, ConfigBuilder +from .pagination import NEXT_PAGE_TOKEN, InstagramPaginationStrategy +from .request_builder import RequestBuilder, get_account_request +from .response_builder import get_account_response +from .utils import config, read_output + +PARENT_FIELDS = [ + "caption", + "comments_count", + "id", + "ig_id", + "is_comment_enabled", + "like_count","media_type", + "media_product_type", + "media_url", + "owner", + "permalink", + "shortcode", + "thumbnail_url", + "timestamp", + "username", + "children" +] +_PARENT_STREAM_NAME = 'media' +_STREAM_NAME = "media_insights" + + + +MEDIA_ID_REELS = "84386203808767123" +MEDIA_ID_VIDEO_FEED = "90014330517797123" +MEDIA_ID_VIDEO = "09894619573775123" +MEDIA_ID_CAROUSEL_ALBUM = "66123508374641123" +MEDIA_ID_GENERAL_MEDIA = "35076616084176123" +MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS = "35076616084176124" +MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS = "35076616084176125" +MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = "35076616084176126" + +REELS = 'reels' +VIDEO_FEED = 'video_feed' +VIDEO = 'video' +CAROUSEL_ALBUM = 'carousel_album' +GENERAL_MEDIA = 'general_media' +ERROR_POSTED_BEFORE_BUSINESS = 'error_posted_before_business' +ERROR_WITH_WRONG_PERMISSIONS = 'error_with_wrong_permissions' +ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = 'error_with_wrong_permissions_code_10' + +_MEDIA_IDS = { + REELS: MEDIA_ID_REELS, + VIDEO_FEED: MEDIA_ID_VIDEO_FEED, + VIDEO: MEDIA_ID_VIDEO, + CAROUSEL_ALBUM: MEDIA_ID_CAROUSEL_ALBUM, + GENERAL_MEDIA: MEDIA_ID_GENERAL_MEDIA +} + +METRICS_GENERAL_MEDIA = ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares"] + +_METRICS = { + MEDIA_ID_REELS: ["comments", "ig_reels_avg_watch_time", "ig_reels_video_view_total_time", "likes", "plays", "reach", "saved", "shares"], + MEDIA_ID_VIDEO_FEED: ["impressions", "reach", "saved", "video_views"], + MEDIA_ID_VIDEO: ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares"], + MEDIA_ID_CAROUSEL_ALBUM: ["impressions", "reach", "saved", "video_views"], + MEDIA_ID_GENERAL_MEDIA: METRICS_GENERAL_MEDIA, + # Reusing general media metrics for error scenarios + MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS: METRICS_GENERAL_MEDIA, + MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS: METRICS_GENERAL_MEDIA, + MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10: METRICS_GENERAL_MEDIA +} + + +def _get_parent_request() -> RequestBuilder: + return ( + RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID) + .with_limit(100) + .with_fields(PARENT_FIELDS) + ) + + +def _get_child_request(media_id, metric) -> RequestBuilder: + return ( + RequestBuilder.get_media_insights_endpoint(item_id=media_id) + .with_custom_param("metric", metric, with_format=True) + ) + + +def _get_response(stream_name: str, test: str = None, with_pagination_strategy: bool = True) -> HttpResponseBuilder: + scenario = '' + if test: + scenario = f'_for_{test}' + kwargs = { + "response_template": find_template(f"{stream_name}{scenario}", __file__), + "records_path": FieldPath("data"), + "pagination_strategy": InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN), + } + if with_pagination_strategy: + kwargs["pagination_strategy"] = InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN) + + return create_response_builder( + **kwargs + ) + + +def _record(stream_name: str, test: str = None) -> RecordBuilder: + scenario = '' + if test: + scenario = f'_for_{test}' + return create_record_builder( + response_template=find_template(f"{stream_name}{scenario}", __file__), + records_path=FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +class TestFullRefresh(TestCase): + + @staticmethod + def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.full_refresh, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_instagram_insights_for_reels(self, http_mocker: HttpMocker) -> None: + test = REELS + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_parent_request().build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_REELS, metric=_METRICS[MEDIA_ID_REELS]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + output = self._read(config_=config()) + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS[MEDIA_ID_REELS]: + assert metric in output.records[0].record.data + + @HttpMocker() + def test_instagram_insights_for_video_feed(self, http_mocker: HttpMocker) -> None: + test = VIDEO_FEED + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_parent_request().build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_VIDEO_FEED, metric=_METRICS[MEDIA_ID_VIDEO_FEED]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + output = self._read(config_=config()) + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS[MEDIA_ID_VIDEO_FEED]: + assert metric in output.records[0].record.data + + @HttpMocker() + def test_instagram_insights_for_video(self, http_mocker: HttpMocker) -> None: + test = VIDEO + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_parent_request().build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_VIDEO, metric=_METRICS[MEDIA_ID_VIDEO]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + output = self._read(config_=config()) + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS[MEDIA_ID_VIDEO]: + assert metric in output.records[0].record.data + + @HttpMocker() + def test_instagram_insights_carousel_album(self, http_mocker: HttpMocker) -> None: + test = CAROUSEL_ALBUM + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_parent_request().build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_CAROUSEL_ALBUM, metric=_METRICS[MEDIA_ID_CAROUSEL_ALBUM]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + output = self._read(config_=config()) + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS[MEDIA_ID_CAROUSEL_ALBUM]: + assert metric in output.records[0].record.data + + @HttpMocker() + def test_instagram_insights_general_media(self, http_mocker: HttpMocker) -> None: + test = GENERAL_MEDIA + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_parent_request().build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + output = self._read(config_=config()) + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS[MEDIA_ID_GENERAL_MEDIA]: + assert metric in output.records[0].record.data + + + @HttpMocker() + def test_instagram_insights_error_posted_before_business(self, http_mocker: HttpMocker) -> None: + test = ERROR_POSTED_BEFORE_BUSINESS + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_parent_request().build(), + HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS, metric=_METRICS[MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + ) + + output = self._read(config_=config()) + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS[MEDIA_ID_GENERAL_MEDIA]: + assert metric in output.records[0].record.data + + @HttpMocker() + def test_instagram_insights_error_with_wrong_permissions(self, http_mocker: HttpMocker) -> None: + test = ERROR_WITH_WRONG_PERMISSIONS + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_parent_request().build(), + HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + ) + + output = self._read(config_=config()) + # error was ignored and correct record was processed + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS[MEDIA_ID_GENERAL_MEDIA]: + assert metric in output.records[0].record.data + + @HttpMocker() + def test_instagram_insights_error_with_wrong_permissions_code_10(self, http_mocker: HttpMocker) -> None: + test = ERROR_WITH_WRONG_PERMISSIONS_CODE_10 + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + http_mocker.get( + _get_parent_request().build(), + HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + ) + + http_mocker.get( + _get_child_request(media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10]).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + ) + + output = self._read(config_=config()) + # error was ignored and correct record was processed + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS[MEDIA_ID_GENERAL_MEDIA]: + assert metric in output.records[0].record.data diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py new file mode 100644 index 000000000000..bd08b8a7e79b --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py @@ -0,0 +1,167 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# +import json +import unittest +from unittest import TestCase + +import pytest +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_protocol.models import SyncMode + +from .config import BUSINESS_ACCOUNT_ID, ConfigBuilder +from .pagination import NEXT_PAGE_TOKEN, InstagramPaginationStrategy +from .request_builder import RequestBuilder, get_account_request +from .response_builder import get_account_response +from .utils import config, read_output + +PARENT_FIELDS = [ + "caption", + "id", + "ig_id", + "like_count", + "media_type", + "media_product_type", + "media_url", + "owner", + "permalink", + "shortcode", + "thumbnail_url", + "timestamp", + "username" +] +_PARENT_STREAM_NAME = 'stories' +_STREAM_NAME = "story_insights" + +STORIES_ID = "3874523487643" +STORIES_ID_ERROR_CODE_10 = "3874523487644" + +HAPPY_PATH = 'story_insights_happy_path' +ERROR_10 = 'story_insights_error_code_10' + +_METRICS = ["impressions", "reach", "replies"] + + + +def _get_parent_request() -> RequestBuilder: + return ( + RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID) + .with_limit(100) + .with_fields(PARENT_FIELDS) + ) + + +def _get_child_request(media_id, metric) -> RequestBuilder: + return ( + RequestBuilder.get_media_insights_endpoint(item_id=media_id) + .with_custom_param("metric", metric, with_format=True) + ) + + +def _get_response(stream_name: str, test: str = None, with_pagination_strategy: bool = True) -> HttpResponseBuilder: + scenario = '' + if test: + scenario = f'_for_{test}' + kwargs = { + "response_template": find_template(f"{stream_name}{scenario}", __file__), + "records_path": FieldPath("data"), + "pagination_strategy": InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN), + } + if with_pagination_strategy: + kwargs["pagination_strategy"] = InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN) + + return create_response_builder( + **kwargs + ) + + +def _record(stream_name: str, test: str = None) -> RecordBuilder: + scenario = '' + if test: + scenario = f'_for_{test}' + return create_record_builder( + response_template=find_template(f"{stream_name}{scenario}", __file__), + records_path=FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +class TestFullRefresh(TestCase): + + @staticmethod + def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=_STREAM_NAME, + sync_mode=SyncMode.full_refresh, + expecting_exception=expecting_exception, + ) + + @HttpMocker() + def test_instagram_story_insights(self, http_mocker: HttpMocker) -> None: + test = HAPPY_PATH + # Mocking API stream + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + # Mocking parent stream + http_mocker.get( + _get_parent_request().build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + ) + + http_mocker.get( + _get_child_request(media_id=STORIES_ID, metric=_METRICS).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + ) + + output = self._read(config_=config()) + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS: + assert metric in output.records[0].record.data + + + @HttpMocker() + def test_instagram_story_insights_for_error_code_30(self, http_mocker: HttpMocker) -> None: + test = ERROR_10 + http_mocker.get( + get_account_request().build(), + get_account_response(), + ) + # Mocking parent stream + http_mocker.get( + _get_parent_request().build(), + HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + ) + # Good response + http_mocker.get( + _get_child_request(media_id=STORIES_ID, metric=_METRICS).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{HAPPY_PATH}", __file__)), 200) + ) + # error 10 + http_mocker.get( + _get_child_request(media_id=STORIES_ID_ERROR_CODE_10, metric=_METRICS).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + ) + + output = self._read(config_=config()) + # error was ignored and correct record was processed + assert len(output.records) == 1 + assert output.records[0].record.data["page_id"] + assert output.records[0].record.data["business_account_id"] + assert output.records[0].record.data["id"] + for metric in _METRICS: + assert metric in output.records[0].record.data diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/records.py b/airbyte-integrations/connectors/source-instagram/unit_tests/records.py index 7ccb49528bb2..ecf06f7b64e5 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/records.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/records.py @@ -427,3 +427,115 @@ }, "id": "17841457631192237/insights/follower_demographics/lifetime" } + +insights_record = { + "data": [ + { + "name": "comments", + "period": "lifetime", + "values": [ + { + "value": 7 + } + ], + "title": "title1", + "description": "Description1.", + "id": "insta_id/insights/comments/lifetime" + }, + { + "name": "ig_reels_avg_watch_time", + "period": "lifetime", + "values": [ + { + "value": 11900 + } + ], + "title": "2", + "description": "Description2.", + "id": "insta_id/insights/ig_reels_avg_watch_time/lifetime" + }, + { + "name": "ig_reels_video_view_total_time", + "period": "lifetime", + "values": [ + { + "value": 25979677 + } + ], + "title": "title3", + "description": "Description3.", + "id": "insta_id/insights/ig_reels_video_view_total_time/lifetime" + }, + { + "name": "likes", + "period": "lifetime", + "values": [ + { + "value": 102 + } + ], + "title": "title4", + "description": "Description4.", + "id": "insta_id/insights/likes/lifetime" + }, + { + "name": "plays", + "period": "lifetime", + "values": [ + { + "value": 2176 + } + ], + "title": "title5", + "description": "Description5.", + "id": "insta_id/insights/plays/lifetime" + }, + { + "name": "reach", + "period": "lifetime", + "values": [ + { + "value": 1842 + } + ], + "title": "title6", + "description": "Description6.", + "id": "insta_id/insights/reach/lifetime" + }, + { + "name": "saved", + "period": "lifetime", + "values": [ + { + "value": 7 + } + ], + "title": "title7", + "description": "Description7.", + "id": "insta_id/insights/saved/lifetime" + }, + { + "name": "shares", + "period": "lifetime", + "values": [ + { + "value": 1 + } + ], + "title": "title8", + "description": "Description8.", + "id": "insta_id/insights/shares/lifetime" + } + ] +} + +insights_record_transformed = { + "comments": 7, + "ig_reels_avg_watch_time": 11900, + "ig_reels_video_view_total_time": 25979677, + "likes": 102, + "plays": 2176, + "reach": 1842, + "saved": 7, + "shares": 1, +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_children_for_children.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_children_for_children.json new file mode 100644 index 000000000000..fae50f016276 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_children_for_children.json @@ -0,0 +1,12 @@ +{ + "id": "children_id", + "ig_id": "id_id", + "media_type": "IMAGE", + "media_url": "https://fake_media_url", + "owner": { + "id": "parent_media_id" + }, + "shortcode": "CL-UY87kjh", + "timestamp": "2021-03-03T22:48:39+0000", + "username": "username" +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_carousel_album.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_carousel_album.json new file mode 100644 index 000000000000..89d0b1c64f32 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_carousel_album.json @@ -0,0 +1,27 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 0, + "id": "66123508374641123", + "ig_id": "2521545922090007555", + "is_comment_enabled": true, + "like_count": 8, + "media_type": "CAROUSEL_ALBUM", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2021-03-03T22:48:39+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_children.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_children.json new file mode 100644 index 000000000000..8864e4f58ff7 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_children.json @@ -0,0 +1,43 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 0, + "id": "66123508374641123", + "ig_id": "2521545922090007555", + "is_comment_enabled": true, + "like_count": 8, + "media_type": "CAROUSEL_ALBUM", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2021-03-03T22:48:39+0000", + "username": "username", + "children": { + "data": [ + { + "id": "07608776690540123" + }, + { + "id": "52896800415362123" + }, + { + "id": "39559889460059123" + }, + { + "id": "17359925580923123" + } + ] + } + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_posted_before_business.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_posted_before_business.json new file mode 100644 index 000000000000..4aca1792b68b --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_posted_before_business.json @@ -0,0 +1,45 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 0, + "id": "35076616084176123", + "ig_id": "2034885879374760912", + "is_comment_enabled": true, + "like_count": 52, + "media_type": "IMAGE", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2019-05-02T11:42:01+0000", + "username": "username" + }, + { + "caption": "a caption", + "comments_count": 0, + "id": "35076616084176124", + "ig_id": "2034885879374760912", + "is_comment_enabled": true, + "like_count": 52, + "media_type": "IMAGE", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2019-05-02T11:42:01+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_with_wrong_permissions.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_with_wrong_permissions.json new file mode 100644 index 000000000000..a5bf32b755db --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_with_wrong_permissions.json @@ -0,0 +1,45 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 0, + "id": "35076616084176123", + "ig_id": "2034885879374760912", + "is_comment_enabled": true, + "like_count": 52, + "media_type": "IMAGE", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2019-05-02T11:42:01+0000", + "username": "username" + }, + { + "caption": "a caption", + "comments_count": 0, + "id": "35076616084176125", + "ig_id": "2034885879374760912", + "is_comment_enabled": true, + "like_count": 52, + "media_type": "IMAGE", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2019-05-02T11:42:01+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_with_wrong_permissions_code_10.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_with_wrong_permissions_code_10.json new file mode 100644 index 000000000000..48e717b65c22 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_error_with_wrong_permissions_code_10.json @@ -0,0 +1,45 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 0, + "id": "35076616084176123", + "ig_id": "2034885879374760912", + "is_comment_enabled": true, + "like_count": 52, + "media_type": "IMAGE", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2019-05-02T11:42:01+0000", + "username": "username" + }, + { + "caption": "a caption", + "comments_count": 0, + "id": "35076616084176126", + "ig_id": "2034885879374760912", + "is_comment_enabled": true, + "like_count": 52, + "media_type": "IMAGE", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2019-05-02T11:42:01+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_general_media.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_general_media.json new file mode 100644 index 000000000000..dac1d04cffd8 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_general_media.json @@ -0,0 +1,27 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 0, + "id": "35076616084176123", + "ig_id": "2034885879374760912", + "is_comment_enabled": true, + "like_count": 52, + "media_type": "IMAGE", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2019-05-02T11:42:01+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_reels.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_reels.json new file mode 100644 index 000000000000..865accb47c09 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_reels.json @@ -0,0 +1,118 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 2, + "id": "84386203808767123", + "ig_id": "3123724930722523505", + "is_comment_enabled": true, + "like_count": 12, + "media_type": "VIDEO", + "media_product_type": "REELS", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "thumbnail_url": "https://fakecontent.cdninstagram.com/v/somepath/", + "timestamp": "2023-06-12T19:20:02+0000", + "username": "username" + }, + { + "caption": "a caption", + "comments_count": 0, + "id": "90014330517797123", + "ig_id": "2927803201761078310", + "is_comment_enabled": true, + "like_count": 2, + "media_type": "VIDEO", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "thumbnail_url": "https://fakecontent.cdninstagram.com/v/somepath/", + "timestamp": "2022-09-15T11:33:25+0000", + "username": "username" + }, + { + "caption": "a caption", + "comments_count": 0, + "id": "09894619573775123", + "ig_id": "3091150577047429035", + "is_comment_enabled": true, + "like_count": 5, + "media_type": "VIDEO", + "media_product_type": "REELS", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "thumbnail_url": "https://fakecontent.cdninstagram.com/v/somepath/", + "timestamp": "2023-04-28T20:32:08+0000", + "username": "username" + }, + { + "caption": "a caption", + "comments_count": 0, + "id": "66123508374641123", + "ig_id": "2521545922090007555", + "is_comment_enabled": true, + "like_count": 8, + "media_type": "CAROUSEL_ALBUM", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2021-03-03T22:48:39+0000", + "username": "username", + "children": { + "data": [ + { + "id": "07608776690540123" + }, + { + "id": "52896800415362123" + }, + { + "id": "39559889460059123" + }, + { + "id": "17359925580923123" + } + ] + } + }, + { + "caption": "a caption", + "comments_count": 0, + "id": "35076616084176123", + "ig_id": "2034885879374760912", + "is_comment_enabled": true, + "like_count": 52, + "media_type": "IMAGE", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "timestamp": "2019-05-02T11:42:01+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_video.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_video.json new file mode 100644 index 000000000000..6b417b7540c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_video.json @@ -0,0 +1,28 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 0, + "id": "09894619573775123", + "ig_id": "3091150577047429035", + "is_comment_enabled": true, + "like_count": 5, + "media_type": "VIDEO", + "media_product_type": "STORY", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "thumbnail_url": "https://fakecontent.cdninstagram.com/v/somepath/", + "timestamp": "2023-04-28T20:32:08+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_video_feed.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_video_feed.json new file mode 100644 index 000000000000..be6ddcd2eb86 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_for_video_feed.json @@ -0,0 +1,28 @@ +{ + "data": [ + { + "caption": "a caption", + "comments_count": 0, + "id": "90014330517797123", + "ig_id": "2927803201761078310", + "is_comment_enabled": true, + "like_count": 2, + "media_type": "VIDEO", + "media_product_type": "FEED", + "media_url": "https://fakecontent.com/path/to/content", + "owner": { + "id": "41408147298757123" + }, + "permalink": "https://instagram.com/permalink/123", + "shortcode": "HGagdsy38", + "thumbnail_url": "https://fakecontent.cdninstagram.com/v/somepath/", + "timestamp": "2022-09-15T11:33:25+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUnZA1cW5ldERDZAlFPOWdfN1NKal9PU2o3R0s5ekY4eGNfRXczN09ud2k2NlFuZA3IzVHd0dlU3cFdJYWw5aXdna0xPVHpCRjRMSUdTYUUzeEtBalp0VDd3" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_carousel_album.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_carousel_album.json new file mode 100644 index 000000000000..5875a6a701b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_carousel_album.json @@ -0,0 +1,52 @@ +{ + "data": [ + { + "name": "impressions", + "period": "lifetime", + "values": [ + { + "value": 11000 + } + ], + "title": "Impressions", + "description": "The number of times your carousel album was displayed.", + "id": "66123508374641123/insights/impressions/lifetime" + }, + { + "name": "reach", + "period": "lifetime", + "values": [ + { + "value": 8500 + } + ], + "title": "Reach", + "description": "The number of unique accounts that viewed your carousel album.", + "id": "66123508374641123/insights/reach/lifetime" + }, + { + "name": "saved", + "period": "lifetime", + "values": [ + { + "value": 250 + } + ], + "title": "Saves", + "description": "The number of saves of your carousel album.", + "id": "66123508374641123/insights/saved/lifetime" + }, + { + "name": "video_views", + "period": "lifetime", + "values": [ + { + "value": 6000 + } + ], + "title": "Video Views", + "description": "The number of times videos in your carousel album were viewed.", + "id": "66123508374641123/insights/video_views/lifetime" + } + ] +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_posted_before_business.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_posted_before_business.json new file mode 100644 index 000000000000..eb83833ee33f --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_posted_before_business.json @@ -0,0 +1,15 @@ +{ + "error": { + "message": "Invalid parameter", + "type": "OAuthException", + "code": 100, + "error_data": { + "blame_field_specs": [[""]] + }, + "error_subcode": 2108006, + "is_transient": false, + "error_user_title": "Media posted before business account conversion", + "error_user_msg": "The media was posted before the most recent time that the user's account was converted to a business account from a personal account.", + "fbtrace_id": "AVul-nTRBqOlFGJxWKQv1g-" + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_with_wrong_permissions.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_with_wrong_permissions.json new file mode 100644 index 000000000000..f77687278920 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_with_wrong_permissions.json @@ -0,0 +1,12 @@ +{ + "error": { + "message": "Invalid parameter", + "type": "OAuthException", + "code": 100, + "error_data": {}, + "error_subcode": 33, + "is_transient": false, + "error_user_msg": "Unsupported get request. Object with ID 'test_id_3' does not exist, cannot be loaded due to missing permissions, or does not support this operation.", + "fbtrace_id": "fake_trace_id" + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_with_wrong_permissions_code_10.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_with_wrong_permissions_code_10.json new file mode 100644 index 000000000000..3754867d238d --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_error_with_wrong_permissions_code_10.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "(#10) Application does not have permission for this action", + "type": "OAuthException", + "code": 10, + "fbtrace_id": "fake_trace_id" + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_general_media.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_general_media.json new file mode 100644 index 000000000000..7e5c3fd640e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_general_media.json @@ -0,0 +1,88 @@ +{ + "data": [ + { + "name": "impressions", + "period": "lifetime", + "values": [ + { + "value": 10000 + } + ], + "title": "Impressions", + "description": "The number of times your media was displayed.", + "id": "35076616084176123/insights/impressions/lifetime" + }, + { + "name": "reach", + "period": "lifetime", + "values": [ + { + "value": 8000 + } + ], + "title": "Reach", + "description": "The number of unique accounts that viewed your media.", + "id": "35076616084176123/insights/reach/lifetime" + }, + { + "name": "saved", + "period": "lifetime", + "values": [ + { + "value": 200 + } + ], + "title": "Saves", + "description": "The number of saves of your media.", + "id": "35076616084176123/insights/saved/lifetime" + }, + { + "name": "video_views", + "period": "lifetime", + "values": [ + { + "value": 5000 + } + ], + "title": "Video Views", + "description": "The number of times your video was viewed.", + "id": "35076616084176123/insights/video_views/lifetime" + }, + { + "name": "likes", + "period": "lifetime", + "values": [ + { + "value": 150 + } + ], + "title": "Likes", + "description": "The number of likes on your media.", + "id": "35076616084176123/insights/likes/lifetime" + }, + { + "name": "comments", + "period": "lifetime", + "values": [ + { + "value": 50 + } + ], + "title": "Comments", + "description": "The number of comments on your media.", + "id": "35076616084176123/insights/comments/lifetime" + }, + { + "name": "shares", + "period": "lifetime", + "values": [ + { + "value": 25 + } + ], + "title": "Shares", + "description": "The number of shares of your media.", + "id": "35076616084176123/insights/shares/lifetime" + } + ] +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_reels.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_reels.json new file mode 100644 index 000000000000..bdc5ba7bfde1 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_reels.json @@ -0,0 +1,100 @@ +{ + "data": [ + { + "name": "comments", + "period": "lifetime", + "values": [ + { + "value": 7 + } + ], + "title": "Comments", + "description": "The number of comments on your reel.", + "id": "84386203808767123/insights/comments/lifetime" + }, + { + "name": "ig_reels_avg_watch_time", + "period": "lifetime", + "values": [ + { + "value": 12549 + } + ], + "title": "Average Watch Time", + "description": "The average watch time of your reel in milliseconds.", + "id": "84386203808767123/insights/ig_reels_avg_watch_time/lifetime" + }, + { + "name": "ig_reels_video_view_total_time", + "period": "lifetime", + "values": [ + { + "value": 30017581 + } + ], + "title": "Total View Time", + "description": "The total view time of your reel in milliseconds.", + "id": "84386203808767123/insights/ig_reels_video_view_total_time/lifetime" + }, + { + "name": "likes", + "period": "lifetime", + "values": [ + { + "value": 108 + } + ], + "title": "Likes", + "description": "The number of likes on your reel.", + "id": "84386203808767123/insights/likes/lifetime" + }, + { + "name": "plays", + "period": "lifetime", + "values": [ + { + "value": 2385 + } + ], + "title": "Plays", + "description": "The number of plays of your reel.", + "id": "84386203808767123/insights/plays/lifetime" + }, + { + "name": "reach", + "period": "lifetime", + "values": [ + { + "value": 2006 + } + ], + "title": "Reach", + "description": "The number of unique accounts that viewed your reel.", + "id": "84386203808767123/insights/reach/lifetime" + }, + { + "name": "saved", + "period": "lifetime", + "values": [ + { + "value": 7 + } + ], + "title": "Saves", + "description": "The number of saves of your reel.", + "id": "84386203808767123/insights/saved/lifetime" + }, + { + "name": "shares", + "period": "lifetime", + "values": [ + { + "value": 1 + } + ], + "title": "Shares", + "description": "The number of shares of your reel.", + "id": "84386203808767123/insights/shares/lifetime" + } + ] +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_video.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_video.json new file mode 100644 index 000000000000..de774fc17e82 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_video.json @@ -0,0 +1,88 @@ +{ + "data": [ + { + "name": "impressions", + "period": "lifetime", + "values": [ + { + "value": 12000 + } + ], + "title": "Impressions", + "description": "The number of times your video was displayed.", + "id": "09894619573775123/insights/impressions/lifetime" + }, + { + "name": "reach", + "period": "lifetime", + "values": [ + { + "value": 9500 + } + ], + "title": "Reach", + "description": "The number of unique accounts that viewed your video.", + "id": "09894619573775123/insights/reach/lifetime" + }, + { + "name": "saved", + "period": "lifetime", + "values": [ + { + "value": 300 + } + ], + "title": "Saves", + "description": "The number of saves of your video.", + "id": "09894619573775123/insights/saved/lifetime" + }, + { + "name": "video_views", + "period": "lifetime", + "values": [ + { + "value": 7000 + } + ], + "title": "Video Views", + "description": "The number of times your video was viewed.", + "id": "09894619573775123/insights/video_views/lifetime" + }, + { + "name": "likes", + "period": "lifetime", + "values": [ + { + "value": 150 + } + ], + "title": "Likes", + "description": "The number of likes on your video.", + "id": "09894619573775123/insights/likes/lifetime" + }, + { + "name": "comments", + "period": "lifetime", + "values": [ + { + "value": 50 + } + ], + "title": "Comments", + "description": "The number of comments on your video.", + "id": "09894619573775123/insights/comments/lifetime" + }, + { + "name": "shares", + "period": "lifetime", + "values": [ + { + "value": 25 + } + ], + "title": "Shares", + "description": "The number of shares of your video.", + "id": "09894619573775123/insights/shares/lifetime" + } + ] +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_video_feed.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_video_feed.json new file mode 100644 index 000000000000..660cdf9c70ab --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/media_insights_for_video_feed.json @@ -0,0 +1,52 @@ +{ + "data": [ + { + "name": "impressions", + "period": "lifetime", + "values": [ + { + "value": 10000 + } + ], + "title": "Impressions", + "description": "The number of times your video was displayed.", + "id": "90014330517797123/insights/impressions/lifetime" + }, + { + "name": "reach", + "period": "lifetime", + "values": [ + { + "value": 8000 + } + ], + "title": "Reach", + "description": "The number of unique accounts that viewed your video.", + "id": "90014330517797123/insights/reach/lifetime" + }, + { + "name": "saved", + "period": "lifetime", + "values": [ + { + "value": 200 + } + ], + "title": "Saves", + "description": "The number of saves of your video.", + "id": "90014330517797123/insights/saved/lifetime" + }, + { + "name": "video_views", + "period": "lifetime", + "values": [ + { + "value": 5000 + } + ], + "title": "Video Views", + "description": "The number of times your video was viewed.", + "id": "90014330517797123/insights/video_views/lifetime" + } + ] +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/stories_for_story_insights_error_code_10.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/stories_for_story_insights_error_code_10.json new file mode 100644 index 000000000000..ab2363498624 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/stories_for_story_insights_error_code_10.json @@ -0,0 +1,41 @@ +{ + "data": [ + { + "id": "3874523487643", + "ig_id": "ig_id", + "like_count": 0, + "media_type": "VIDEO", + "media_product_type": "STORY", + "media_url": "https://fakecontent.cdninstagram.com/path1/path2/some_value", + "owner": { + "id": "owner_id" + }, + "permalink": "https://placeholder.com/stories/username/some_id_value", + "shortcode": "ERUY34867_3", + "thumbnail_url": "https://content.cdnfaker.com/path1/path2/some_value", + "timestamp": "2024-06-17T19:39:18+0000", + "username": "username" + }, + { + "id": "3874523487644", + "ig_id": "ig_id", + "like_count": 0, + "media_type": "VIDEO", + "media_product_type": "STORY", + "media_url": "https://fakecontent.cdninstagram.com/path1/path2/some_value", + "owner": { + "id": "owner_id" + }, + "permalink": "https://placeholder.com/stories/username/some_id_value", + "shortcode": "ERUY34867_3", + "thumbnail_url": "https://content.cdnfaker.com/path1/path2/some_value", + "timestamp": "2024-06-17T19:39:18+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUjNIWl9ISTNCN3V" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/stories_for_story_insights_happy_path.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/stories_for_story_insights_happy_path.json new file mode 100644 index 000000000000..a71935c2361d --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/stories_for_story_insights_happy_path.json @@ -0,0 +1,25 @@ +{ + "data": [ + { + "id": "3874523487643", + "ig_id": "ig_id", + "like_count": 0, + "media_type": "VIDEO", + "media_product_type": "STORY", + "media_url": "https://fakecontent.cdninstagram.com/path1/path2/some_value", + "owner": { + "id": "owner_id" + }, + "permalink": "https://placeholder.com/stories/username/some_id_value", + "shortcode": "ERUY34867_3", + "thumbnail_url": "https://content.cdnfaker.com/path1/path2/some_value", + "timestamp": "2024-06-17T19:39:18+0000", + "username": "username" + } + ], + "paging": { + "cursors": { + "before": "QVFIUjNIWl9ISTNCN3V" + } + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/story_insights_for_story_insights_error_code_10.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/story_insights_for_story_insights_error_code_10.json new file mode 100644 index 000000000000..3754867d238d --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/story_insights_for_story_insights_error_code_10.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "(#10) Application does not have permission for this action", + "type": "OAuthException", + "code": 10, + "fbtrace_id": "fake_trace_id" + } +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/story_insights_for_story_insights_happy_path.json b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/story_insights_for_story_insights_happy_path.json new file mode 100644 index 000000000000..bc0fb3164579 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/resource/http/response/story_insights_for_story_insights_happy_path.json @@ -0,0 +1,40 @@ +{ + "data": [ + { + "name": "replies", + "period": "lifetime", + "values": [ + { + "value": 0 + } + ], + "title": "replies title", + "description": "replies description", + "id": "3874523487643/insights/replies/lifetime" + }, + { + "name": "impressions", + "period": "lifetime", + "values": [ + { + "value": 105 + } + ], + "title": "impressions title", + "description": "Impressions description.", + "id": "3874523487643/insights/impressions/lifetime" + }, + { + "name": "reach", + "period": "lifetime", + "values": [ + { + "value": 105 + } + ], + "title": "Reach title", + "description": "Reach description.", + "id": "3874523487643/insights/reach/lifetime" + } + ] +} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_components.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_components.py index 46204e80bdb0..fcd72fb5047d 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_components.py @@ -9,11 +9,14 @@ clear_url_record_transformed, expected_breakdown_record_transformed, expected_children_transformed, + insights_record, + insights_record_transformed, ) from source_instagram.components import ( GRAPH_URL, InstagramBreakDownResultsTransformation, InstagramClearUrlTransformation, + InstagramInsightsTransformation, InstagramMediaChildrenTransformation, ) @@ -43,3 +46,8 @@ def test_instagram_clear_url_transformation(): def test_break_down_results_transformation(): record_transformation_result = InstagramBreakDownResultsTransformation().transform(breakdowns_record) assert record_transformation_result == expected_breakdown_record_transformed + + +def test_instagram_insights_transformation(config): + record_transformation = InstagramInsightsTransformation().transform(insights_record) + assert record_transformation == insights_record_transformed diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 46b19b4679e8..58fe97e5002f 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -8,7 +8,7 @@ import pytest from airbyte_cdk.models import SyncMode from facebook_business import FacebookAdsApi, FacebookSession -from source_instagram.streams import DatetimeTransformerMixin, InstagramStream, MediaInsights, StoryInsights, UserInsights +from source_instagram.streams import DatetimeTransformerMixin, InstagramStream, UserInsights from utils import read_full_refresh, read_incremental FB_API_VERSION = FacebookAdsApi.API_VERSION @@ -30,101 +30,6 @@ def test_state_is_not_outdated(api, config): assert not UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format({"state": {}}) -def test_media_insights_read(api, user_stories_data, user_media_insights_data, requests_mock): - test_id = "test_id" - stream = MediaInsights(api=api) - - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/media", [{"json": user_stories_data}]) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", [{"json": user_media_insights_data}]) - - records = read_full_refresh(stream) - assert records == [{"business_account_id": "test_id", "id": "test_id", "impressions": 264, "page_id": "act_unknown_account"}] - - -def test_media_insights_read_error(api, requests_mock): - test_id = "test_id" - stream = MediaInsights(api=api) - media_response = [{"id": "test_id"}, {"id": "test_id_2"}, {"id": "test_id_3"}, {"id": "test_id_4"}, {"id": "test_id_5"}] - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/media", json={"data": media_response}) - - media_insights_response_test_id = { - "name": "impressions", - "period": "lifetime", - "values": [{"value": 264}], - "title": "Impressions", - "description": "Total number of times the media object has been seen", - "id": "test_id/insights/impressions/lifetime", - } - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", json=media_insights_response_test_id) - - error_response_oauth = { - "error": { - "message": "Invalid parameter", - "type": "OAuthException", - "code": 100, - "error_data": {}, - "error_subcode": 2108006, - "is_transient": False, - "error_user_title": "Media posted before business account conversion", - "error_user_msg": "The media was posted before the most recent time that the user's account was converted to a business account from a personal account.", - "fbtrace_id": "fake_trace_id", - } - } - requests_mock.register_uri( - "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_2/insights", json=error_response_oauth, status_code=400 - ) - - error_response_wrong_permissions = { - "error": { - "message": "Invalid parameter", - "type": "OAuthException", - "code": 100, - "error_data": {}, - "error_subcode": 33, - "is_transient": False, - "error_user_msg": "Unsupported get request. Object with ID 'test_id_3' does not exist, cannot be loaded due to missing permissions, or does not support this operation.", - "fbtrace_id": "fake_trace_id", - } - } - requests_mock.register_uri( - "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_3/insights", json=error_response_wrong_permissions, status_code=400 - ) - - media_insights_response_test_id_4 = { - "name": "impressions", - "period": "lifetime", - "values": [{"value": 300}], - "title": "Impressions", - "description": "Total number of times the media object has been seen", - "id": "test_id_3/insights/impressions/lifetime", - } - requests_mock.register_uri( - "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_4/insights", json=media_insights_response_test_id_4 - ) - - error_response_wrong_permissions_code_10 = { - "error": { - "message": "(#10) Application does not have permission for this action", - "type": "OAuthException", - "code": 10, - "fbtrace_id": "fake_trace_id", - } - } - requests_mock.register_uri( - "GET", - FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_5/insights", - json=error_response_wrong_permissions_code_10, - status_code=400, - ) - - records = read_full_refresh(stream) - expected_records = [ - {"business_account_id": "test_id", "id": "test_id", "impressions": 264, "page_id": "act_unknown_account"}, - {"business_account_id": "test_id", "id": "test_id_4", "impressions": 300, "page_id": "act_unknown_account"}, - ] - assert records == expected_records - - def test_user_insights_read(api, config, user_insight_data, requests_mock): test_id = "test_id" @@ -194,17 +99,6 @@ def test_user_insights_state(api, user_insights, values, slice_dates, expected): assert stream.state == expected -def test_stories_insights_read(api, requests_mock, user_stories_data, user_media_insights_data): - test_id = "test_id" - stream = StoryInsights(api=api) - - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/stories", [{"json": user_stories_data}]) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", [{"json": user_media_insights_data}]) - - records = read_full_refresh(stream) - assert records == [{"business_account_id": "test_id", "id": "test_id", "impressions": 264, "page_id": "act_unknown_account"}] - - @pytest.mark.parametrize( "error_response", [ @@ -251,6 +145,7 @@ def test_common_error_retry(config, error_response, requests_mock, api, account_ assert records + def test_exit_gracefully(api, config, requests_mock, caplog): test_id = "test_id" stream = UserInsights(api=api, start_date=config["start_date"]) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index 53b41209594a..2721072f0c84 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -3,7 +3,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.38.1' + cdkVersionRequired = '0.40.7' features = ['db-sources', 'datastore-mongo'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml index 412e0255bc63..f0694bbd723e 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: database connectorType: source definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 1.4.1 + dockerImageTag: 1.4.2 dockerRepository: airbyte/source-mongodb-v2 documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 githubIssueLabel: source-mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle index fbc0839042b9..087101dbb966 100644 --- a/airbyte-integrations/connectors/source-mssql/build.gradle +++ b/airbyte-integrations/connectors/source-mssql/build.gradle @@ -3,7 +3,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.40.1' + cdkVersionRequired = '0.40.7' features = ['db-sources'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/source-mssql/metadata.yaml b/airbyte-integrations/connectors/source-mssql/metadata.yaml index 616badf675c2..f900cdf411b1 100644 --- a/airbyte-integrations/connectors/source-mssql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mssql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 4.0.32 + dockerImageTag: 4.0.34 dockerRepository: airbyte/source-mssql documentationUrl: https://docs.airbyte.com/integrations/sources/mssql githubIssueLabel: source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index db3d3fa7ca96..2574b401e614 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -30,9 +30,6 @@ import io.airbyte.cdk.integrations.base.Source; import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; -import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.cdk.integrations.debezium.CdcStateHandler; -import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; import io.airbyte.cdk.integrations.debezium.internals.*; import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.cdk.integrations.source.relationaldb.InitialLoadHandler; @@ -49,7 +46,6 @@ import io.airbyte.commons.stream.AirbyteStreamStatusHolder; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.source.mssql.cursor_based.MssqlCursorBasedStateManager; import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadHandler; import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadStateManager; @@ -60,8 +56,6 @@ import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.*; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import io.debezium.connector.sqlserver.Lsn; -import io.debezium.engine.ChangeEvent; import java.io.IOException; import java.net.URI; import java.security.KeyStoreException; @@ -71,7 +65,6 @@ import java.time.Duration; import java.time.Instant; import java.util.*; -import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -456,42 +449,6 @@ protected void assertSqlServerAgentRunning(final JdbcDatabase database) throws S return super.getIncrementalIterators(database, catalog, tableNameToTable, stateManager, emittedAt); } - public AutoCloseableIterator getDebeziumSnapshotIterators( - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final CdcTargetPosition targetPosition, - final Duration firstRecordWaitTime, - final Duration subsequentRecordWaitTime, - final MssqlCdcConnectorMetadataInjector cdcMetadataInjector, - final Properties properties, - final CdcStateHandler cdcStateHandler, - final Instant emittedAt) { - - LOGGER.info("Running snapshot for " + catalog.getStreams().size() + " new tables"); - final var queue = new LinkedBlockingQueue>(AirbyteDebeziumHandler.QUEUE_CAPACITY); - - final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeDummyStateForSnapshotPurpose(); - final var emptyHistory = new AirbyteSchemaHistoryStorage.SchemaHistory>(Optional.empty(), false); - final var schemaHistoryManager = AirbyteSchemaHistoryStorage.initializeDBHistory(emptyHistory, cdcStateHandler.compressSchemaHistoryForState()); - final var propertiesManager = new RelationalDbDebeziumPropertiesManager(properties, config, catalog, Collections.emptyList()); - final DebeziumRecordPublisher tableSnapshotPublisher = new DebeziumRecordPublisher(propertiesManager); - tableSnapshotPublisher.start(queue, offsetManager, Optional.of(schemaHistoryManager)); - - final AutoCloseableIterator eventIterator = new DebeziumRecordIterator<>( - queue, - targetPosition, - tableSnapshotPublisher::hasClosed, - new DebeziumShutdownProcedure<>(queue, tableSnapshotPublisher::close, tableSnapshotPublisher::hasClosed), - firstRecordWaitTime); - - final var eventConverter = new RelationalDbDebeziumEventConverter(cdcMetadataInjector, emittedAt); - return AutoCloseableIterators.concatWithEagerClose( - AutoCloseableIterators.transform(eventIterator, eventConverter::toAirbyteMessage), - AutoCloseableIterators.fromIterator( - MoreIterators.singletonIteratorFromSupplier( - cdcStateHandler::saveStateAfterCompletionOfSnapshotOfNewStreams))); - } - @Override protected int getStateEmissionFrequency() { return INTERMEDIATE_STATE_EMISSION_FREQUENCY; diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadHandler.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadHandler.java index 0a5e6a5f0a79..afda9c70acb7 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadHandler.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/initialsync/MssqlInitialLoadHandler.java @@ -9,6 +9,7 @@ import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_SCHEMA_NAME; import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_TABLE_NAME; import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_TYPE; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.getFullyQualifiedTableName; import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY; import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; @@ -42,6 +43,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,7 +94,7 @@ public static String discoverClusteredIndexForStream(final JdbcDatabase database if (r.getShort(JDBC_COLUMN_TYPE) == DatabaseMetaData.tableIndexClustered) { final String schemaName = r.getObject(JDBC_COLUMN_SCHEMA_NAME) != null ? r.getString(JDBC_COLUMN_SCHEMA_NAME) : r.getString(JDBC_COLUMN_DATABASE_NAME); - final String streamName = JdbcUtils.getFullyQualifiedTableName(schemaName, r.getString(JDBC_COLUMN_TABLE_NAME)); + final String streamName = getFullyQualifiedTableName(schemaName, r.getString(JDBC_COLUMN_TABLE_NAME)); final String columnName = r.getString(JDBC_COLUMN_COLUMN_NAME); return new ClusteredIndexAttributesFromDb(streamName, columnName); } else { @@ -102,7 +104,12 @@ public static String discoverClusteredIndexForStream(final JdbcDatabase database } catch (final SQLException e) { LOGGER.debug(String.format("Could not retrieve clustered indexes without a table name (%s), not blocking, fall back to use pk.", e)); } - return clusteredIndexes.getOrDefault(stream.getName(), null); + LOGGER.debug("clusteredIndexes: {}", StringUtils.join(clusteredIndexes)); + final String streamName = stream.getName(); + final String namespace = stream.getNamespace(); + + return clusteredIndexes.getOrDefault( + getFullyQualifiedTableName(namespace, streamName), null); } @VisibleForTesting diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java index 38a8b554a1f0..6daecf68817d 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java @@ -7,20 +7,18 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mssql.initialsync.MssqlInitialLoadHandler; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; +import io.airbyte.protocol.models.v0.*; +import java.sql.SQLException; import java.util.Collections; import java.util.List; import org.junit.jupiter.api.*; @@ -117,4 +115,34 @@ public void testTableWithNullCursorValueShouldThrowException() throws Exception "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='dbo.id_and_name', cursorColumnName='id', cursorSqlType=INTEGER, cause=Cursor column contains NULL value}"); } + @Test + void testDiscoverWithNonClusteredPk() throws SQLException { + testdb + .with("ALTER TABLE id_and_name ADD CONSTRAINT i3pk PRIMARY KEY NONCLUSTERED (id);") + .with("CREATE INDEX i1 ON id_and_name (id);") + .with("CREATE CLUSTERED INDEX n1 ON id_and_name (name)"); + final AirbyteCatalog actual = source().discover(getConfig()); + assertEquals(CATALOG, actual); + final var db = source().createDatabase(getConfig()); + final String oc = MssqlInitialLoadHandler.discoverClusteredIndexForStream(db, + new AirbyteStream().withName( + actual.getStreams().get(0).getName()).withNamespace(actual.getStreams().get(0).getNamespace())); + assertEquals(oc, "name"); + } + + @Test + void testDiscoverWithNoClusteredIndex() throws SQLException { + testdb + .with("ALTER TABLE id_and_name ADD CONSTRAINT i3pk PRIMARY KEY NONCLUSTERED (id);") + .with("CREATE INDEX i1 ON id_and_name (id);") + .with("CREATE NONCLUSTERED INDEX n1 ON id_and_name (name)"); + final AirbyteCatalog actual = source().discover(getConfig()); + assertEquals(CATALOG, actual); + final var db = source().createDatabase(getConfig()); + final String oc = MssqlInitialLoadHandler.discoverClusteredIndexForStream(db, + new AirbyteStream().withName( + actual.getStreams().get(0).getName()).withNamespace(actual.getStreams().get(0).getNamespace())); + assertNull(oc); + } + } diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index a754846d6dcc..b503df498236 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -6,7 +6,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.40.1' + cdkVersionRequired = '0.40.7' features = ['db-sources'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/source-mysql/metadata.yaml b/airbyte-integrations/connectors/source-mysql/metadata.yaml index 1112746e9c6a..55ea492844af 100644 --- a/airbyte-integrations/connectors/source-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.4.11 + dockerImageTag: 3.4.12 dockerRepository: airbyte/source-mysql documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index f5ba6d9efeea..0d6f1543d68e 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -12,7 +12,7 @@ java { } airbyteJavaConnector { - cdkVersionRequired = '0.40.1' + cdkVersionRequired = '0.40.7' features = ['db-sources', 'datastore-postgres'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index 968d32d5e9ce..91381808ad53 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 3.4.20 + dockerImageTag: 3.4.21 dockerRepository: airbyte/source-postgres documentationUrl: https://docs.airbyte.com/integrations/sources/postgres githubIssueLabel: source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java index bfd4903cef9f..4f54e714f282 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java @@ -198,7 +198,7 @@ public static String prettyPrintConfiguredAirbyteStreamList(final List=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] -name = "airbyte-protocol-models" +name = "airbyte-protocol-models-pdv2" version = "0.12.2" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.12.2-py3-none-any.whl", hash = "sha256:1780db5b26285865b858d26502933def8e11919c9436ccf7b8b9cb0170b07c2a"}, - {file = "airbyte_protocol_models-0.12.2.tar.gz", hash = "sha256:b7c4d9a7c32c0691601c2b9416af090a858e126666e2c8c880d7a1798eb519f0"}, + {file = "airbyte_protocol_models_pdv2-0.12.2-py3-none-any.whl", hash = "sha256:8b3f9d0388928547cdf2e9134c0d589e4bcaa6f63bf71a21299f6824bfb7ad0e"}, + {file = "airbyte_protocol_models_pdv2-0.12.2.tar.gz", hash = "sha256:130c9ab289f3f53749ce63ff1abbfb67a44b7e5bd2794865315a2976138b672b"}, ] [package.dependencies] -pydantic = ">=1.9.2,<2.0.0" +pydantic = ">=2.7.2,<3.0.0" + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] [[package]] name = "attrs" @@ -449,22 +460,22 @@ files = [ [[package]] name = "graphql-query" -version = "1.1.1" -description = "Complete GraphQL query string generation for python." +version = "1.3.2" +description = "Complete Domain Specific Language (DSL) for GraphQL query in Python." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "graphql_query-1.1.1-py3-none-any.whl", hash = "sha256:c88e052d5739cbc0c20ed4597892025b2e3c363c75b44a06ea55005f09ecbd3d"}, - {file = "graphql_query-1.1.1.tar.gz", hash = "sha256:b0f52b560977fdf48c5ca48bb35772d86632018e13e948b10681815a58945b07"}, + {file = "graphql_query-1.3.2-py3-none-any.whl", hash = "sha256:5a2e771c9a78817fa2b9a7186f5fb7f7d6b02b5589bd4e80e5175cf4d4a4882e"}, + {file = "graphql_query-1.3.2.tar.gz", hash = "sha256:d54e8d9edad1a9b769e3fd6711753aa0cc8ec0609737a2c429051c37ed488c28"}, ] [package.dependencies] jinja2 = ">=3.1,<3.2" -pydantic = ">=1.10,<1.11" +pydantic = ">=2" [package.extras] dev = ["black", "mypy", "pylint-pydantic", "ruff", "wheel"] -docs = ["sphinx", "sphinx-argparse", "sphinx-rtd-theme", "sphinxcontrib-github"] +docs = ["mkdocs", "mkdocs-material"] test = ["pytest", "pytest-cov", "pytest-mock"] [[package]] @@ -828,62 +839,113 @@ files = [ [[package]] name = "pydantic" -version = "1.10.17" -description = "Data validation and settings management using python type hints" +version = "2.7.4" +description = "Data validation using Python type hints" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pydantic-1.10.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fa51175313cc30097660b10eec8ca55ed08bfa07acbfe02f7a42f6c242e9a4b"}, - {file = "pydantic-1.10.17-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7e8988bb16988890c985bd2093df9dd731bfb9d5e0860db054c23034fab8f7a"}, - {file = "pydantic-1.10.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:371dcf1831f87c9e217e2b6a0c66842879a14873114ebb9d0861ab22e3b5bb1e"}, - {file = "pydantic-1.10.17-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4866a1579c0c3ca2c40575398a24d805d4db6cb353ee74df75ddeee3c657f9a7"}, - {file = "pydantic-1.10.17-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:543da3c6914795b37785703ffc74ba4d660418620cc273490d42c53949eeeca6"}, - {file = "pydantic-1.10.17-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7623b59876f49e61c2e283551cc3647616d2fbdc0b4d36d3d638aae8547ea681"}, - {file = "pydantic-1.10.17-cp310-cp310-win_amd64.whl", hash = "sha256:409b2b36d7d7d19cd8310b97a4ce6b1755ef8bd45b9a2ec5ec2b124db0a0d8f3"}, - {file = "pydantic-1.10.17-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fa43f362b46741df8f201bf3e7dff3569fa92069bcc7b4a740dea3602e27ab7a"}, - {file = "pydantic-1.10.17-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2a72d2a5ff86a3075ed81ca031eac86923d44bc5d42e719d585a8eb547bf0c9b"}, - {file = "pydantic-1.10.17-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4ad32aed3bf5eea5ca5decc3d1bbc3d0ec5d4fbcd72a03cdad849458decbc63"}, - {file = "pydantic-1.10.17-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeb4e741782e236ee7dc1fb11ad94dc56aabaf02d21df0e79e0c21fe07c95741"}, - {file = "pydantic-1.10.17-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d2f89a719411cb234105735a520b7c077158a81e0fe1cb05a79c01fc5eb59d3c"}, - {file = "pydantic-1.10.17-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db3b48d9283d80a314f7a682f7acae8422386de659fffaba454b77a083c3937d"}, - {file = "pydantic-1.10.17-cp311-cp311-win_amd64.whl", hash = "sha256:9c803a5113cfab7bbb912f75faa4fc1e4acff43e452c82560349fff64f852e1b"}, - {file = "pydantic-1.10.17-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:820ae12a390c9cbb26bb44913c87fa2ff431a029a785642c1ff11fed0a095fcb"}, - {file = "pydantic-1.10.17-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1e51d1af306641b7d1574d6d3307eaa10a4991542ca324f0feb134fee259815"}, - {file = "pydantic-1.10.17-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e53fb834aae96e7b0dadd6e92c66e7dd9cdf08965340ed04c16813102a47fab"}, - {file = "pydantic-1.10.17-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e2495309b1266e81d259a570dd199916ff34f7f51f1b549a0d37a6d9b17b4dc"}, - {file = "pydantic-1.10.17-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:098ad8de840c92ea586bf8efd9e2e90c6339d33ab5c1cfbb85be66e4ecf8213f"}, - {file = "pydantic-1.10.17-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:525bbef620dac93c430d5d6bdbc91bdb5521698d434adf4434a7ef6ffd5c4b7f"}, - {file = "pydantic-1.10.17-cp312-cp312-win_amd64.whl", hash = "sha256:6654028d1144df451e1da69a670083c27117d493f16cf83da81e1e50edce72ad"}, - {file = "pydantic-1.10.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c87cedb4680d1614f1d59d13fea353faf3afd41ba5c906a266f3f2e8c245d655"}, - {file = "pydantic-1.10.17-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11289fa895bcbc8f18704efa1d8020bb9a86314da435348f59745473eb042e6b"}, - {file = "pydantic-1.10.17-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94833612d6fd18b57c359a127cbfd932d9150c1b72fea7c86ab58c2a77edd7c7"}, - {file = "pydantic-1.10.17-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d4ecb515fa7cb0e46e163ecd9d52f9147ba57bc3633dca0e586cdb7a232db9e3"}, - {file = "pydantic-1.10.17-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7017971ffa7fd7808146880aa41b266e06c1e6e12261768a28b8b41ba55c8076"}, - {file = "pydantic-1.10.17-cp37-cp37m-win_amd64.whl", hash = "sha256:e840e6b2026920fc3f250ea8ebfdedf6ea7a25b77bf04c6576178e681942ae0f"}, - {file = "pydantic-1.10.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bfbb18b616abc4df70591b8c1ff1b3eabd234ddcddb86b7cac82657ab9017e33"}, - {file = "pydantic-1.10.17-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ebb249096d873593e014535ab07145498957091aa6ae92759a32d40cb9998e2e"}, - {file = "pydantic-1.10.17-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8c209af63ccd7b22fba94b9024e8b7fd07feffee0001efae50dd99316b27768"}, - {file = "pydantic-1.10.17-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d4b40c9e13a0b61583e5599e7950490c700297b4a375b55b2b592774332798b7"}, - {file = "pydantic-1.10.17-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c31d281c7485223caf6474fc2b7cf21456289dbaa31401844069b77160cab9c7"}, - {file = "pydantic-1.10.17-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ae5184e99a060a5c80010a2d53c99aee76a3b0ad683d493e5f0620b5d86eeb75"}, - {file = "pydantic-1.10.17-cp38-cp38-win_amd64.whl", hash = "sha256:ad1e33dc6b9787a6f0f3fd132859aa75626528b49cc1f9e429cdacb2608ad5f0"}, - {file = "pydantic-1.10.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e17c0ee7192e54a10943f245dc79e36d9fe282418ea05b886e1c666063a7b54"}, - {file = "pydantic-1.10.17-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cafb9c938f61d1b182dfc7d44a7021326547b7b9cf695db5b68ec7b590214773"}, - {file = "pydantic-1.10.17-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95ef534e3c22e5abbdbdd6f66b6ea9dac3ca3e34c5c632894f8625d13d084cbe"}, - {file = "pydantic-1.10.17-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62d96b8799ae3d782df7ec9615cb59fc32c32e1ed6afa1b231b0595f6516e8ab"}, - {file = "pydantic-1.10.17-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ab2f976336808fd5d539fdc26eb51f9aafc1f4b638e212ef6b6f05e753c8011d"}, - {file = "pydantic-1.10.17-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8ad363330557beac73159acfbeed220d5f1bfcd6b930302a987a375e02f74fd"}, - {file = "pydantic-1.10.17-cp39-cp39-win_amd64.whl", hash = "sha256:48db882e48575ce4b39659558b2f9f37c25b8d348e37a2b4e32971dd5a7d6227"}, - {file = "pydantic-1.10.17-py3-none-any.whl", hash = "sha256:e41b5b973e5c64f674b3b4720286ded184dcc26a691dd55f34391c62c6934688"}, - {file = "pydantic-1.10.17.tar.gz", hash = "sha256:f434160fb14b353caf634149baaf847206406471ba70e64657c1e8330277a991"}, + {file = "pydantic-2.7.4-py3-none-any.whl", hash = "sha256:ee8538d41ccb9c0a9ad3e0e5f07bf15ed8015b481ced539a1759d8cc89ae90d0"}, + {file = "pydantic-2.7.4.tar.gz", hash = "sha256:0c84efd9548d545f63ac0060c1e4d39bb9b14db8b3c0652338aecc07b5adec52"}, ] [package.dependencies] -typing-extensions = ">=4.2.0" +annotated-types = ">=0.4.0" +pydantic-core = "2.18.4" +typing-extensions = ">=4.6.1" [package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.18.4" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.18.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f76d0ad001edd426b92233d45c746fd08f467d56100fd8f30e9ace4b005266e4"}, + {file = "pydantic_core-2.18.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:59ff3e89f4eaf14050c8022011862df275b552caef8082e37b542b066ce1ff26"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a55b5b16c839df1070bc113c1f7f94a0af4433fcfa1b41799ce7606e5c79ce0a"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4d0dcc59664fcb8974b356fe0a18a672d6d7cf9f54746c05f43275fc48636851"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8951eee36c57cd128f779e641e21eb40bc5073eb28b2d23f33eb0ef14ffb3f5d"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4701b19f7e3a06ea655513f7938de6f108123bf7c86bbebb1196eb9bd35cf724"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e00a3f196329e08e43d99b79b286d60ce46bed10f2280d25a1718399457e06be"}, + {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:97736815b9cc893b2b7f663628e63f436018b75f44854c8027040e05230eeddb"}, + {file = "pydantic_core-2.18.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6891a2ae0e8692679c07728819b6e2b822fb30ca7445f67bbf6509b25a96332c"}, + {file = "pydantic_core-2.18.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bc4ff9805858bd54d1a20efff925ccd89c9d2e7cf4986144b30802bf78091c3e"}, + {file = "pydantic_core-2.18.4-cp310-none-win32.whl", hash = "sha256:1b4de2e51bbcb61fdebd0ab86ef28062704f62c82bbf4addc4e37fa4b00b7cbc"}, + {file = "pydantic_core-2.18.4-cp310-none-win_amd64.whl", hash = "sha256:6a750aec7bf431517a9fd78cb93c97b9b0c496090fee84a47a0d23668976b4b0"}, + {file = "pydantic_core-2.18.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:942ba11e7dfb66dc70f9ae66b33452f51ac7bb90676da39a7345e99ffb55402d"}, + {file = "pydantic_core-2.18.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b2ebef0e0b4454320274f5e83a41844c63438fdc874ea40a8b5b4ecb7693f1c4"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a642295cd0c8df1b86fc3dced1d067874c353a188dc8e0f744626d49e9aa51c4"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f09baa656c904807e832cf9cce799c6460c450c4ad80803517032da0cd062e2"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98906207f29bc2c459ff64fa007afd10a8c8ac080f7e4d5beff4c97086a3dabd"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19894b95aacfa98e7cb093cd7881a0c76f55731efad31073db4521e2b6ff5b7d"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fbbdc827fe5e42e4d196c746b890b3d72876bdbf160b0eafe9f0334525119c8"}, + {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f85d05aa0918283cf29a30b547b4df2fbb56b45b135f9e35b6807cb28bc47951"}, + {file = "pydantic_core-2.18.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e85637bc8fe81ddb73fda9e56bab24560bdddfa98aa64f87aaa4e4b6730c23d2"}, + {file = "pydantic_core-2.18.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2f5966897e5461f818e136b8451d0551a2e77259eb0f73a837027b47dc95dab9"}, + {file = "pydantic_core-2.18.4-cp311-none-win32.whl", hash = "sha256:44c7486a4228413c317952e9d89598bcdfb06399735e49e0f8df643e1ccd0558"}, + {file = "pydantic_core-2.18.4-cp311-none-win_amd64.whl", hash = "sha256:8a7164fe2005d03c64fd3b85649891cd4953a8de53107940bf272500ba8a788b"}, + {file = "pydantic_core-2.18.4-cp311-none-win_arm64.whl", hash = "sha256:4e99bc050fe65c450344421017f98298a97cefc18c53bb2f7b3531eb39bc7805"}, + {file = "pydantic_core-2.18.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6f5c4d41b2771c730ea1c34e458e781b18cc668d194958e0112455fff4e402b2"}, + {file = "pydantic_core-2.18.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2fdf2156aa3d017fddf8aea5adfba9f777db1d6022d392b682d2a8329e087cef"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4748321b5078216070b151d5271ef3e7cc905ab170bbfd27d5c83ee3ec436695"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847a35c4d58721c5dc3dba599878ebbdfd96784f3fb8bb2c356e123bdcd73f34"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c40d4eaad41f78e3bbda31b89edc46a3f3dc6e171bf0ecf097ff7a0ffff7cb1"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:21a5e440dbe315ab9825fcd459b8814bb92b27c974cbc23c3e8baa2b76890077"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01dd777215e2aa86dfd664daed5957704b769e726626393438f9c87690ce78c3"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b06beb3b3f1479d32befd1f3079cc47b34fa2da62457cdf6c963393340b56e9"}, + {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:564d7922e4b13a16b98772441879fcdcbe82ff50daa622d681dd682175ea918c"}, + {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0eb2a4f660fcd8e2b1c90ad566db2b98d7f3f4717c64fe0a83e0adb39766d5b8"}, + {file = "pydantic_core-2.18.4-cp312-none-win32.whl", hash = "sha256:8b8bab4c97248095ae0c4455b5a1cd1cdd96e4e4769306ab19dda135ea4cdb07"}, + {file = "pydantic_core-2.18.4-cp312-none-win_amd64.whl", hash = "sha256:14601cdb733d741b8958224030e2bfe21a4a881fb3dd6fbb21f071cabd48fa0a"}, + {file = "pydantic_core-2.18.4-cp312-none-win_arm64.whl", hash = "sha256:c1322d7dd74713dcc157a2b7898a564ab091ca6c58302d5c7b4c07296e3fd00f"}, + {file = "pydantic_core-2.18.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:823be1deb01793da05ecb0484d6c9e20baebb39bd42b5d72636ae9cf8350dbd2"}, + {file = "pydantic_core-2.18.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ebef0dd9bf9b812bf75bda96743f2a6c5734a02092ae7f721c048d156d5fabae"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae1d6df168efb88d7d522664693607b80b4080be6750c913eefb77e34c12c71a"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9899c94762343f2cc2fc64c13e7cae4c3cc65cdfc87dd810a31654c9b7358cc"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99457f184ad90235cfe8461c4d70ab7dd2680e28821c29eca00252ba90308c78"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18f469a3d2a2fdafe99296a87e8a4c37748b5080a26b806a707f25a902c040a8"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cdf28938ac6b8b49ae5e92f2735056a7ba99c9b110a474473fd71185c1af5d"}, + {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:938cb21650855054dc54dfd9120a851c974f95450f00683399006aa6e8abb057"}, + {file = "pydantic_core-2.18.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:44cd83ab6a51da80fb5adbd9560e26018e2ac7826f9626bc06ca3dc074cd198b"}, + {file = "pydantic_core-2.18.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:972658f4a72d02b8abfa2581d92d59f59897d2e9f7e708fdabe922f9087773af"}, + {file = "pydantic_core-2.18.4-cp38-none-win32.whl", hash = "sha256:1d886dc848e60cb7666f771e406acae54ab279b9f1e4143babc9c2258213daa2"}, + {file = "pydantic_core-2.18.4-cp38-none-win_amd64.whl", hash = "sha256:bb4462bd43c2460774914b8525f79b00f8f407c945d50881568f294c1d9b4443"}, + {file = "pydantic_core-2.18.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:44a688331d4a4e2129140a8118479443bd6f1905231138971372fcde37e43528"}, + {file = "pydantic_core-2.18.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a2fdd81edd64342c85ac7cf2753ccae0b79bf2dfa063785503cb85a7d3593223"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86110d7e1907ab36691f80b33eb2da87d780f4739ae773e5fc83fb272f88825f"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46387e38bd641b3ee5ce247563b60c5ca098da9c56c75c157a05eaa0933ed154"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:123c3cec203e3f5ac7b000bd82235f1a3eced8665b63d18be751f115588fea30"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dc1803ac5c32ec324c5261c7209e8f8ce88e83254c4e1aebdc8b0a39f9ddb443"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53db086f9f6ab2b4061958d9c276d1dbe3690e8dd727d6abf2321d6cce37fa94"}, + {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abc267fa9837245cc28ea6929f19fa335f3dc330a35d2e45509b6566dc18be23"}, + {file = "pydantic_core-2.18.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0d829524aaefdebccb869eed855e2d04c21d2d7479b6cada7ace5448416597b"}, + {file = "pydantic_core-2.18.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:509daade3b8649f80d4e5ff21aa5673e4ebe58590b25fe42fac5f0f52c6f034a"}, + {file = "pydantic_core-2.18.4-cp39-none-win32.whl", hash = "sha256:ca26a1e73c48cfc54c4a76ff78df3727b9d9f4ccc8dbee4ae3f73306a591676d"}, + {file = "pydantic_core-2.18.4-cp39-none-win_amd64.whl", hash = "sha256:c67598100338d5d985db1b3d21f3619ef392e185e71b8d52bceacc4a7771ea7e"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:574d92eac874f7f4db0ca653514d823a0d22e2354359d0759e3f6a406db5d55d"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1f4d26ceb5eb9eed4af91bebeae4b06c3fb28966ca3a8fb765208cf6b51102ab"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77450e6d20016ec41f43ca4a6c63e9fdde03f0ae3fe90e7c27bdbeaece8b1ed4"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d323a01da91851a4f17bf592faf46149c9169d68430b3146dcba2bb5e5719abc"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43d447dd2ae072a0065389092a231283f62d960030ecd27565672bd40746c507"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:578e24f761f3b425834f297b9935e1ce2e30f51400964ce4801002435a1b41ef"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:81b5efb2f126454586d0f40c4d834010979cb80785173d1586df845a632e4e6d"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ab86ce7c8f9bea87b9d12c7f0af71102acbf5ecbc66c17796cff45dae54ef9a5"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:90afc12421df2b1b4dcc975f814e21bc1754640d502a2fbcc6d41e77af5ec312"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:51991a89639a912c17bef4b45c87bd83593aee0437d8102556af4885811d59f5"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:293afe532740370aba8c060882f7d26cfd00c94cae32fd2e212a3a6e3b7bc15e"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b48ece5bde2e768197a2d0f6e925f9d7e3e826f0ad2271120f8144a9db18d5c8"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eae237477a873ab46e8dd748e515c72c0c804fb380fbe6c85533c7de51f23a8f"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:834b5230b5dfc0c1ec37b2fda433b271cbbc0e507560b5d1588e2cc1148cf1ce"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e858ac0a25074ba4bce653f9b5d0a85b7456eaddadc0ce82d3878c22489fa4ee"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2fd41f6eff4c20778d717af1cc50eca52f5afe7805ee530a4fbd0bae284f16e9"}, + {file = "pydantic_core-2.18.4.tar.gz", hash = "sha256:ec3beeada09ff865c344ff3bc2f427f5e6c26401cc6113d77e372c3fdac73864"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pyjwt" @@ -1369,4 +1431,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "b17002936aa7fb4b8619fad1ea94d9d780f48d338bddc9644e2c7c4c27532138" +content-hash = "d51eabdeccc8e8f30149b5953fb06300e93d00d702ab31cd2acf67df0319a08b" diff --git a/airbyte-integrations/connectors/source-shopify/pyproject.toml b/airbyte-integrations/connectors/source-shopify/pyproject.toml index 2468512784b8..5b6d9088e434 100644 --- a/airbyte-integrations/connectors/source-shopify/pyproject.toml +++ b/airbyte-integrations/connectors/source-shopify/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.4.6" +version = "2.4.7" name = "source-shopify" description = "Source CDK implementation for Shopify." authors = [ "Airbyte ",] @@ -17,9 +17,9 @@ include = "source_shopify" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^1" +airbyte-cdk = "^2" sgqlc = "==16.3" -graphql-query = "^1.1.1" +graphql-query = "^1" [tool.poetry.scripts] source-shopify = "source_shopify.run:run" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py index abb8f4052543..9d58eac5c0d9 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py @@ -1203,23 +1203,29 @@ class InventoryLevel(ShopifyBulkQuery): node { __typename id - inventoryLevels(query: "updated_at:>='2023-04-14T00:00:00+00:00'") { + inventoryLevels(query: "updated_at:>='2020-04-13T00:00:00+00:00'") { edges { node { - __typename - id - available - item { - inventory_item_id: id - inventoryHistoryUrl - locationsCount { - count - } + __typename + id + canDeactivate + createdAt + deactivationAlert + updatedAt + item { + inventory_history_url: inventoryHistoryUrl + inventory_item_id: id + locations_count: locationsCount { + count + } + } + quantities( + names: ["available", "incoming", "committed", "damaged", "on_hand", "quality_control", "reserved", "safety_stock"]) { + id + name + quantity + updatedAt } - updatedAt - canDeactivate - createdAt - deactivationAlert } } } @@ -1251,13 +1257,6 @@ class InventoryLevel(ShopifyBulkQuery): '"reserved"', '"safety_stock"', ] - # quantities fields - quantities_fields: List[str] = [ - "id", - "name", - "quantity", - "updatedAt", - ] item_fields: List[Field] = [ Field(name="inventoryHistoryUrl", alias="inventory_history_url"), @@ -1280,12 +1279,27 @@ def _quantities_query(self) -> Query: Defines the `quantities` nested query. """ - return Query( + return Field( name="quantities", arguments=[Argument(name="names", value=self.quantities_names_filter)], - fields=self.quantities_fields, + fields=[ + "id", + "name", + "quantity", + "updatedAt", + ], ) + def _get_inventory_levels_fields(self, filter_query: Optional[str] = None) -> List[Field]: + nested_fields = self.inventory_levels_fields + [self._quantities_query()] + return self.query_nodes + [ + Field( + name="inventoryLevels", + arguments=[Argument(name="query", value=f'"{filter_query}"')], + fields=[Field(name="edges", fields=[Field(name="node", fields=nested_fields)])], + ) + ] + def _process_quantities(self, quantities: Iterable[MutableMapping[str, Any]] = None) -> Iterable[Mapping[str, Any]]: if quantities: for quantity in quantities: @@ -1299,15 +1313,10 @@ def _process_quantities(self, quantities: Iterable[MutableMapping[str, Any]] = N return [] def query(self, filter_query: Optional[str] = None) -> Query: - # construct the `quantities` query piece - quantities: List[Query] = [self._quantities_query()] - # build the nested query first with `filter_query` to have the incremental syncs - inventory_levels: List[Query] = [self.build("inventoryLevels", self.inventory_levels_fields + quantities, filter_query)] # build the main query around previous - # return the constructed query operation return self.build( name=self.query_name, - edges=self.query_nodes + inventory_levels, + edges=self._get_inventory_levels_fields(filter_query), # passing more query args for `locations` query additional_query_args=self.locations_query_args, ) @@ -2311,8 +2320,9 @@ def _merge_with_media(self, record_components: List[dict]) -> Optional[Iterable[ if BULK_PARENT_KEY in item: item.pop(BULK_PARENT_KEY) - image_url = item.get("image", {}).get("url") - if image_url in url_map: + image = item.get("image", {}) + image_url = image.get("url") if image else None + if image_url and image_url in url_map: # Merge images into media item.update(url_map.get(image_url)) # remove lefovers diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py index 496e527b09e4..238739377b3d 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py @@ -65,7 +65,7 @@ def _first_non_null_type(schema_types: List[str]) -> str: @staticmethod def _transform_number(value: Any): - return Decimal(value) + return float(Decimal(value)) @staticmethod def _transform_string(value: Any): diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py index af914a00c8b8..2b4ce1409daa 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py @@ -539,7 +539,9 @@ def product_images_jsonl_content_example(): {"__typename":"Image","id":"gid:\/\/shopify\/ProductImage\/111","height":280,"alt":"","src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/white-t-shirt.jpg?v=1673029759","url":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/white-t-shirt.jpg?v=1673029759","width":265,"__parentId":"gid:\/\/shopify\/Product\/123"} {"__typename":"Product","id":"gid:\/\/shopify\/Product\/456"} {"__typename":"MediaImage","createdAt":"2021-06-23T01:09:47Z","updatedAt":"2023-04-24T17:27:15Z","image":{"url":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587"},"__parentId":"gid:\/\/shopify\/Product\/456"} -{"__typename":"Image","id":"gid:\/\/shopify\/ProductImage\/222","height":1467,"alt":"updated_mon_24.04.2023","src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587","url":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587","width":2200,"__parentId":"gid:\/\/shopify\/Product\/456"}\n""" +{"__typename":"Image","id":"gid:\/\/shopify\/ProductImage\/222","height":1467,"alt":"updated_mon_24.04.2023","src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587","url":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587","width":2200,"__parentId":"gid:\/\/shopify\/Product\/456"} +{"__typename":"Product","id":"gid:\/\/shopify\/Product\/9062091161885"} +{"__typename":"MediaImage","createdAt":"2024-06-12T23:41:27Z","updatedAt":"2024-06-12T23:41:28Z","image":null,"__parentId":"gid:\/\/shopify\/Product\/9062091161885"}\n""" @pytest.fixture @@ -812,6 +814,12 @@ def product_images_response_expected_result(): "product_id": 456, "shop_url": "test_shop", }, + { + "created_at": "2024-06-12T23:41:27+00:00", + "updated_at": "2024-06-12T23:41:28+00:00", + "image": None, + "shop_url": "test_shop" + }, ] diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py index 03108425e039..87781b8e5538 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py @@ -187,7 +187,7 @@ def test_base_build_query(query_name, fields, filter_field, start, end, expected fields=[ '__typename', 'id', - Query( + Field( name="inventoryLevels", arguments=[ Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), @@ -206,7 +206,7 @@ def test_base_build_query(query_name, fields, filter_field, start, end, expected "deactivationAlert", "updatedAt", Field(name="item", fields=[Field(name="inventoryHistoryUrl", alias="inventory_history_url"), Field(name="id", alias="inventory_item_id"), Field(name="locationsCount", alias="locations_count", fields=["count"])]), - Query( + Field( name="quantities", arguments=[ Argument(name="names", value=['"available"', '"incoming"', '"committed"', '"damaged"', '"on_hand"', '"quality_control"', '"reserved"', '"safety_stock"']) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py index 0c74e94d763e..5101fb1585f2 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py @@ -125,7 +125,7 @@ def test_enforcer_correct_type(transform_object, schema, checks): "path": [ "total_discounts", ], - "expected_type": Decimal, + "expected_type": float, }, ], ), @@ -144,7 +144,7 @@ def test_enforcer_correct_type(transform_object, schema, checks): "path": [ "total_price", ], - "expected_type": Decimal, + "expected_type": float, }, ], ), @@ -168,7 +168,7 @@ def test_enforcer_string_to_number(transform_object, schema, checks): }, }, [ - {"path": ["customer", "total_spent"], "expected_type": Decimal}, + {"path": ["customer", "total_spent"], "expected_type": float}, ], ) ], @@ -194,7 +194,7 @@ def test_enforcer_nested_object(transform_object, schema, checks): }, }, [ - {"path": ["shipping_lines", 0, "price"], "expected_type": Decimal}, + {"path": ["shipping_lines", 0, "price"], "expected_type": float}, ], ), ( @@ -219,7 +219,7 @@ def test_enforcer_nested_object(transform_object, schema, checks): }, }, [ - {"path": ["line_items", 0, "line_price"], "expected_type": Decimal}, + {"path": ["line_items", 0, "line_price"], "expected_type": float}, ], ), ], @@ -247,8 +247,8 @@ def test_enforcer_nested_array(transform_object, schema, checks): }, }, [ - {"path": ["line_items", 0], "expected_type": Decimal}, - {"path": ["line_items", 1], "expected_type": Decimal}, + {"path": ["line_items", 0], "expected_type": float}, + {"path": ["line_items", 1], "expected_type": float}, ], ), ], diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/.coveragerc b/airbyte-integrations/connectors/source-snapchat-marketing/.coveragerc new file mode 100644 index 000000000000..b57f62f211fd --- /dev/null +++ b/airbyte-integrations/connectors/source-snapchat-marketing/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_snapchat_marketing/run.py diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state.json index 1a1e0f1c30fc..70b5c9c7636c 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state.json @@ -2,77 +2,244 @@ { "type": "STREAM", "stream": { - "stream_state": { - "updated_at": "2221-01-01T00:00:00Z" - }, "stream_descriptor": { - "name": "adaccounts" + "name": "adaccounts", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "updated_at": "2221-01-01T00:00:00Z" - }, "stream_descriptor": { - "name": "creatives" + "name": "creatives", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + }, + { + "partition": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "updated_at": "2221-01-01T00:00:00Z" - }, "stream_descriptor": { - "name": "media" + "name": "media", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + }, + { + "partition": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "updated_at": "2221-01-01T00:00:00Z" - }, "stream_descriptor": { - "name": "campaigns" + "name": "campaigns", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + }, + { + "partition": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "updated_at": "2221-01-01T00:00:00Z" - }, "stream_descriptor": { - "name": "ads" + "name": "ads", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + }, + { + "partition": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "updated_at": "2221-01-01T00:00:00Z" - }, "stream_descriptor": { - "name": "adsquads" + "name": "adsquads", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + }, + { + "partition": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "updated_at": "2221-01-01T00:00:00Z" - }, "stream_descriptor": { - "name": "segments" + "name": "segments", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + }, + { + "partition": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "updated_at": "2221-01-01T00:00:00.000Z" + } + } + ] } } } diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state_daily.json b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state_daily.json index 8b5afe5cedd4..771eb9ebd0f0 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state_daily.json +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state_daily.json @@ -2,44 +2,178 @@ { "type": "STREAM", "stream": { - "stream_state": { - "start_time": "2221-07-05" - }, "stream_descriptor": { - "name": "adaccounts_stats_daily" + "name": "adaccounts_stats_daily", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + }, + { + "partition": { + "id": "7a205dba-557f-4f00-82d7-c78853dcad39", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + }, + { + "partition": { + "id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "start_time": "2221-07-05" - }, "stream_descriptor": { - "name": "ads_stats_daily" + "name": "ads_stats_daily", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "ad_id": "96e5549f-4065-490e-93dd-ffb9f7973b77", + "parent_slice": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + }, + { + "partition": { + "id": "8831ae74-bd1e-4ea1-9628-76e549041bea", + "parent_slice": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "start_time": "2221-07-05" - }, "stream_descriptor": { - "name": "adsquads_stats_daily" + "name": "adsquads_stats_daily", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "adsquad_id": "ac6548b1-419e-4137-8320-e4e536766f72", + "parent_slice": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + }, + { + "partition": { + "id": "87f00f80-8ae6-44ac-ab63-2c976d43de64", + "parent_slice": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + } + ] } } }, { "type": "STREAM", "stream": { - "stream_state": { - "start_time": "2221-07-05" - }, "stream_descriptor": { - "name": "campaigns_stats_daily" + "name": "campaigns_stats_daily", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "campaign_id": "3369834f-5bc0-47bd-a4e9-ee48d424d90c", + "parent_slice": { + "adaccount_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + }, + { + "partition": { + "id": "d180d36e-1212-479b-84a5-8b0663ceaf82", + "parent_slice": { + "adaccount_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", + "parent_slice": { + "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", + "parent_slice": {} + } + } + }, + "cursor": { + "start_time": "2221-01-01T00:00:00.000-07:00" + } + } + ] } } } diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/catalog_hourly.json b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/catalog_hourly.json new file mode 100644 index 000000000000..1fe163b6c367 --- /dev/null +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/catalog_hourly.json @@ -0,0 +1,60 @@ +{ + "streams": [ + { + "stream": { + "name": "adaccounts_stats_hourly", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["start_time"], + "source_defined_primary_key": [["id"], ["granularity"], ["start_time"]] + }, + "sync_mode": "incremental", + "cursor_field": ["start_time"], + "primary_key": [["id"], ["granularity"], ["start_time"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads_stats_hourly", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["start_time"], + "source_defined_primary_key": [["id"], ["granularity"], ["start_time"]] + }, + "sync_mode": "incremental", + "cursor_field": ["start_time"], + "primary_key": [["id"], ["granularity"], ["start_time"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "adsquads_stats_hourly", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["start_time"], + "source_defined_primary_key": [["id"], ["granularity"], ["start_time"]] + }, + "sync_mode": "incremental", + "cursor_field": ["start_time"], + "primary_key": [["id"], ["granularity"], ["start_time"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns_stats_hourly", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["start_time"], + "source_defined_primary_key": [["id"], ["granularity"], ["start_time"]] + }, + "sync_mode": "incremental", + "cursor_field": ["start_time"], + "destination_sync_mode": "append", + "primary_key": [["id"], ["granularity"], ["start_time"]] + } + ] +} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/expected_records.jsonl index 8d96064dd009..bc6ec9ebac68 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/expected_records.jsonl @@ -22,4 +22,4 @@ {"stream": "adsquads_stats_hourly", "data": {"start_time": "2024-05-14T23:00:00.000-07:00", "end_time": "2024-05-15T00:00:00.000-07:00", "id": "87f00f80-8ae6-44ac-ab63-2c976d43de64", "type": "AD_SQUAD", "granularity": "HOUR", "android_installs": 0, "attachment_avg_view_time_millis": 0, "attachment_impressions": 0, "attachment_quartile_1": 0, "attachment_quartile_2": 0, "attachment_quartile_3": 0, "attachment_total_view_time_millis": 0, "attachment_view_completion": 0, "avg_screen_time_millis": 0, "avg_view_time_millis": 0, "impressions": 0, "ios_installs": 0, "quartile_1": 0, "quartile_2": 0, "quartile_3": 0, "screen_time_millis": 0, "swipe_up_percent": 0.0, "swipes": 0, "total_installs": 0, "video_views": 0, "video_views_time_based": 0, "video_views_15s": 0, "view_completion": 0, "view_time_millis": 0, "paid_impressions": 0, "earned_impressions": 0, "total_impressions": 0, "play_time_millis": 0, "shares": 0, "saves": 0, "native_leads": 0, "conversion_purchases": 0, "conversion_purchases_value": 0, "conversion_save": 0, "conversion_start_checkout": 0, "conversion_add_cart": 0, "conversion_view_content": 0, "conversion_add_billing": 0, "conversion_searches": 0, "conversion_level_completes": 0, "conversion_app_opens": 0, "conversion_page_views": 0, "conversion_subscribe": 0, "conversion_ad_click": 0, "conversion_ad_view": 0, "conversion_complete_tutorial": 0, "conversion_invite": 0, "conversion_login": 0, "conversion_share": 0, "conversion_reserve": 0, "conversion_achievement_unlocked": 0, "conversion_add_to_wishlist": 0, "conversion_spend_credits": 0, "conversion_rate": 0, "conversion_start_trial": 0, "conversion_list_view": 0, "custom_event_1": 0, "custom_event_2": 0, "custom_event_3": 0, "custom_event_4": 0, "custom_event_5": 0, "story_opens": 0, "story_completes": 0}, "emitted_at": 1718737703937} {"stream": "adsquads_stats_hourly", "data": {"start_time": "2024-05-15T00:00:00.000-07:00", "end_time": "2024-05-15T01:00:00.000-07:00", "id": "87f00f80-8ae6-44ac-ab63-2c976d43de64", "type": "AD_SQUAD", "granularity": "HOUR", "android_installs": 0, "attachment_avg_view_time_millis": 0, "attachment_impressions": 0, "attachment_quartile_1": 0, "attachment_quartile_2": 0, "attachment_quartile_3": 0, "attachment_total_view_time_millis": 0, "attachment_view_completion": 0, "avg_screen_time_millis": 0, "avg_view_time_millis": 0, "impressions": 0, "ios_installs": 0, "quartile_1": 0, "quartile_2": 0, "quartile_3": 0, "screen_time_millis": 0, "swipe_up_percent": 0.0, "swipes": 0, "total_installs": 0, "video_views": 0, "video_views_time_based": 0, "video_views_15s": 0, "view_completion": 0, "view_time_millis": 0, "paid_impressions": 0, "earned_impressions": 0, "total_impressions": 0, "play_time_millis": 0, "shares": 0, "saves": 0, "native_leads": 0, "conversion_purchases": 0, "conversion_purchases_value": 0, "conversion_save": 0, "conversion_start_checkout": 0, "conversion_add_cart": 0, "conversion_view_content": 0, "conversion_add_billing": 0, "conversion_searches": 0, "conversion_level_completes": 0, "conversion_app_opens": 0, "conversion_page_views": 0, "conversion_subscribe": 0, "conversion_ad_click": 0, "conversion_ad_view": 0, "conversion_complete_tutorial": 0, "conversion_invite": 0, "conversion_login": 0, "conversion_share": 0, "conversion_reserve": 0, "conversion_achievement_unlocked": 0, "conversion_add_to_wishlist": 0, "conversion_spend_credits": 0, "conversion_rate": 0, "conversion_start_trial": 0, "conversion_list_view": 0, "custom_event_1": 0, "custom_event_2": 0, "custom_event_3": 0, "custom_event_4": 0, "custom_event_5": 0, "story_opens": 0, "story_completes": 0}, "emitted_at": 1718737704063} {"stream": "campaigns_stats_hourly", "data": {"start_time": "2024-05-14T21:00:00.000-07:00", "end_time": "2024-05-14T22:00:00.000-07:00", "id": "d180d36e-1212-479b-84a5-8b0663ceaf82", "type": "CAMPAIGN", "granularity": "HOUR", "android_installs": 0, "attachment_avg_view_time_millis": 0, "attachment_impressions": 0, "attachment_quartile_1": 0, "attachment_quartile_2": 0, "attachment_quartile_3": 0, "attachment_total_view_time_millis": 0, "attachment_view_completion": 0, "avg_screen_time_millis": 0, "avg_view_time_millis": 0, "impressions": 0, "ios_installs": 0, "quartile_1": 0, "quartile_2": 0, "quartile_3": 0, "screen_time_millis": 0, "swipe_up_percent": 0.0, "swipes": 0, "total_installs": 0, "video_views": 0, "video_views_time_based": 0, "video_views_15s": 0, "view_completion": 0, "view_time_millis": 0, "paid_impressions": 0, "earned_impressions": 0, "total_impressions": 0, "play_time_millis": 0, "shares": 0, "saves": 0, "native_leads": 0, "conversion_purchases": 0, "conversion_purchases_value": 0, "conversion_save": 0, "conversion_start_checkout": 0, "conversion_add_cart": 0, "conversion_view_content": 0, "conversion_add_billing": 0, "conversion_searches": 0, "conversion_level_completes": 0, "conversion_app_opens": 0, "conversion_page_views": 0, "conversion_subscribe": 0, "conversion_ad_click": 0, "conversion_ad_view": 0, "conversion_complete_tutorial": 0, "conversion_invite": 0, "conversion_login": 0, "conversion_share": 0, "conversion_reserve": 0, "conversion_achievement_unlocked": 0, "conversion_add_to_wishlist": 0, "conversion_spend_credits": 0, "conversion_rate": 0, "conversion_start_trial": 0, "conversion_list_view": 0, "custom_event_1": 0, "custom_event_2": 0, "custom_event_3": 0, "custom_event_4": 0, "custom_event_5": 0, "story_opens": 0, "story_completes": 0}, "emitted_at": 1718737739971} -{"stream": "campaigns_stats_hourly", "data": {"start_time": "2024-05-14T22:00:00.000-07:00", "end_time": "2024-05-14T23:00:00.000-07:00", "id": "d180d36e-1212-479b-84a5-8b0663ceaf82", "type": "CAMPAIGN", "granularity": "HOUR", "android_installs": 0, "attachment_avg_view_time_millis": 0, "attachment_impressions": 0, "attachment_quartile_1": 0, "attachment_quartile_2": 0, "attachment_quartile_3": 0, "attachment_total_view_time_millis": 0, "attachment_view_completion": 0, "avg_screen_time_millis": 0, "avg_view_time_millis": 0, "impressions": 0, "ios_installs": 0, "quartile_1": 0, "quartile_2": 0, "quartile_3": 0, "screen_time_millis": 0, "swipe_up_percent": 0.0, "swipes": 0, "total_installs": 0, "video_views": 0, "video_views_time_based": 0, "video_views_15s": 0, "view_completion": 0, "view_time_millis": 0, "paid_impressions": 0, "earned_impressions": 0, "total_impressions": 0, "play_time_millis": 0, "shares": 0, "saves": 0, "native_leads": 0, "conversion_purchases": 0, "conversion_purchases_value": 0, "conversion_save": 0, "conversion_start_checkout": 0, "conversion_add_cart": 0, "conversion_view_content": 0, "conversion_add_billing": 0, "conversion_searches": 0, "conversion_level_completes": 0, "conversion_app_opens": 0, "conversion_page_views": 0, "conversion_subscribe": 0, "conversion_ad_click": 0, "conversion_ad_view": 0, "conversion_complete_tutorial": 0, "conversion_invite": 0, "conversion_login": 0, "conversion_share": 0, "conversion_reserve": 0, "conversion_achievement_unlocked": 0, "conversion_add_to_wishlist": 0, "conversion_spend_credits": 0, "conversion_rate": 0, "conversion_start_trial": 0, "conversion_list_view": 0, "custom_event_1": 0, "custom_event_2": 0, "custom_event_3": 0, "custom_event_4": 0, "custom_event_5": 0, "story_opens": 0, "story_completes": 0}, "emitted_at": 1718737740086} \ No newline at end of file +{"stream": "campaigns_stats_hourly", "data": {"start_time": "2024-05-14T22:00:00.000-07:00", "end_time": "2024-05-14T23:00:00.000-07:00", "id": "d180d36e-1212-479b-84a5-8b0663ceaf82", "type": "CAMPAIGN", "granularity": "HOUR", "android_installs": 0, "attachment_avg_view_time_millis": 0, "attachment_impressions": 0, "attachment_quartile_1": 0, "attachment_quartile_2": 0, "attachment_quartile_3": 0, "attachment_total_view_time_millis": 0, "attachment_view_completion": 0, "avg_screen_time_millis": 0, "avg_view_time_millis": 0, "impressions": 0, "ios_installs": 0, "quartile_1": 0, "quartile_2": 0, "quartile_3": 0, "screen_time_millis": 0, "swipe_up_percent": 0.0, "swipes": 0, "total_installs": 0, "video_views": 0, "video_views_time_based": 0, "video_views_15s": 0, "view_completion": 0, "view_time_millis": 0, "paid_impressions": 0, "earned_impressions": 0, "total_impressions": 0, "play_time_millis": 0, "shares": 0, "saves": 0, "native_leads": 0, "conversion_purchases": 0, "conversion_purchases_value": 0, "conversion_save": 0, "conversion_start_checkout": 0, "conversion_add_cart": 0, "conversion_view_content": 0, "conversion_add_billing": 0, "conversion_searches": 0, "conversion_level_completes": 0, "conversion_app_opens": 0, "conversion_page_views": 0, "conversion_subscribe": 0, "conversion_ad_click": 0, "conversion_ad_view": 0, "conversion_complete_tutorial": 0, "conversion_invite": 0, "conversion_login": 0, "conversion_share": 0, "conversion_reserve": 0, "conversion_achievement_unlocked": 0, "conversion_add_to_wishlist": 0, "conversion_spend_credits": 0, "conversion_rate": 0, "conversion_start_trial": 0, "conversion_list_view": 0, "custom_event_1": 0, "custom_event_2": 0, "custom_event_3": 0, "custom_event_4": 0, "custom_event_5": 0, "story_opens": 0, "story_completes": 0}, "emitted_at": 1718737740086} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/spec.json similarity index 96% rename from airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json rename to airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/spec.json index 28389e0a3fd0..83d7237fe2b2 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/spec.json @@ -1,5 +1,4 @@ { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/snapchat-marketing", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Snapchat Marketing Spec", @@ -74,7 +73,10 @@ }, "advanced_auth": { "auth_flow_type": "oauth2.0", + "predicate_key": null, + "predicate_value": null, "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": null, "complete_oauth_output_specification": { "type": "object", "properties": { diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml index 4c1d73c5f725..3e454cb7064b 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: 200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b - dockerImageTag: 0.6.2 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-snapchat-marketing githubIssueLabel: source-snapchat-marketing icon: snapchat.svg @@ -25,10 +25,42 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 1.0.0: + message: >- + The source Snapchat Marketing connector is being migrated from the Python CDK to our declarative low-code CDK. + Due to changes to the incremental stream state, this migration constitutes a breaking change. Additionally, + added incremental functionality to organizations. After updating, please reset your source before resuming syncs. + For more information, see our migration documentation for source Snapchat Marketing. + upgradeDeadline: "2024-07-16" + scopedImpact: + - scopeType: stream + impactedScopes: + - "organizations" + - "adaccounts" + - "creatives" + - "ads" + - "adsquads" + - "segments" + - "media" + - "campaigns" + - "adaccounts_stats_hourly" + - "adaccounts_stats_daily" + - "adaccounts_stats_lifetime" + - "ads_stats_hourly" + - "ads_stats_daily" + - "ads_stats_lifetime" + - "adsquads_stats_hourly" + - "adsquads_stats_daily" + - "adsquads_stats_lifetime" + - "campaigns_stats_hourly" + - "campaigns_stats_daily" + - "campaigns_stats_lifetime" documentationUrl: https://docs.airbyte.com/integrations/sources/snapchat-marketing tags: - language:python - - cdk:python + - cdk:low-code ab_internal: sl: 200 ql: 400 diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/poetry.lock b/airbyte-integrations/connectors/source-snapchat-marketing/poetry.lock index 6c3ca47ed888..f8bb45150f90 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/poetry.lock +++ b/airbyte-integrations/connectors/source-snapchat-marketing/poetry.lock @@ -1,14 +1,14 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" -version = "0.90.0" +version = "1.8.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-0.90.0-py3-none-any.whl", hash = "sha256:bd0aa5843cdc4901f2e482f0e86695ca4e6db83b65c5017799255dd20535cf56"}, - {file = "airbyte_cdk-0.90.0.tar.gz", hash = "sha256:25cefc010718bada5cce3f87e7ae93068630732c0d34ce5145f8ddf7457d4d3c"}, + {file = "airbyte_cdk-1.8.0-py3-none-any.whl", hash = "sha256:ca23d7877005fe87ffc4a3a3de29ee55eed625d874eb59b49664b156f9ae9ee2"}, + {file = "airbyte_cdk-1.8.0.tar.gz", hash = "sha256:ac82fbfd6b650b7ed015900748e30fdd2a4c574caa54d1bcc03cb584a17f1533"}, ] [package.dependencies] @@ -17,7 +17,7 @@ backoff = "*" cachetools = "*" cryptography = ">=42.0.5,<43.0.0" Deprecated = ">=1.2,<1.3" -dpath = ">=2.0.1,<2.1.0" +dpath = ">=2.1.6,<3.0.0" genson = "1.2.2" isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" @@ -42,13 +42,13 @@ vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings [[package]] name = "airbyte-protocol-models" -version = "0.11.0" +version = "0.12.2" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.11.0-py3-none-any.whl", hash = "sha256:2157757c1af8c13e471ab6a0304fd2f9a2a6af8cc9173937be1348a9553f7c32"}, - {file = "airbyte_protocol_models-0.11.0.tar.gz", hash = "sha256:1c7e46251b0d5a292b4aa382df24f415ac2a2a2b4719361b3c0f76368a043c23"}, + {file = "airbyte_protocol_models-0.12.2-py3-none-any.whl", hash = "sha256:1780db5b26285865b858d26502933def8e11919c9436ccf7b8b9cb0170b07c2a"}, + {file = "airbyte_protocol_models-0.12.2.tar.gz", hash = "sha256:b7c4d9a7c32c0691601c2b9416af090a858e126666e2c8c880d7a1798eb519f0"}, ] [package.dependencies] @@ -143,13 +143,13 @@ ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.6.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] [[package]] @@ -328,43 +328,43 @@ files = [ [[package]] name = "cryptography" -version = "42.0.7" +version = "42.0.8" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"}, - {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"}, - {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"}, - {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"}, - {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"}, - {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"}, - {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, ] [package.dependencies] @@ -399,13 +399,13 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "dpath" -version = "2.0.8" +version = "2.2.0" description = "Filesystem-like pathing and searching for dictionaries" optional = false python-versions = ">=3.7" files = [ - {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, - {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, + {file = "dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576"}, + {file = "dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e"}, ] [[package]] @@ -501,13 +501,13 @@ jsonpointer = ">=1.9" [[package]] name = "jsonpointer" -version = "2.4" +version = "3.0.0" description = "Identify specific nodes in a JSON document (RFC 6901)" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +python-versions = ">=3.7" files = [ - {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, - {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, + {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"}, + {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"}, ] [[package]] @@ -566,13 +566,13 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langsmith" -version = "0.1.60" +version = "0.1.81" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.60-py3-none-any.whl", hash = "sha256:3c3520ea473de0a984237b3e9d638fdf23ef3acc5aec89a42e693225e72d6120"}, - {file = "langsmith-0.1.60.tar.gz", hash = "sha256:6a145b5454437f9e0f81525f23c4dcdbb8c07b1c91553b8f697456c418d6a599"}, + {file = "langsmith-0.1.81-py3-none-any.whl", hash = "sha256:3251d823225eef23ee541980b9d9e506367eabbb7f985a086b5d09e8f78ba7e9"}, + {file = "langsmith-0.1.81.tar.gz", hash = "sha256:585ef3a2251380bd2843a664c9a28da4a7d28432e3ee8bcebf291ffb8e1f0af0"}, ] [package.dependencies] @@ -651,57 +651,57 @@ files = [ [[package]] name = "orjson" -version = "3.10.3" +version = "3.10.5" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = false python-versions = ">=3.8" files = [ - {file = "orjson-3.10.3-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9fb6c3f9f5490a3eb4ddd46fc1b6eadb0d6fc16fb3f07320149c3286a1409dd8"}, - {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:252124b198662eee80428f1af8c63f7ff077c88723fe206a25df8dc57a57b1fa"}, - {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9f3e87733823089a338ef9bbf363ef4de45e5c599a9bf50a7a9b82e86d0228da"}, - {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8334c0d87103bb9fbbe59b78129f1f40d1d1e8355bbed2ca71853af15fa4ed3"}, - {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1952c03439e4dce23482ac846e7961f9d4ec62086eb98ae76d97bd41d72644d7"}, - {file = "orjson-3.10.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c0403ed9c706dcd2809f1600ed18f4aae50be263bd7112e54b50e2c2bc3ebd6d"}, - {file = "orjson-3.10.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:382e52aa4270a037d41f325e7d1dfa395b7de0c367800b6f337d8157367bf3a7"}, - {file = "orjson-3.10.3-cp310-none-win32.whl", hash = "sha256:be2aab54313752c04f2cbaab4515291ef5af8c2256ce22abc007f89f42f49109"}, - {file = "orjson-3.10.3-cp310-none-win_amd64.whl", hash = "sha256:416b195f78ae461601893f482287cee1e3059ec49b4f99479aedf22a20b1098b"}, - {file = "orjson-3.10.3-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:73100d9abbbe730331f2242c1fc0bcb46a3ea3b4ae3348847e5a141265479700"}, - {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:544a12eee96e3ab828dbfcb4d5a0023aa971b27143a1d35dc214c176fdfb29b3"}, - {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:520de5e2ef0b4ae546bea25129d6c7c74edb43fc6cf5213f511a927f2b28148b"}, - {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ccaa0a401fc02e8828a5bedfd80f8cd389d24f65e5ca3954d72c6582495b4bcf"}, - {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a7bc9e8bc11bac40f905640acd41cbeaa87209e7e1f57ade386da658092dc16"}, - {file = "orjson-3.10.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3582b34b70543a1ed6944aca75e219e1192661a63da4d039d088a09c67543b08"}, - {file = "orjson-3.10.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1c23dfa91481de880890d17aa7b91d586a4746a4c2aa9a145bebdbaf233768d5"}, - {file = "orjson-3.10.3-cp311-none-win32.whl", hash = "sha256:1770e2a0eae728b050705206d84eda8b074b65ee835e7f85c919f5705b006c9b"}, - {file = "orjson-3.10.3-cp311-none-win_amd64.whl", hash = "sha256:93433b3c1f852660eb5abdc1f4dd0ced2be031ba30900433223b28ee0140cde5"}, - {file = "orjson-3.10.3-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a39aa73e53bec8d410875683bfa3a8edf61e5a1c7bb4014f65f81d36467ea098"}, - {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0943a96b3fa09bee1afdfccc2cb236c9c64715afa375b2af296c73d91c23eab2"}, - {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e852baafceff8da3c9defae29414cc8513a1586ad93e45f27b89a639c68e8176"}, - {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18566beb5acd76f3769c1d1a7ec06cdb81edc4d55d2765fb677e3eaa10fa99e0"}, - {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bd2218d5a3aa43060efe649ec564ebedec8ce6ae0a43654b81376216d5ebd42"}, - {file = "orjson-3.10.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cf20465e74c6e17a104ecf01bf8cd3b7b252565b4ccee4548f18b012ff2f8069"}, - {file = "orjson-3.10.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba7f67aa7f983c4345eeda16054a4677289011a478ca947cd69c0a86ea45e534"}, - {file = "orjson-3.10.3-cp312-none-win32.whl", hash = "sha256:17e0713fc159abc261eea0f4feda611d32eabc35708b74bef6ad44f6c78d5ea0"}, - {file = "orjson-3.10.3-cp312-none-win_amd64.whl", hash = "sha256:4c895383b1ec42b017dd2c75ae8a5b862fc489006afde06f14afbdd0309b2af0"}, - {file = "orjson-3.10.3-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:be2719e5041e9fb76c8c2c06b9600fe8e8584e6980061ff88dcbc2691a16d20d"}, - {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0175a5798bdc878956099f5c54b9837cb62cfbf5d0b86ba6d77e43861bcec2"}, - {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:978be58a68ade24f1af7758626806e13cff7748a677faf95fbb298359aa1e20d"}, - {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16bda83b5c61586f6f788333d3cf3ed19015e3b9019188c56983b5a299210eb5"}, - {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ad1f26bea425041e0a1adad34630c4825a9e3adec49079b1fb6ac8d36f8b754"}, - {file = "orjson-3.10.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:9e253498bee561fe85d6325ba55ff2ff08fb5e7184cd6a4d7754133bd19c9195"}, - {file = "orjson-3.10.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0a62f9968bab8a676a164263e485f30a0b748255ee2f4ae49a0224be95f4532b"}, - {file = "orjson-3.10.3-cp38-none-win32.whl", hash = "sha256:8d0b84403d287d4bfa9bf7d1dc298d5c1c5d9f444f3737929a66f2fe4fb8f134"}, - {file = "orjson-3.10.3-cp38-none-win_amd64.whl", hash = "sha256:8bc7a4df90da5d535e18157220d7915780d07198b54f4de0110eca6b6c11e290"}, - {file = "orjson-3.10.3-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9059d15c30e675a58fdcd6f95465c1522b8426e092de9fff20edebfdc15e1cb0"}, - {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d40c7f7938c9c2b934b297412c067936d0b54e4b8ab916fd1a9eb8f54c02294"}, - {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4a654ec1de8fdaae1d80d55cee65893cb06494e124681ab335218be6a0691e7"}, - {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:831c6ef73f9aa53c5f40ae8f949ff7681b38eaddb6904aab89dca4d85099cb78"}, - {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99b880d7e34542db89f48d14ddecbd26f06838b12427d5a25d71baceb5ba119d"}, - {file = "orjson-3.10.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2e5e176c994ce4bd434d7aafb9ecc893c15f347d3d2bbd8e7ce0b63071c52e25"}, - {file = "orjson-3.10.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b69a58a37dab856491bf2d3bbf259775fdce262b727f96aafbda359cb1d114d8"}, - {file = "orjson-3.10.3-cp39-none-win32.whl", hash = "sha256:b8d4d1a6868cde356f1402c8faeb50d62cee765a1f7ffcfd6de732ab0581e063"}, - {file = "orjson-3.10.3-cp39-none-win_amd64.whl", hash = "sha256:5102f50c5fc46d94f2033fe00d392588564378260d64377aec702f21a7a22912"}, - {file = "orjson-3.10.3.tar.gz", hash = "sha256:2b166507acae7ba2f7c315dcf185a9111ad5e992ac81f2d507aac39193c2c818"}, + {file = "orjson-3.10.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b"}, + {file = "orjson-3.10.5-cp310-none-win32.whl", hash = "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2"}, + {file = "orjson-3.10.5-cp310-none-win_amd64.whl", hash = "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228"}, + {file = "orjson-3.10.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa"}, + {file = "orjson-3.10.5-cp311-none-win32.whl", hash = "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04"}, + {file = "orjson-3.10.5-cp311-none-win_amd64.whl", hash = "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c"}, + {file = "orjson-3.10.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3"}, + {file = "orjson-3.10.5-cp312-none-win32.whl", hash = "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2"}, + {file = "orjson-3.10.5-cp312-none-win_amd64.whl", hash = "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5"}, + {file = "orjson-3.10.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b"}, + {file = "orjson-3.10.5-cp38-none-win32.whl", hash = "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4"}, + {file = "orjson-3.10.5-cp38-none-win_amd64.whl", hash = "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09"}, + {file = "orjson-3.10.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86"}, + {file = "orjson-3.10.5-cp39-none-win32.whl", hash = "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47"}, + {file = "orjson-3.10.5-cp39-none-win_amd64.whl", hash = "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7"}, + {file = "orjson-3.10.5.tar.gz", hash = "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d"}, ] [[package]] @@ -804,47 +804,47 @@ files = [ [[package]] name = "pydantic" -version = "1.10.15" +version = "1.10.16" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:22ed12ee588b1df028a2aa5d66f07bf8f8b4c8579c2e96d5a9c1f96b77f3bb55"}, - {file = "pydantic-1.10.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75279d3cac98186b6ebc2597b06bcbc7244744f6b0b44a23e4ef01e5683cc0d2"}, - {file = "pydantic-1.10.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50f1666a9940d3d68683c9d96e39640f709d7a72ff8702987dab1761036206bb"}, - {file = "pydantic-1.10.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82790d4753ee5d00739d6cb5cf56bceb186d9d6ce134aca3ba7befb1eedbc2c8"}, - {file = "pydantic-1.10.15-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d207d5b87f6cbefbdb1198154292faee8017d7495a54ae58db06762004500d00"}, - {file = "pydantic-1.10.15-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e49db944fad339b2ccb80128ffd3f8af076f9f287197a480bf1e4ca053a866f0"}, - {file = "pydantic-1.10.15-cp310-cp310-win_amd64.whl", hash = "sha256:d3b5c4cbd0c9cb61bbbb19ce335e1f8ab87a811f6d589ed52b0254cf585d709c"}, - {file = "pydantic-1.10.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c3d5731a120752248844676bf92f25a12f6e45425e63ce22e0849297a093b5b0"}, - {file = "pydantic-1.10.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c365ad9c394f9eeffcb30a82f4246c0006417f03a7c0f8315d6211f25f7cb654"}, - {file = "pydantic-1.10.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3287e1614393119c67bd4404f46e33ae3be3ed4cd10360b48d0a4459f420c6a3"}, - {file = "pydantic-1.10.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be51dd2c8596b25fe43c0a4a59c2bee4f18d88efb8031188f9e7ddc6b469cf44"}, - {file = "pydantic-1.10.15-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6a51a1dd4aa7b3f1317f65493a182d3cff708385327c1c82c81e4a9d6d65b2e4"}, - {file = "pydantic-1.10.15-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4e316e54b5775d1eb59187f9290aeb38acf620e10f7fd2f776d97bb788199e53"}, - {file = "pydantic-1.10.15-cp311-cp311-win_amd64.whl", hash = "sha256:0d142fa1b8f2f0ae11ddd5e3e317dcac060b951d605fda26ca9b234b92214986"}, - {file = "pydantic-1.10.15-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7ea210336b891f5ea334f8fc9f8f862b87acd5d4a0cbc9e3e208e7aa1775dabf"}, - {file = "pydantic-1.10.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3453685ccd7140715e05f2193d64030101eaad26076fad4e246c1cc97e1bb30d"}, - {file = "pydantic-1.10.15-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bea1f03b8d4e8e86702c918ccfd5d947ac268f0f0cc6ed71782e4b09353b26f"}, - {file = "pydantic-1.10.15-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:005655cabc29081de8243126e036f2065bd7ea5b9dff95fde6d2c642d39755de"}, - {file = "pydantic-1.10.15-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:af9850d98fc21e5bc24ea9e35dd80a29faf6462c608728a110c0a30b595e58b7"}, - {file = "pydantic-1.10.15-cp37-cp37m-win_amd64.whl", hash = "sha256:d31ee5b14a82c9afe2bd26aaa405293d4237d0591527d9129ce36e58f19f95c1"}, - {file = "pydantic-1.10.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5e09c19df304b8123938dc3c53d3d3be6ec74b9d7d0d80f4f4b5432ae16c2022"}, - {file = "pydantic-1.10.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7ac9237cd62947db00a0d16acf2f3e00d1ae9d3bd602b9c415f93e7a9fc10528"}, - {file = "pydantic-1.10.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:584f2d4c98ffec420e02305cf675857bae03c9d617fcfdc34946b1160213a948"}, - {file = "pydantic-1.10.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbc6989fad0c030bd70a0b6f626f98a862224bc2b1e36bfc531ea2facc0a340c"}, - {file = "pydantic-1.10.15-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d573082c6ef99336f2cb5b667b781d2f776d4af311574fb53d908517ba523c22"}, - {file = "pydantic-1.10.15-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6bd7030c9abc80134087d8b6e7aa957e43d35714daa116aced57269a445b8f7b"}, - {file = "pydantic-1.10.15-cp38-cp38-win_amd64.whl", hash = "sha256:3350f527bb04138f8aff932dc828f154847fbdc7a1a44c240fbfff1b57f49a12"}, - {file = "pydantic-1.10.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:51d405b42f1b86703555797270e4970a9f9bd7953f3990142e69d1037f9d9e51"}, - {file = "pydantic-1.10.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a980a77c52723b0dc56640ced396b73a024d4b74f02bcb2d21dbbac1debbe9d0"}, - {file = "pydantic-1.10.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67f1a1fb467d3f49e1708a3f632b11c69fccb4e748a325d5a491ddc7b5d22383"}, - {file = "pydantic-1.10.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:676ed48f2c5bbad835f1a8ed8a6d44c1cd5a21121116d2ac40bd1cd3619746ed"}, - {file = "pydantic-1.10.15-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:92229f73400b80c13afcd050687f4d7e88de9234d74b27e6728aa689abcf58cc"}, - {file = "pydantic-1.10.15-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2746189100c646682eff0bce95efa7d2e203420d8e1c613dc0c6b4c1d9c1fde4"}, - {file = "pydantic-1.10.15-cp39-cp39-win_amd64.whl", hash = "sha256:394f08750bd8eaad714718812e7fab615f873b3cdd0b9d84e76e51ef3b50b6b7"}, - {file = "pydantic-1.10.15-py3-none-any.whl", hash = "sha256:28e552a060ba2740d0d2aabe35162652c1459a0b9069fe0db7f4ee0e18e74d58"}, - {file = "pydantic-1.10.15.tar.gz", hash = "sha256:ca832e124eda231a60a041da4f013e3ff24949d94a01154b137fc2f2a43c3ffb"}, + {file = "pydantic-1.10.16-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1a539ac40551b01a85e899829aa43ca8036707474af8d74b48be288d4d2d2846"}, + {file = "pydantic-1.10.16-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a4fcc7b0b8038dbda2dda642cff024032dfae24a7960cc58e57a39eb1949b9b"}, + {file = "pydantic-1.10.16-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4660dd697de1ae2d4305a85161312611f64d5360663a9ba026cd6ad9e3fe14c3"}, + {file = "pydantic-1.10.16-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:900a787c574f903a97d0bf52a43ff3b6cf4fa0119674bcfc0e5fd1056d388ad9"}, + {file = "pydantic-1.10.16-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d30192a63e6d3334c3f0c0506dd6ae9f1dce7b2f8845518915291393a5707a22"}, + {file = "pydantic-1.10.16-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:16cf23ed599ca5ca937e37ba50ab114e6b5c387eb43a6cc533701605ad1be611"}, + {file = "pydantic-1.10.16-cp310-cp310-win_amd64.whl", hash = "sha256:8d23111f41d1e19334edd51438fd57933f3eee7d9d2fa8cc3f5eda515a272055"}, + {file = "pydantic-1.10.16-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef287b8d7fc0e86a8bd1f902c61aff6ba9479c50563242fe88ba39692e98e1e0"}, + {file = "pydantic-1.10.16-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b9ded699bfd3b3912d796ff388b0c607e6d35d41053d37aaf8fd6082c660de9a"}, + {file = "pydantic-1.10.16-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:daeb199814333e4426c5e86d7fb610f4e230289f28cab90eb4de27330bef93cf"}, + {file = "pydantic-1.10.16-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5973843f1fa99ec6c3ac8d1a8698ac9340b35e45cca6c3e5beb5c3bd1ef15de6"}, + {file = "pydantic-1.10.16-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6b8a7788a8528a558828fe4a48783cafdcf2612d13c491594a8161dc721629c"}, + {file = "pydantic-1.10.16-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8abaecf54dacc9d991dda93c3b880d41092a8924cde94eeb811d7d9ab55df7d8"}, + {file = "pydantic-1.10.16-cp311-cp311-win_amd64.whl", hash = "sha256:ddc7b682fbd23f051edc419dc6977e11dd2dbdd0cef9d05f0e15d1387862d230"}, + {file = "pydantic-1.10.16-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:067c2b5539f7839653ad8c3d1fc2f1343338da8677b7b2172abf3cd3fdc8f719"}, + {file = "pydantic-1.10.16-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d1fc943583c046ecad0ff5d6281ee571b64e11b5503d9595febdce54f38b290"}, + {file = "pydantic-1.10.16-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18548b30ccebe71d380b0886cc44ea5d80afbcc155e3518792f13677ad06097d"}, + {file = "pydantic-1.10.16-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4e92292f9580fc5ea517618580fac24e9f6dc5657196e977c194a8e50e14f5a9"}, + {file = "pydantic-1.10.16-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5da8bc4bb4f85b8c97cc7f11141fddbbd29eb25e843672e5807e19cc3d7c1b7f"}, + {file = "pydantic-1.10.16-cp37-cp37m-win_amd64.whl", hash = "sha256:a04ee1ea34172b87707a6ecfcdb120d7656892206b7c4dbdb771a73e90179fcb"}, + {file = "pydantic-1.10.16-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4fa86469fd46e732242c7acb83282d33f83591a7e06f840481327d5bf6d96112"}, + {file = "pydantic-1.10.16-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:89c2783dc261726fe7a5ce1121bce29a2f7eb9b1e704c68df2b117604e3b346f"}, + {file = "pydantic-1.10.16-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78e59fa919fa7a192f423d190d8660c35dd444efa9216662273f36826765424b"}, + {file = "pydantic-1.10.16-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7e82a80068c77f4b074032e031e642530b6d45cb8121fc7c99faa31fb6c6b72"}, + {file = "pydantic-1.10.16-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d82d5956cee27a30e26a5b88d00a6a2a15a4855e13c9baf50175976de0dc282c"}, + {file = "pydantic-1.10.16-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b7b99424cc0970ff08deccb549b5a6ec1040c0b449eab91723e64df2bd8fdca"}, + {file = "pydantic-1.10.16-cp38-cp38-win_amd64.whl", hash = "sha256:d97a35e1ba59442775201657171f601a2879e63517a55862a51f8d67cdfc0017"}, + {file = "pydantic-1.10.16-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9d91f6866fd3e303c632207813ef6bc4d86055e21c5e5a0a311983a9ac5f0192"}, + {file = "pydantic-1.10.16-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8d3c71d14c8bd26d2350c081908dbf59d5a6a8f9596d9ef2b09cc1e61c8662b"}, + {file = "pydantic-1.10.16-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b73e6386b439b4881d79244e9fc1e32d1e31e8d784673f5d58a000550c94a6c0"}, + {file = "pydantic-1.10.16-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f039881fb2ef86f6de6eacce6e71701b47500355738367413ccc1550b2a69cf"}, + {file = "pydantic-1.10.16-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3895ddb26f22bdddee7e49741486aa7b389258c6f6771943e87fc00eabd79134"}, + {file = "pydantic-1.10.16-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:55b945da2756b5cef93d792521ad0d457fdf2f69fd5a2d10a27513f5281717dd"}, + {file = "pydantic-1.10.16-cp39-cp39-win_amd64.whl", hash = "sha256:22dd265c77c3976a34be78409b128cb84629284dfd1b69d2fa1507a36f84dc8b"}, + {file = "pydantic-1.10.16-py3-none-any.whl", hash = "sha256:aa2774ba5412fd1c5cb890d08e8b0a3bb5765898913ba1f61a65a4810f03cf29"}, + {file = "pydantic-1.10.16.tar.gz", hash = "sha256:8bb388f6244809af69ee384900b10b677a69f1980fdc655ea419710cffcb5610"}, ] [package.dependencies] @@ -1066,13 +1066,13 @@ files = [ [[package]] name = "requests" -version = "2.32.2" +version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" files = [ - {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, - {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -1087,13 +1087,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-cache" -version = "1.2.0" +version = "1.2.1" description = "A persistent cache for python requests" optional = false python-versions = ">=3.8" files = [ - {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, - {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, + {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"}, + {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"}, ] [package.dependencies] @@ -1134,18 +1134,18 @@ fixture = ["fixtures"] [[package]] name = "setuptools" -version = "70.0.0" +version = "70.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, - {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, + {file = "setuptools-70.1.0-py3-none-any.whl", hash = "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267"}, + {file = "setuptools-70.1.0.tar.gz", hash = "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -1160,13 +1160,13 @@ files = [ [[package]] name = "tenacity" -version = "8.3.0" +version = "8.4.1" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" files = [ - {file = "tenacity-8.3.0-py3-none-any.whl", hash = "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185"}, - {file = "tenacity-8.3.0.tar.gz", hash = "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2"}, + {file = "tenacity-8.4.1-py3-none-any.whl", hash = "sha256:28522e692eda3e1b8f5e99c51464efcc0b9fc86933da92415168bc1c4e2308fa"}, + {file = "tenacity-8.4.1.tar.gz", hash = "sha256:54b1412b878ddf7e1f1577cd49527bad8cdef32421bd599beac0c6c3f10582fd"}, ] [package.extras] @@ -1186,13 +1186,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.11.0" +version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, - {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [[package]] @@ -1211,13 +1211,13 @@ six = "*" [[package]] name = "urllib3" -version = "2.2.1" +version = "2.2.2" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, - {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, ] [package.extras] @@ -1322,4 +1322,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "64a5d2e5e85ad74cbb53a4395ac422f44e923cbd418171327c4dc49f47167f74" +content-hash = "912963301c039ac2f732e49803dddb895253a3f531d279fb210b45c6ea2e435a" diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/pyproject.toml b/airbyte-integrations/connectors/source-snapchat-marketing/pyproject.toml index 5cfd7df9adaf..12366701d053 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/pyproject.toml +++ b/airbyte-integrations/connectors/source-snapchat-marketing/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.6.2" +version = "1.0.0" name = "source-snapchat-marketing" description = "Source implementation for Snapchat Marketing." authors = [ "Airbyte ",] @@ -17,7 +17,7 @@ include = "source_snapchat_marketing" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "0.90.0" +airbyte-cdk = "^1" [tool.poetry.scripts] source-snapchat-marketing = "source_snapchat_marketing.run:run" diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/manifest.yaml b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/manifest.yaml new file mode 100644 index 000000000000..9f84b77dcb55 --- /dev/null +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/manifest.yaml @@ -0,0 +1,4102 @@ +version: 1.4.0 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - organizations + +definitions: + streams: + organizations: + type: DeclarativeStream + name: organizations + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: me/organizations + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - organizations + - "*" + - organization + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + - "%Y-%m-%d" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + is_client_side_incremental: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/organizations" + adaccounts: + type: DeclarativeStream + name: adaccounts + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: organizations/{{ stream_slice["organization_id"] }}/adaccounts + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - adaccounts + - "*" + - adaccount + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: organization_id + stream: + $ref: "#/definitions/streams/organizations" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + is_client_side_incremental: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/adaccounts" + creatives: + type: DeclarativeStream + name: creatives + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['adaccount_id'] }}/creatives + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - creatives + - "*" + - creative + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: adaccount_id + stream: + $ref: "#/definitions/streams/adaccounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + is_client_side_incremental: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/creatives" + ads: + type: DeclarativeStream + name: ads + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['adaccount_id'] }}/ads + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - ads + - "*" + - ad + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: adaccount_id + stream: + $ref: "#/definitions/streams/adaccounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + is_client_side_incremental: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/ads" + adsquads: + type: DeclarativeStream + name: adsquads + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['adaccount_id'] }}/adsquads + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - adsquads + - "*" + - adsquad + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: adaccount_id + stream: + $ref: "#/definitions/streams/adaccounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + is_client_side_incremental: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/adsquads" + segments: + type: DeclarativeStream + name: segments + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['adaccount_id'] }}/segments + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - segments + - "*" + - segment + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: adaccount_id + stream: + $ref: "#/definitions/streams/adaccounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + is_client_side_incremental: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/segments" + media: + type: DeclarativeStream + name: media + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['adaccount_id'] }}/media + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - media + - "*" + - media + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: adaccount_id + stream: + $ref: "#/definitions/streams/adaccounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + is_client_side_incremental: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/media" + campaigns: + type: DeclarativeStream + name: campaigns + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['adaccount_id'] }}/campaigns + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - campaigns + - "*" + - campaign + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: adaccount_id + stream: + $ref: "#/definitions/streams/adaccounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + is_client_side_incremental: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/campaigns" + adaccounts_stats_hourly: + type: DeclarativeStream + name: adaccounts_stats_hourly + primary_key: + - id + - granularity + - start_time + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: HOUR + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: spend + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - timeseries_stats + - "*" + - timeseries_stat + - timeseries + - "*" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/adaccounts" + incremental_sync: + type: DatetimeBasedCursor + is_compare_strictly: true + cursor_field: start_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: end_time + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + step: P1W + cursor_granularity: PT0S + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ stream_slice['id'] }}" + - path: ["type"] + value: "AD_ACCOUNT" + - path: ["granularity"] + value: "HOUR" + - path: ["spend"] + value: "{{ record.get('stats', {}).get('spend') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + adaccounts_stats_daily: + type: DeclarativeStream + name: adaccounts_stats_daily + primary_key: + - id + - granularity + - start_time + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: DAY + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: spend + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - timeseries_stats + - "*" + - timeseries_stat + - timeseries + - "*" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/adaccounts" + incremental_sync: + type: DatetimeBasedCursor + is_compare_strictly: true + cursor_field: start_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: end_time + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + step: P1M + cursor_granularity: PT0S + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ stream_slice['id'] }}" + - path: ["type"] + value: "AD_ACCOUNT" + - path: ["granularity"] + value: "DAY" + - path: ["spend"] + value: "{{ record.get('stats', {}).get('spend') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + adaccounts_stats_lifetime: + type: DeclarativeStream + name: adaccounts_stats_lifetime + primary_key: + - id + - granularity + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adaccounts/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: LIFETIME + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: spend + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - lifetime_stats + - "*" + - lifetime_stat + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/adaccounts" + transformations: + - type: AddFields + fields: + - path: ["spend"] + value: "{{ record.get('stats', {}).get('spend') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + ads_stats_hourly: + type: DeclarativeStream + name: ads_stats_hourly + primary_key: + - id + - granularity + - start_time + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: ads/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: HOUR + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - timeseries_stats + - "*" + - timeseries_stat + - timeseries + - "*" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/ads" + incremental_sync: + type: DatetimeBasedCursor + is_compare_strictly: true + cursor_field: start_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: end_time + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + step: P1W + cursor_granularity: PT0S + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ stream_slice['id'] }}" + - path: ["type"] + value: "AD" + - path: ["granularity"] + value: "HOUR" + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + ads_stats_daily: + type: DeclarativeStream + name: ads_stats_daily + primary_key: + - id + - granularity + - start_time + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: ads/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: DAY + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes,attachment_frequency,attachment_uniques,frequency,uniques,total_reach,earned_reach + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - timeseries_stats + - "*" + - timeseries_stat + - timeseries + - "*" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/ads" + incremental_sync: + type: DatetimeBasedCursor + is_compare_strictly: true + cursor_field: start_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: end_time + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + step: P1M + cursor_granularity: PT0S + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ stream_slice['id'] }}" + - path: ["type"] + value: "AD" + - path: ["granularity"] + value: "DAY" + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - path: ["attachment_frequency"] + value: "{{ record.get('stats', {}).get('attachment_frequency') }}" + - path: ["attachment_uniques"] + value: "{{ record.get('stats', {}).get('attachment_uniques') }}" + - path: ["frequency"] + value: "{{ record.get('stats', {}).get('frequency') }}" + - path: ["uniques"] + value: "{{ record.get('stats', {}).get('uniques') }}" + - path: ["total_reach"] + value: "{{ record.get('stats', {}).get('total_reach') }}" + - path: ["earned_reach"] + value: "{{ record.get('stats', {}).get('earned_reach') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + ads_stats_lifetime: + type: DeclarativeStream + name: ads_stats_lifetime + primary_key: + - id + - granularity + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: ads/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: LIFETIME + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes,attachment_frequency,attachment_uniques,frequency,uniques,total_reach,earned_reach + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - lifetime_stats + - "*" + - lifetime_stat + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/ads" + transformations: + - type: AddFields + fields: + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - path: ["attachment_frequency"] + value: "{{ record.get('stats', {}).get('attachment_frequency') }}" + - path: ["attachment_uniques"] + value: "{{ record.get('stats', {}).get('attachment_uniques') }}" + - path: ["frequency"] + value: "{{ record.get('stats', {}).get('frequency') }}" + - path: ["uniques"] + value: "{{ record.get('stats', {}).get('uniques') }}" + - path: ["total_reach"] + value: "{{ record.get('stats', {}).get('total_reach') }}" + - path: ["earned_reach"] + value: "{{ record.get('stats', {}).get('earned_reach') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + adsquads_stats_hourly: + type: DeclarativeStream + name: adsquads_stats_hourly + primary_key: + - id + - granularity + - start_time + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adsquads/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: HOUR + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - timeseries_stats + - "*" + - timeseries_stat + - timeseries + - "*" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/adsquads" + incremental_sync: + type: DatetimeBasedCursor + is_compare_strictly: true + cursor_field: start_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: end_time + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + step: P1W + cursor_granularity: PT0S + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ stream_slice['id'] }}" + - path: ["type"] + value: "AD_SQUAD" + - path: ["granularity"] + value: "HOUR" + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + adsquads_stats_daily: + type: DeclarativeStream + name: adsquads_stats_daily + primary_key: + - id + - granularity + - start_time + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adsquads/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: DAY + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes,attachment_frequency,attachment_uniques,frequency,uniques,total_reach,earned_reach + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - timeseries_stats + - "*" + - timeseries_stat + - timeseries + - "*" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/adsquads" + incremental_sync: + type: DatetimeBasedCursor + is_compare_strictly: true + cursor_field: start_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: end_time + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + step: P1M + cursor_granularity: PT0S + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ stream_slice['id'] }}" + - path: ["type"] + value: "AD_SQUAD" + - path: ["granularity"] + value: "DAY" + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - path: ["attachment_frequency"] + value: "{{ record.get('stats', {}).get('attachment_frequency') }}" + - path: ["attachment_uniques"] + value: "{{ record.get('stats', {}).get('attachment_uniques') }}" + - path: ["frequency"] + value: "{{ record.get('stats', {}).get('frequency') }}" + - path: ["uniques"] + value: "{{ record.get('stats', {}).get('uniques') }}" + - path: ["total_reach"] + value: "{{ record.get('stats', {}).get('total_reach') }}" + - path: ["earned_reach"] + value: "{{ record.get('stats', {}).get('earned_reach') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + adsquads_stats_lifetime: + type: DeclarativeStream + name: adsquads_stats_lifetime + primary_key: + - id + - granularity + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: adsquads/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: LIFETIME + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes,attachment_frequency,attachment_uniques,frequency,uniques,total_reach,earned_reach + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - lifetime_stats + - "*" + - lifetime_stat + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/adsquads" + transformations: + - type: AddFields + fields: + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - path: ["attachment_frequency"] + value: "{{ record.get('stats', {}).get('attachment_frequency') }}" + - path: ["attachment_uniques"] + value: "{{ record.get('stats', {}).get('attachment_uniques') }}" + - path: ["frequency"] + value: "{{ record.get('stats', {}).get('frequency') }}" + - path: ["uniques"] + value: "{{ record.get('stats', {}).get('uniques') }}" + - path: ["total_reach"] + value: "{{ record.get('stats', {}).get('total_reach') }}" + - path: ["earned_reach"] + value: "{{ record.get('stats', {}).get('earned_reach') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + campaigns_stats_hourly: + type: DeclarativeStream + name: campaigns_stats_hourly + primary_key: + - id + - granularity + - start_time + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: campaigns/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: HOUR + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - timeseries_stats + - "*" + - timeseries_stat + - timeseries + - "*" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/campaigns" + incremental_sync: + type: DatetimeBasedCursor + is_compare_strictly: true + cursor_field: start_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: end_time + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + step: P1W + cursor_granularity: PT0S + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ stream_slice['id'] }}" + - path: ["type"] + value: "CAMPAIGN" + - path: ["granularity"] + value: "HOUR" + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + campaigns_stats_daily: + type: DeclarativeStream + name: campaigns_stats_daily + primary_key: + - id + - granularity + - start_time + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: campaigns/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: DAY + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes,attachment_frequency,attachment_uniques,frequency,uniques,total_reach,earned_reach + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - timeseries_stats + - "*" + - timeseries_stat + - timeseries + - "*" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/campaigns" + incremental_sync: + type: DatetimeBasedCursor + is_compare_strictly: true + cursor_field: start_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config.get("start_date", "2011-09-01") }}' + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: end_time + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', now_utc().strftime('%Y-%m-%d')) }}" + datetime_format: "%Y-%m-%d" + step: P1M + cursor_granularity: PT0S + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ stream_slice['id'] }}" + - path: ["type"] + value: "CAMPAIGN" + - path: ["granularity"] + value: "DAY" + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - path: ["attachment_frequency"] + value: "{{ record.get('stats', {}).get('attachment_frequency') }}" + - path: ["attachment_uniques"] + value: "{{ record.get('stats', {}).get('attachment_uniques') }}" + - path: ["frequency"] + value: "{{ record.get('stats', {}).get('frequency') }}" + - path: ["uniques"] + value: "{{ record.get('stats', {}).get('uniques') }}" + - path: ["total_reach"] + value: "{{ record.get('stats', {}).get('total_reach') }}" + - path: ["earned_reach"] + value: "{{ record.get('stats', {}).get('earned_reach') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + campaigns_stats_lifetime: + type: DeclarativeStream + name: campaigns_stats_lifetime + primary_key: + - id + - granularity + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: campaigns/{{ stream_slice['id'] }}/stats + http_method: GET + request_parameters: + granularity: LIFETIME + action_report_time: "{{ config['action_report_time'] }}" + swipe_up_attribution_window: '{{ config["swipe_up_attribution_window"] }}' + view_attribution_window: '{{ config["view_attribution_window"] }}' + fields: >- + android_installs,attachment_avg_view_time_millis,attachment_impressions,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,impressions,ios_installs,quartile_1,quartile_2,quartile_3,screen_time_millis,spend,swipe_up_percent,swipes,total_installs,video_views,video_views_time_based,video_views_15s,view_completion,view_time_millis,paid_impressions,earned_impressions,total_impressions,play_time_millis,shares,saves,native_leads,conversion_purchases,conversion_purchases_value,conversion_save,conversion_start_checkout,conversion_add_cart,conversion_view_content,conversion_add_billing,conversion_searches,conversion_level_completes,conversion_app_opens,conversion_page_views,conversion_subscribe,conversion_ad_click,conversion_ad_view,conversion_complete_tutorial,conversion_invite,conversion_login,conversion_share,conversion_reserve,conversion_achievement_unlocked,conversion_add_to_wishlist,conversion_spend_credits,conversion_rate,conversion_start_trial,conversion_list_view,custom_event_1,custom_event_2,custom_event_3,custom_event_4,custom_event_5,story_opens,story_completes,attachment_frequency,attachment_uniques,frequency,uniques,total_reach,earned_reach + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - lifetime_stats + - "*" + - lifetime_stat + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next_link", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next_link", {}) }}' + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/streams/campaigns" + transformations: + - type: AddFields + fields: + - path: ["android_installs"] + value: "{{ record.get('stats', {}).get('android_installs') }}" + - path: ["attachment_avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_avg_view_time_millis') }}" + - path: ["attachment_impressions"] + value: "{{ record.get('stats', {}).get('attachment_impressions') }}" + - path: ["attachment_quartile_1"] + value: "{{ record.get('stats', {}).get('attachment_quartile_1') }}" + - path: ["attachment_quartile_2"] + value: "{{ record.get('stats', {}).get('attachment_quartile_2') }}" + - path: ["attachment_quartile_3"] + value: "{{ record.get('stats', {}).get('attachment_quartile_3') }}" + - path: ["attachment_total_view_time_millis"] + value: "{{ record.get('stats', {}).get('attachment_total_view_time_millis') }}" + - path: ["attachment_view_completion"] + value: "{{ record.get('stats', {}).get('attachment_view_completion') }}" + - path: ["avg_screen_time_millis"] + value: "{{ record.get('stats', {}).get('avg_screen_time_millis') }}" + - path: ["avg_view_time_millis"] + value: "{{ record.get('stats', {}).get('avg_view_time_millis') }}" + - path: ["impressions"] + value: "{{ record.get('stats', {}).get('impressions') }}" + - path: ["ios_installs"] + value: "{{ record.get('stats', {}).get('ios_installs') }}" + - path: ["quartile_1"] + value: "{{ record.get('stats', {}).get('quartile_1') }}" + - path: ["quartile_2"] + value: "{{ record.get('stats', {}).get('quartile_2') }}" + - path: ["quartile_3"] + value: "{{ record.get('stats', {}).get('quartile_3') }}" + - path: ["screen_time_millis"] + value: "{{ record.get('stats', {}).get('screen_time_millis') }}" + - path: ["swipe_up_percent"] + value: "{{ record.get('stats', {}).get('swipe_up_percent') }}" + - path: ["swipes"] + value: "{{ record.get('stats', {}).get('swipes') }}" + - path: ["total_installs"] + value: "{{ record.get('stats', {}).get('total_installs') }}" + - path: ["video_views"] + value: "{{ record.get('stats', {}).get('video_views') }}" + - path: ["video_views_time_based"] + value: "{{ record.get('stats', {}).get('video_views_time_based') }}" + - path: ["video_views_15s"] + value: "{{ record.get('stats', {}).get('video_views_15s') }}" + - path: ["view_completion"] + value: "{{ record.get('stats', {}).get('view_completion') }}" + - path: ["view_time_millis"] + value: "{{ record.get('stats', {}).get('view_time_millis') }}" + - path: ["paid_impressions"] + value: "{{ record.get('stats', {}).get('paid_impressions') }}" + - path: ["earned_impressions"] + value: "{{ record.get('stats', {}).get('earned_impressions') }}" + - path: ["total_impressions"] + value: "{{ record.get('stats', {}).get('total_impressions') }}" + - path: ["play_time_millis"] + value: "{{ record.get('stats', {}).get('play_time_millis') }}" + - path: ["shares"] + value: "{{ record.get('stats', {}).get('shares') }}" + - path: ["saves"] + value: "{{ record.get('stats', {}).get('saves') }}" + - path: ["native_leads"] + value: "{{ record.get('stats', {}).get('native_leads') }}" + - path: ["conversion_purchases"] + value: "{{ record.get('stats', {}).get('conversion_purchases') }}" + - path: ["conversion_purchases_value"] + value: "{{ record.get('stats', {}).get('conversion_purchases_value') }}" + - path: ["conversion_save"] + value: "{{ record.get('stats', {}).get('conversion_save') }}" + - path: ["conversion_start_checkout"] + value: "{{ record.get('stats', {}).get('conversion_start_checkout') }}" + - path: ["conversion_add_cart"] + value: "{{ record.get('stats', {}).get('conversion_add_cart') }}" + - path: ["conversion_view_content"] + value: "{{ record.get('stats', {}).get('conversion_view_content') }}" + - path: ["conversion_add_billing"] + value: "{{ record.get('stats', {}).get('conversion_add_billing') }}" + - path: ["conversion_searches"] + value: "{{ record.get('stats', {}).get('conversion_searches') }}" + - path: ["conversion_level_completes"] + value: "{{ record.get('stats', {}).get('conversion_level_completes') }}" + - path: ["conversion_app_opens"] + value: "{{ record.get('stats', {}).get('conversion_app_opens') }}" + - path: ["conversion_page_views"] + value: "{{ record.get('stats', {}).get('conversion_page_views') }}" + - path: ["conversion_subscribe"] + value: "{{ record.get('stats', {}).get('conversion_subscribe') }}" + - path: ["conversion_ad_click"] + value: "{{ record.get('stats', {}).get('conversion_ad_click') }}" + - path: ["conversion_ad_view"] + value: "{{ record.get('stats', {}).get('conversion_ad_view') }}" + - path: ["conversion_complete_tutorial"] + value: "{{ record.get('stats', {}).get('conversion_complete_tutorial') }}" + - path: ["conversion_invite"] + value: "{{ record.get('stats', {}).get('conversion_invite') }}" + - path: ["conversion_login"] + value: "{{ record.get('stats', {}).get('conversion_login') }}" + - path: ["conversion_share"] + value: "{{ record.get('stats', {}).get('conversion_share') }}" + - path: ["conversion_reserve"] + value: "{{ record.get('stats', {}).get('conversion_reserve') }}" + - path: ["conversion_achievement_unlocked"] + value: "{{ record.get('stats', {}).get('conversion_achievement_unlocked') }}" + - path: ["conversion_add_to_wishlist"] + value: "{{ record.get('stats', {}).get('conversion_add_to_wishlist') }}" + - path: ["conversion_spend_credits"] + value: "{{ record.get('stats', {}).get('conversion_spend_credits') }}" + - path: ["conversion_rate"] + value: "{{ record.get('stats', {}).get('conversion_rate') }}" + - path: ["conversion_start_trial"] + value: "{{ record.get('stats', {}).get('conversion_start_trial') }}" + - path: ["conversion_list_view"] + value: "{{ record.get('stats', {}).get('conversion_list_view') }}" + - path: ["custom_event_1"] + value: "{{ record.get('stats', {}).get('custom_event_1') }}" + - path: ["custom_event_2"] + value: "{{ record.get('stats', {}).get('custom_event_2') }}" + - path: ["custom_event_3"] + value: "{{ record.get('stats', {}).get('custom_event_3') }}" + - path: ["custom_event_4"] + value: "{{ record.get('stats', {}).get('custom_event_4') }}" + - path: ["custom_event_5"] + value: "{{ record.get('stats', {}).get('custom_event_5') }}" + - path: ["story_opens"] + value: "{{ record.get('stats', {}).get('story_opens') }}" + - path: ["story_completes"] + value: "{{ record.get('stats', {}).get('story_completes') }}" + - path: ["attachment_frequency"] + value: "{{ record.get('stats', {}).get('attachment_frequency') }}" + - path: ["attachment_uniques"] + value: "{{ record.get('stats', {}).get('attachment_uniques') }}" + - path: ["frequency"] + value: "{{ record.get('stats', {}).get('frequency') }}" + - path: ["uniques"] + value: "{{ record.get('stats', {}).get('uniques') }}" + - path: ["total_reach"] + value: "{{ record.get('stats', {}).get('total_reach') }}" + - path: ["earned_reach"] + value: "{{ record.get('stats', {}).get('earned_reach') }}" + - type: RemoveFields + field_pointers: + - - stats + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/basic_stats" + base_requester: + type: HttpRequester + url_base: https://adsapi.snapchat.com/v1/ + authenticator: + type: OAuthAuthenticator + refresh_request_body: {} + token_refresh_endpoint: https://accounts.snapchat.com/login/oauth2/access_token + grant_type: refresh_token + client_id: '{{ config["client_id"] }}' + client_secret: '{{ config["client_secret"] }}' + refresh_token: '{{ config["refresh_token"] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + action: RETRY + http_codes: + - 403 + error_message: >- + Got permission error when accessing URL. Skipping {{self.name}} stream. + +streams: + - $ref: "#/definitions/streams/organizations" + - $ref: "#/definitions/streams/adaccounts" + - $ref: "#/definitions/streams/creatives" + - $ref: "#/definitions/streams/ads" + - $ref: "#/definitions/streams/adsquads" + - $ref: "#/definitions/streams/segments" + - $ref: "#/definitions/streams/media" + - $ref: "#/definitions/streams/campaigns" + - $ref: "#/definitions/streams/adaccounts_stats_hourly" + - $ref: "#/definitions/streams/adaccounts_stats_daily" + - $ref: "#/definitions/streams/adaccounts_stats_lifetime" + - $ref: "#/definitions/streams/ads_stats_hourly" + - $ref: "#/definitions/streams/ads_stats_daily" + - $ref: "#/definitions/streams/ads_stats_lifetime" + - $ref: "#/definitions/streams/adsquads_stats_hourly" + - $ref: "#/definitions/streams/adsquads_stats_daily" + - $ref: "#/definitions/streams/adsquads_stats_lifetime" + - $ref: "#/definitions/streams/campaigns_stats_hourly" + - $ref: "#/definitions/streams/campaigns_stats_daily" + - $ref: "#/definitions/streams/campaigns_stats_lifetime" + +spec: + type: Spec + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Snapchat Marketing Spec + type: object + required: + - client_id + - client_secret + - refresh_token + properties: + client_id: + title: Client ID + type: string + description: The Client ID of your Snapchat developer application. + airbyte_secret: true + order: 0 + client_secret: + title: Client Secret + type: string + description: The Client Secret of your Snapchat developer application. + airbyte_secret: true + order: 1 + refresh_token: + title: Refresh Token + type: string + description: Refresh Token to renew the expired Access Token. + airbyte_secret: true + order: 2 + start_date: + title: Start Date + type: string + description: Date in the format 2022-01-01. Any data before this date will not be replicated. + examples: + - "2022-01-01" + default: "2022-01-01" + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + order: 3 + format: date + end_date: + type: string + title: End Date + description: Date in the format 2017-01-25. Any data after this date will not be replicated. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - "2022-01-30" + order: 4 + format: date + action_report_time: + type: string + enum: + - conversion + - impression + title: Action Report Time + description: Specifies the principle for conversion reporting. + default: conversion + order: 5 + swipe_up_attribution_window: + type: string + title: Swipe Up Attribution Window + description: Attribution window for swipe ups. + enum: + - 1_DAY + - 7_DAY + - 28_DAY + default: 28_DAY + order: 6 + view_attribution_window: + type: string + title: View Attribution Window + description: Attribution window for views. + enum: + - 1_HOUR + - 3_HOUR + - 6_HOUR + - 1_DAY + - 7_DAY + default: 1_DAY + order: 7 + advanced_auth: + auth_flow_type: oauth2.0 + oauth_config_specification: + complete_oauth_output_specification: + type: object + properties: + refresh_token: + type: string + path_in_connector_config: + - refresh_token + complete_oauth_server_input_specification: + type: object + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + properties: + client_id: + type: string + path_in_connector_config: + - client_id + client_secret: + type: string + path_in_connector_config: + - client_secret + +metadata: + autoImportSchema: + organizations: false + adaccounts: false + creatives: false + ads: false + adsquads: false + segments: false + media: false + campaigns: false + adaccounts_stats_hourly: false + adaccounts_stats_daily: false + adaccounts_stats_lifetime: false + ads_stats_hourly: false + ads_stats_daily: false + ads_stats_lifetime: false + adsquads_stats_hourly: false + adsquads_stats_daily: false + adsquads_stats_lifetime: false + campaigns_stats_hourly: false + campaigns_stats_daily: false + campaigns_stats_lifetime: false + +schemas: + organizations: + $schema: http://json-schema.org/schema# + additionalProperties: true + type: + - object + properties: + id: + description: Unique identifier for the organization record. + type: + - "null" + - string + updated_at: + description: Timestamp indicating the last update date of the organization record. + type: + - "null" + - string + created_at: + description: Timestamp indicating the creation date of the organization record. + type: + - "null" + - string + name: + description: The name of the organization. + type: + - "null" + - string + country: + description: The country where the organization is located. + type: + - "null" + - string + postal_code: + description: The postal code of the organization's location. + type: + - "null" + - string + locality: + description: The locality or city where the organization is situated. + type: + - "null" + - string + contact_name: + description: The name of the contact person within the organization. + type: + - "null" + - string + contact_email: + description: The email address used for contact purposes. + type: + - "null" + - string + contact_phone: + description: The phone number of the contact person within the organization. + type: + - "null" + - string + address_line_1: + description: The first line of the organization's address. + type: + - "null" + - string + administrative_district_level_1: + description: The first-level administrative district of the organization's location. + type: + - "null" + - string + accepted_term_version: + description: The version of the terms and conditions accepted by the organization. + type: + - "null" + - string + contact_phone_optin: + description: Flag indicating if contact person has opted in for phone contact. + type: + - "null" + - boolean + configuration_settings: + description: Settings related to organization configurations + type: + - "null" + - object + properties: + notifications_enabled: + description: Flag indicating if notifications are enabled for the organization. + type: + - "null" + - boolean + type: + description: The type or category of the organization. + type: + - "null" + - string + state: + description: The state or region where the organization is located. + type: + - "null" + - string + roles: + description: List of roles assigned to the organization. + type: + - "null" + - array + items: + description: A specific role assigned to the organization. + type: + - "null" + - string + my_display_name: + description: The public display name of the organization. + type: + - "null" + - string + my_invited_email: + description: Email address of the member invited to the organization. + type: + - "null" + - string + my_member_id: + description: Unique identifier of the member within the organization. + type: + - "null" + - string + createdByCaller: + description: Record of the creator of the organization within the system. + type: + - "null" + - boolean + adaccounts: + $schema: http://json-schema.org/schema# + additionalProperties: true + type: + - object + properties: + id: + description: The unique identifier for the ad account. + type: + - "null" + - string + updated_at: + description: The timestamp when the ad account details were last updated. + type: + - "null" + - string + created_at: + description: The timestamp when the ad account was created. + type: + - "null" + - string + name: + description: The name or title of the ad account. + type: + - "null" + - string + type: + description: The type or category of the ad account. + type: + - "null" + - string + status: + description: The current status or state of the ad account. + type: + - "null" + - string + organization_id: + description: The ID of the organization that owns the ad account. + type: + - "null" + - string + funding_source_ids: + description: IDs of the funding sources linked to the ad account. + type: + - "null" + - array + items: + description: Individual funding source ID. + type: + - "null" + - string + currency: + description: The currency used for financial transactions within the ad account. + type: + - "null" + - string + timezone: + description: The timezone setting for the ad account operations. + type: + - "null" + - string + advertiser_organization_id: + description: The ID of the organization that is advertising on the platform. + type: + - "null" + - string + billing_center_id: + description: The ID of the billing center associated with the ad account. + type: + - "null" + - string + billing_type: + description: The type of billing arrangement for the ad account. + type: + - "null" + - string + agency_representing_client: + description: The agency representing the client for ad account management. + type: + - "null" + - boolean + client_paying_invoices: + description: Indicates if the client is responsible for paying the invoices. + type: + - "null" + - boolean + regulations: + description: Regulatory information associated with the ad account. + type: + - "null" + - object + properties: + restricted_delivery_signals: + description: Signals or content types subject to delivery restrictions. + type: + - "null" + - boolean + creatives: + $schema: http://json-schema.org/schema# + additionalProperties: true + type: + - object + properties: + id: + description: The unique identifier of the creative + type: + - "null" + - string + updated_at: + description: The timestamp indicating when the creative was last updated + type: + - "null" + - string + created_at: + description: The timestamp indicating when the creative was created + type: + - "null" + - string + name: + description: The name or title given to the creative + type: + - "null" + - string + ad_account_id: + description: The ID of the advertising account linked to the creative + type: + - "null" + - string + type: + description: The type or category of the creative + type: + - "null" + - string + packaging_status: + description: The status of the packaging for the creative + type: + - "null" + - string + review_status: + description: The overall review status of the creative + type: + - "null" + - string + review_status_details: + description: Additional details related to the review status + type: + - "null" + - string + shareable: + description: Indicates if the creative is shareable + type: + - "null" + - boolean + forced_view_eligibility: + description: Whether the creative is eligible for forced views + type: + - "null" + - string + headline: + description: The headline or title of the creative + type: + - "null" + - string + brand_name: + description: The name of the brand associated with the creative + type: + - "null" + - string + call_to_action: + description: The call to action prompt for the creative + type: + - "null" + - string + render_type: + description: The type of rendering used for the creative + type: + - "null" + - string + top_snap_media_id: + description: The ID of the media file for the top snap + type: + - "null" + - string + top_snap_crop_position: + description: The crop position for the top snap of the creative + type: + - "null" + - string + web_view_properties: + description: Properties related to the web view functionality when a user interacts with the ad. + type: + - "null" + - object + properties: + url: + description: The URL to be loaded in the web view + type: + - "null" + - string + allow_snap_javascript_sdk: + description: Whether to allow the use of Snap JavaScript SDK in the web view + type: + - "null" + - boolean + use_immersive_mode: + description: Whether to use immersive mode in the web view + type: + - "null" + - boolean + deep_link_urls: + description: List of deep link URLs associated with the web view + type: + - "null" + - array + block_preload: + description: Indicates if preloading is blocked for the web view + type: + - "null" + - boolean + ad_product: + description: The type of Snapchat advertising product used for the creative + type: + - "null" + - string + ad_to_place_properties: + description: Properties related to the ad to be displayed within the Snapchat interface. + type: + - "null" + - object + properties: + place_id: + description: The ID of the place where the ad is to be displayed + type: + - "null" + - string + ads: + $schema: http://json-schema.org/schema# + additionalProperties: true + type: + - object + properties: + id: + description: The unique identifier of the ad. + type: + - "null" + - string + updated_at: + description: The date and time when the ad was last updated. + type: + - "null" + - string + created_at: + description: The date and time when the ad was created. + type: + - "null" + - string + name: + description: The name or title of the ad. + type: + - "null" + - string + ad_squad_id: + description: The unique identifier of the ad squad this ad belongs to. + type: + - "null" + - string + creative_id: + description: The unique identifier of the creative content used in the ad. + type: + - "null" + - string + status: + description: The current status of the ad. + type: + - "null" + - string + type: + description: The type or category of the ad. + type: + - "null" + - string + render_type: + description: The type of rendering used for the ad. + type: + - "null" + - string + review_status: + description: The review status of the ad. + type: + - "null" + - string + review_status_reasons: + description: Reasons for the review status of the ad. + type: + - "null" + - array + items: + type: + - "null" + - string + delivery_status: + description: The delivery status of the ad. + type: + - "null" + - array + items: + type: + - "null" + - string + adsquads: + $schema: http://json-schema.org/schema# + additionalProperties: true + type: + - object + properties: + id: + description: Unique identifier for the ad squad. + type: + - "null" + - string + updated_at: + description: Timestamp indicating when the ad squad was last updated. + type: + - "null" + - string + created_at: + description: Timestamp indicating when the ad squad was created. + type: + - "null" + - string + name: + description: Name of the ad squad. + type: + - "null" + - string + status: + description: Current status of the ad squad. + type: + - "null" + - string + campaign_id: + description: ID of the campaign associated with the ad squad. + type: + - "null" + - string + type: + description: Type of the ad squad. + type: + - "null" + - string + targeting: + description: Targeting settings for the ad squad. + type: + - "null" + - object + properties: + regulated_content: + description: Indicates whether regulated content targeting is enabled. + type: + - "null" + - boolean + geos: + description: Geographic targeting criteria. + type: + - "null" + - array + items: + description: Individual geographic item. + type: + - "null" + - object + properties: + country_code: + description: Country code for geographic targeting. + type: + - "null" + - string + operation: + description: Operation applied for geographic targeting. + type: + - "null" + - string + locations: + description: Location targeting criteria. + type: + - "null" + - array + items: + description: Individual location item. + type: + - "null" + - object + properties: + circles: + description: Circular location details. + type: + - "null" + - array + items: + description: Individual circle item. + type: + - "null" + - object + properties: + latitude: + description: Latitude coordinate of the circle center. + type: + - "null" + - number + longitude: + description: Longitude coordinate of the circle center. + type: + - "null" + - number + name: + description: Name of the circle location. + type: + - "null" + - string + radius: + description: Radius of the circle in specified unit. + type: + - "null" + - number + unit: + description: Unit used to measure the radius of the circle. + type: + - "null" + - string + operation: + description: Operation applied for location targeting. + type: + - "null" + - string + auto_expansion_options: + description: Options for automatic expansion in targeting. + type: + - "null" + - object + properties: + interest_expansion_option: + description: Settings for interest expansion option. + type: + - "null" + - object + properties: + enabled: + description: Indicates whether interest expansion is enabled. + type: + - "null" + - boolean + enable_targeting_expansion: + description: Option to enable targeting expansion. + type: + - "null" + - boolean + interests: + description: Interest targeting criteria. + type: + - "null" + - array + items: + description: Individual interest item. + type: + - "null" + - object + properties: + category_id: + description: Category ID for interest targeting. + type: + - "null" + - array + items: + description: Individual category ID item. + type: + - "null" + - string + operation: + description: Operation applied for interest targeting. + type: + - "null" + - string + demographics: + description: Demographic targeting criteria. + type: + - "null" + - array + items: + description: Individual demographic item. + type: + - "null" + - object + targeting_reach_status: + description: Status of targeting reach for the ad squad. + type: + - "null" + - string + placement: + description: Ad placement settings for the ad squad. + type: + - "null" + - string + billing_event: + description: Defines the billing event for the ad squad. + type: + - "null" + - string + auto_bid: + description: Indicates whether the ad squad is using auto-bidding for its bids. + type: + - "null" + - boolean + target_bid: + description: Target bid value for the ad squad. + type: + - "null" + - boolean + bid_strategy: + description: Specifies the bidding strategy used by the ad squad. + type: + - "null" + - string + daily_budget_micro: + description: Daily budget in micro currency units for the ad squad. + type: + - "null" + - integer + start_time: + description: Start time for the ad squad's run. + type: + - "null" + - string + optimization_goal: + description: Goal used for optimizing ad delivery. + type: + - "null" + - string + delivery_constraint: + description: Constraints applied to the delivery of the ad squad. + type: + - "null" + - string + delivery_properties_version: + description: Version of delivery properties for the ad squad. + type: + - "null" + - integer + pacing_type: + description: Type of pacing applied to the ad squad's delivery. + type: + - "null" + - string + child_ad_type: + description: Type of child ads under the ad squad. + type: + - "null" + - string + forced_view_setting: + description: Setting to force views for the ad squad. + type: + - "null" + - string + creation_state: + description: Current state of the ad squad creation process. + type: + - "null" + - string + delivery_status: + description: Status of ad delivery for the ad squad. + type: + - "null" + - array + items: + description: Individual delivery status item. + type: + - "null" + - string + event_sources: + description: Sources of events associated with the ad squad. + type: + - "null" + - object + properties: + PLACE: + description: Event source related to a specific place. + type: + - "null" + - array + items: + description: Individual event source item. + type: + - "null" + - string + skadnetwork_properties: + description: Properties related to SKAdNetwork integration for the ad squad. + type: + - "null" + - object + properties: + ecid_enrollment_status: + description: Status of ECID enrollment. + type: + - "null" + - string + enable_skoverlay: + description: Option to enable SKOverlay. + type: + - "null" + - boolean + status: + description: Current status of SKAdNetwork. + type: + - "null" + - string + lifetime_budget_micro: + description: Total budget in micro currency units for the lifetime of the ad squad. + type: + - "null" + - integer + end_time: + description: End time for the ad squad's run. + type: + - "null" + - string + segments: + $schema: http://json-schema.org/schema# + additionalProperties: true + type: + - object + properties: + id: + description: Unique identifier for the segment. + type: + - "null" + - string + updated_at: + description: The date and time when the segment was last updated. + type: + - "null" + - string + created_at: + description: The date and time when the segment was created. + type: + - "null" + - string + name: + description: Name or title of the segment. + type: + - "null" + - string + ad_account_id: + description: The unique identifier for the ad account associated with the segment. + type: + - "null" + - string + organization_id: + description: Unique identifier for the organization to which the segment belongs. + type: + - "null" + - string + description: + description: Brief description of the segment. + type: + - "null" + - string + status: + description: Current status of the segment (e.g., active, inactive). + type: + - "null" + - string + targetable_status: + description: Status indicating whether the segment can be targeted in marketing campaigns. + type: + - "null" + - string + upload_status: + description: Status of the segment upload process (e.g., pending, completed). + type: + - "null" + - string + source_type: + description: Type of the source data used to create the segment. + type: + - "null" + - string + retention_in_days: + description: Number of days for which the segment data is retained. + type: + - "null" + - integer + approximate_number_users: + description: Approximate number of users within the segment. + type: + - "null" + - integer + visible_to: + description: Visibility setting for the segment, determining who can access it. + type: + - "null" + - array + items: + type: + - "null" + - string + media: + $schema: http://json-schema.org/schema# + additionalProperties: true + type: + - object + properties: + id: + description: The unique identifier of the media. + type: + - "null" + - string + updated_at: + description: The timestamp when the media was last updated. + type: + - "null" + - string + created_at: + description: The timestamp when the media was created. + type: + - "null" + - string + ad_account_id: + description: The unique identifier of the ad account associated with the media. + type: + - "null" + - string + type: + description: The type of media (e.g., image, video). + type: + - "null" + - string + media_status: + description: The status of the media (e.g., active, inactive). + type: + - "null" + - string + media_usages: + description: Items associated with the media usage. + type: + - "null" + - array + items: + description: Details of the media usage. + type: + - "null" + - string + file_name: + description: The name of the media file. + type: + - "null" + - string + download_link: + description: The link to download the media file. + type: + - "null" + - string + duration_in_seconds: + description: The duration of the media in seconds. + type: + - "null" + - number + image_metadata: + description: Additional metadata related to the image media. + type: + - "null" + - object + video_metadata: + description: Additional metadata related to the video media. + type: + - "null" + - object + file_size_in_bytes: + description: The file size of the media in bytes. + type: + - "null" + - integer + is_demo_media: + description: Indicates if the media is a demo or not. + type: + - "null" + - boolean + hash: + description: The hash value of the media file for identification purposes. + type: + - "null" + - string + visibility: + description: The visibility settings of the media (e.g., public, private). + type: + - "null" + - string + name: + description: The name or title of the media. + type: + - "null" + - string + campaigns: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: + - object + properties: + id: + description: The unique identifier of the campaign. + type: + - "null" + - string + updated_at: + description: The timestamp indicating when the campaign was last updated. + type: + - "null" + - string + created_at: + description: The timestamp indicating when the campaign was created. + type: + - "null" + - string + name: + description: The name assigned to the campaign for easy identification. + type: + - "null" + - string + ad_account_id: + description: The unique identifier of the advertising account associated with the campaign. + type: + - "null" + - string + status: + description: The current status of the campaign (e.g., paused, active). + type: + - "null" + - string + objective: + description: The primary goal or objective of the campaign. + type: + - "null" + - string + start_time: + description: The starting date and time for the campaign to begin running. + type: + - "null" + - string + buy_model: + description: The purchasing model used for the campaign (e.g., auction, reach and frequency). + type: + - "null" + - string + delivery_status: + description: The status of campaign delivery. + type: + - "null" + - array + items: + type: + - "null" + - string + creation_state: + description: The current state of the campaign creation process. + type: + - "null" + - string + basic_stats: + $schema: http://json-schema.org/schema# + additionalProperties: true + type: + - object + properties: + id: + description: Identifier for the data entry. + type: + - string + type: + description: Type of data entry. + type: + - string + granularity: + description: Granularity of the data. + type: + - string + start_time: + description: Start time of the data snapshot. + type: + - "null" + - string + format: date-time + end_time: + description: End time of the data snapshot. + type: + - "null" + - string + format: date-time + android_installs: + description: Total number of installs on Android devices. + type: + - "null" + - number + attachment_avg_view_time_millis: + description: Average time in milliseconds a viewer spends viewing an attachment. + type: + - "null" + - number + attachment_impressions: + description: Total number of times an attachment is displayed to users. + type: + - "null" + - number + attachment_quartile_1: + description: First quartile of attachment viewing time. + type: + - "null" + - number + attachment_quartile_2: + description: Second quartile of attachment viewing time. + type: + - "null" + - number + attachment_quartile_3: + description: Third quartile of attachment viewing time. + type: + - "null" + - number + attachment_total_view_time_millis: + description: Total time in milliseconds viewers spend on attachment views. + type: + - "null" + - number + attachment_view_completion: + description: Percentage of attachment views that are completed. + type: + - "null" + - number + avg_screen_time_millis: + description: Average time in milliseconds users spent on the screen. + type: + - "null" + - number + avg_view_time_millis: + description: Average time in milliseconds viewers spend on viewing content. + type: + - "null" + - number + impressions: + description: Total number of impressions recorded. + type: + - "null" + - number + ios_installs: + description: Total number of installs on iOS devices. + type: + - "null" + - number + quartile_1: + description: First quartile of viewing time. + type: + - "null" + - number + quartile_2: + description: Second quartile of viewing time. + type: + - "null" + - number + quartile_3: + description: Third quartile of viewing time. + type: + - "null" + - number + screen_time_millis: + description: Total time in milliseconds users spend on the screen. + type: + - "null" + - number + spend: + description: Total spend for marketing activities. + type: + - "null" + - number + swipe_up_percent: + description: Percentage of users who swiped up. + type: + - "null" + - number + swipes: + description: Total number of swipe gestures. + type: + - "null" + - number + total_installs: + description: Total number of installations. + type: + - "null" + - number + video_views: + description: Total number of video views. + type: + - "null" + - number + video_views_time_based: + description: Video views based on time duration. + type: + - "null" + - number + video_views_15s: + description: Total number of video views that last at least 15 seconds. + type: + - "null" + - number + view_completion: + description: Percentage of views that are completed. + type: + - "null" + - number + view_time_millis: + description: Total time viewers spend on viewing content in milliseconds. + type: + - "null" + - number + paid_impressions: + description: Total number of impressions from paid sources. + type: + - "null" + - number + earned_impressions: + description: Impressions earned through engagements. + type: + - "null" + - number + total_impressions: + description: Total number of impressions including earned and paid. + type: + - "null" + - number + play_time_millis: + description: Total time in milliseconds users spend playing content. + type: + - "null" + - number + shares: + description: Total number of content shares. + type: + - "null" + - number + saves: + description: Number of saves performed. + type: + - "null" + - number + native_leads: + description: Number of leads generated natively. + type: + - "null" + - number + conversion_purchases: + description: Number of total purchases made. + type: + - "null" + - number + conversion_purchases_value: + description: Total value of purchases made. + type: + - "null" + - number + conversion_save: + description: Number of saves performed. + type: + - "null" + - number + conversion_start_checkout: + description: Number of times checkout process is initiated. + type: + - "null" + - number + conversion_add_cart: + description: Conversion events where items are added to cart. + type: + - "null" + - number + conversion_view_content: + description: Number of content views. + type: + - "null" + - number + conversion_add_billing: + description: Conversion events where billing information is added. + type: + - "null" + - number + conversion_searches: + description: Number of search queries made. + type: + - "null" + - number + conversion_level_completes: + description: Number of game levels completed. + type: + - "null" + - number + conversion_app_opens: + description: Number of times the app is opened. + type: + - "null" + - number + conversion_page_views: + description: Total number of page views. + type: + - "null" + - number + conversion_subscribe: + description: Number of subscriptions made. + type: + - "null" + - number + conversion_ad_click: + description: Number of ad clicks recorded. + type: + - "null" + - number + conversion_ad_view: + description: Number of ad views. + type: + - "null" + - number + conversion_complete_tutorial: + description: Number of completed tutorials. + type: + - "null" + - number + conversion_invite: + description: Number of invitations sent by users. + type: + - "null" + - number + conversion_login: + description: Number of logins recorded. + type: + - "null" + - number + conversion_share: + description: Number of shares made. + type: + - "null" + - number + conversion_reserve: + description: Reserve conversion events. + type: + - "null" + - number + conversion_achievement_unlocked: + description: Number of times achievements are unlocked. + type: + - "null" + - number + conversion_add_to_wishlist: + description: Conversion events where items are added to the wishlist. + type: + - "null" + - number + conversion_spend_credits: + description: Total credits spent during conversion events. + type: + - "null" + - number + conversion_rate: + description: Rate of conversion events. + type: + - "null" + - number + conversion_start_trial: + description: Number of trials started. + type: + - "null" + - number + conversion_list_view: + description: Number of views on the list content. + type: + - "null" + - number + custom_event_1: + description: Custom event tracking 1. + type: + - "null" + - number + custom_event_2: + description: Custom event tracking 2. + type: + - "null" + - number + custom_event_3: + description: Custom event tracking 3. + type: + - "null" + - number + custom_event_4: + description: Custom event tracking 4. + type: + - "null" + - number + custom_event_5: + description: Custom event tracking 5. + type: + - "null" + - number + attachment_frequency: + description: Frequency of attachment views. + type: + - "null" + - number + attachment_uniques: + description: Unique viewers of attachments. + type: + - "null" + - number + frequency: + description: Frequency of events tracked. + type: + - "null" + - number + uniques: + description: Total unique events or users. + type: + - "null" + - number + total_reach: + description: Total reach including earned and paid. + type: + - "null" + - number + earned_reach: + description: Total reach earned through engagements. + type: + - "null" + - number + story_opens: + description: Total number of story opens. + type: + - "null" + - number + story_completes: + description: Total number of completed stories. + type: + - "null" + - number diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/adaccounts.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/adaccounts.json deleted file mode 100644 index 7df344fabc81..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/adaccounts.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "The unique identifier for the ad account.", - "type": ["null", "string"] - }, - "updated_at": { - "description": "The timestamp when the ad account details were last updated.", - "type": ["null", "string"] - }, - "created_at": { - "description": "The timestamp when the ad account was created.", - "type": ["null", "string"] - }, - "name": { - "description": "The name or title of the ad account.", - "type": ["null", "string"] - }, - "type": { - "description": "The type or category of the ad account.", - "type": ["null", "string"] - }, - "status": { - "description": "The current status or state of the ad account.", - "type": ["null", "string"] - }, - "organization_id": { - "description": "The ID of the organization that owns the ad account.", - "type": ["null", "string"] - }, - "funding_source_ids": { - "description": "IDs of the funding sources linked to the ad account.", - "type": ["null", "array"], - "items": { - "description": "Individual funding source ID.", - "type": ["null", "string"] - } - }, - "currency": { - "description": "The currency used for financial transactions within the ad account.", - "type": ["null", "string"] - }, - "timezone": { - "description": "The timezone setting for the ad account operations.", - "type": ["null", "string"] - }, - "advertiser_organization_id": { - "description": "The ID of the organization that is advertising on the platform.", - "type": ["null", "string"] - }, - "billing_center_id": { - "description": "The ID of the billing center associated with the ad account.", - "type": ["null", "string"] - }, - "billing_type": { - "description": "The type of billing arrangement for the ad account.", - "type": ["null", "string"] - }, - "agency_representing_client": { - "description": "The agency representing the client for ad account management.", - "type": ["null", "boolean"] - }, - "client_paying_invoices": { - "description": "Indicates if the client is responsible for paying the invoices.", - "type": ["null", "boolean"] - }, - "regulations": { - "description": "Regulatory information associated with the ad account.", - "type": ["null", "object"], - "properties": { - "restricted_delivery_signals": { - "description": "Signals or content types subject to delivery restrictions.", - "type": ["null", "boolean"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/ads.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/ads.json deleted file mode 100644 index 1ff38f054200..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/ads.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "The unique identifier of the ad.", - "type": ["null", "string"] - }, - "updated_at": { - "description": "The date and time when the ad was last updated.", - "type": ["null", "string"] - }, - "created_at": { - "description": "The date and time when the ad was created.", - "type": ["null", "string"] - }, - "name": { - "description": "The name or title of the ad.", - "type": ["null", "string"] - }, - "ad_squad_id": { - "description": "The unique identifier of the ad squad this ad belongs to.", - "type": ["null", "string"] - }, - "creative_id": { - "description": "The unique identifier of the creative content used in the ad.", - "type": ["null", "string"] - }, - "status": { - "description": "The current status of the ad.", - "type": ["null", "string"] - }, - "type": { - "description": "The type or category of the ad.", - "type": ["null", "string"] - }, - "render_type": { - "description": "The type of rendering used for the ad.", - "type": ["null", "string"] - }, - "review_status": { - "description": "The review status of the ad.", - "type": ["null", "string"] - }, - "review_status_reasons": { - "description": "Reasons for the review status of the ad.", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "delivery_status": { - "description": "The delivery status of the ad.", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/adsquads.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/adsquads.json deleted file mode 100644 index 9c8ee054f5bd..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/adsquads.json +++ /dev/null @@ -1,263 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "Unique identifier for the ad squad.", - "type": ["null", "string"] - }, - "updated_at": { - "description": "Timestamp indicating when the ad squad was last updated.", - "type": ["null", "string"] - }, - "created_at": { - "description": "Timestamp indicating when the ad squad was created.", - "type": ["null", "string"] - }, - "name": { - "description": "Name of the ad squad.", - "type": ["null", "string"] - }, - "status": { - "description": "Current status of the ad squad.", - "type": ["null", "string"] - }, - "campaign_id": { - "description": "ID of the campaign associated with the ad squad.", - "type": ["null", "string"] - }, - "type": { - "description": "Type of the ad squad.", - "type": ["null", "string"] - }, - "targeting": { - "description": "Targeting settings for the ad squad.", - "type": ["null", "object"], - "properties": { - "regulated_content": { - "description": "Indicates whether regulated content targeting is enabled.", - "type": ["null", "boolean"] - }, - "geos": { - "description": "Geographic targeting criteria.", - "type": ["null", "array"], - "items": { - "description": "Individual geographic item.", - "type": ["null", "object"], - "properties": { - "country_code": { - "description": "Country code for geographic targeting.", - "type": ["null", "string"] - }, - "operation": { - "description": "Operation applied for geographic targeting.", - "type": ["null", "string"] - } - } - } - }, - "locations": { - "description": "Location targeting criteria.", - "type": ["null", "array"], - "items": { - "description": "Individual location item.", - "type": ["null", "object"], - "properties": { - "circles": { - "description": "Circular location details.", - "type": ["null", "array"], - "items": { - "description": "Individual circle item.", - "type": ["null", "object"], - "properties": { - "latitude": { - "description": "Latitude coordinate of the circle center.", - "type": ["null", "number"] - }, - "longitude": { - "description": "Longitude coordinate of the circle center.", - "type": ["null", "number"] - }, - "name": { - "description": "Name of the circle location.", - "type": ["null", "string"] - }, - "radius": { - "description": "Radius of the circle in specified unit.", - "type": ["null", "number"] - }, - "unit": { - "description": "Unit used to measure the radius of the circle.", - "type": ["null", "string"] - } - } - } - }, - "operation": { - "description": "Operation applied for location targeting.", - "type": ["null", "string"] - } - } - } - }, - "auto_expansion_options": { - "description": "Options for automatic expansion in targeting.", - "type": ["null", "object"], - "properties": { - "interest_expansion_option": { - "description": "Settings for interest expansion option.", - "type": ["null", "object"], - "properties": { - "enabled": { - "description": "Indicates whether interest expansion is enabled.", - "type": ["null", "boolean"] - } - } - } - } - }, - "enable_targeting_expansion": { - "description": "Option to enable targeting expansion.", - "type": ["null", "boolean"] - }, - "interests": { - "description": "Interest targeting criteria.", - "type": ["null", "array"], - "items": { - "description": "Individual interest item.", - "type": ["null", "object"], - "properties": { - "category_id": { - "description": "Category ID for interest targeting.", - "type": ["null", "array"], - "items": { - "description": "Individual category ID item.", - "type": ["null", "string"] - } - }, - "operation": { - "description": "Operation applied for interest targeting.", - "type": ["null", "string"] - } - } - } - }, - "demographics": { - "description": "Demographic targeting criteria.", - "type": ["null", "array"], - "items": { - "description": "Individual demographic item.", - "type": ["null", "object"] - } - } - } - }, - "targeting_reach_status": { - "description": "Status of targeting reach for the ad squad.", - "type": ["null", "string"] - }, - "placement": { - "description": "Ad placement settings for the ad squad.", - "type": ["null", "string"] - }, - "billing_event": { - "description": "Defines the billing event for the ad squad.", - "type": ["null", "string"] - }, - "auto_bid": { - "description": "Indicates whether the ad squad is using auto-bidding for its bids.", - "type": ["null", "boolean"] - }, - "target_bid": { - "description": "Target bid value for the ad squad.", - "type": ["null", "boolean"] - }, - "bid_strategy": { - "description": "Specifies the bidding strategy used by the ad squad.", - "type": ["null", "string"] - }, - "daily_budget_micro": { - "description": "Daily budget in micro currency units for the ad squad.", - "type": ["null", "integer"] - }, - "start_time": { - "description": "Start time for the ad squad's run.", - "type": ["null", "string"] - }, - "optimization_goal": { - "description": "Goal used for optimizing ad delivery.", - "type": ["null", "string"] - }, - "delivery_constraint": { - "description": "Constraints applied to the delivery of the ad squad.", - "type": ["null", "string"] - }, - "delivery_properties_version": { - "description": "Version of delivery properties for the ad squad.", - "type": ["null", "integer"] - }, - "pacing_type": { - "description": "Type of pacing applied to the ad squad's delivery.", - "type": ["null", "string"] - }, - "child_ad_type": { - "description": "Type of child ads under the ad squad.", - "type": ["null", "string"] - }, - "forced_view_setting": { - "description": "Setting to force views for the ad squad.", - "type": ["null", "string"] - }, - "creation_state": { - "description": "Current state of the ad squad creation process.", - "type": ["null", "string"] - }, - "delivery_status": { - "description": "Status of ad delivery for the ad squad.", - "type": ["null", "array"], - "items": { - "description": "Individual delivery status item.", - "type": ["null", "string"] - } - }, - "event_sources": { - "description": "Sources of events associated with the ad squad.", - "type": ["null", "object"], - "properties": { - "PLACE": { - "description": "Event source related to a specific place.", - "type": ["null", "array"], - "items": { - "description": "Individual event source item.", - "type": ["null", "string"] - } - } - } - }, - "skadnetwork_properties": { - "description": "Properties related to SKAdNetwork integration for the ad squad.", - "type": ["null", "object"], - "properties": { - "ecid_enrollment_status": { - "description": "Status of ECID enrollment.", - "type": ["null", "string"] - }, - "enable_skoverlay": { - "description": "Option to enable SKOverlay.", - "type": ["null", "boolean"] - }, - "status": { - "description": "Current status of SKAdNetwork.", - "type": ["null", "string"] - } - } - }, - "lifetime_budget_micro": { - "description": "Total budget in micro currency units for the lifetime of the ad squad.", - "type": ["null", "integer"] - }, - "end_time": { - "description": "End time for the ad squad's run.", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/basic_stats.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/basic_stats.json deleted file mode 100644 index 46fac3695562..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/basic_stats.json +++ /dev/null @@ -1,307 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "Identifier for the data entry.", - "type": ["string"] - }, - "type": { - "description": "Type of data entry.", - "type": ["string"] - }, - "granularity": { - "description": "Granularity of the data.", - "type": ["string"] - }, - "start_time": { - "description": "Start time of the data snapshot.", - "type": ["null", "string"], - "format": "date-time" - }, - "end_time": { - "description": "End time of the data snapshot.", - "type": ["null", "string"], - "format": "date-time" - }, - "android_installs": { - "description": "Total number of installs on Android devices.", - "type": ["null", "number"] - }, - "attachment_avg_view_time_millis": { - "description": "Average time in milliseconds a viewer spends viewing an attachment.", - "type": ["null", "number"] - }, - "attachment_impressions": { - "description": "Total number of times an attachment is displayed to users.", - "type": ["null", "number"] - }, - "attachment_quartile_1": { - "description": "First quartile of attachment viewing time.", - "type": ["null", "number"] - }, - "attachment_quartile_2": { - "description": "Second quartile of attachment viewing time.", - "type": ["null", "number"] - }, - "attachment_quartile_3": { - "description": "Third quartile of attachment viewing time.", - "type": ["null", "number"] - }, - "attachment_total_view_time_millis": { - "description": "Total time in milliseconds viewers spend on attachment views.", - "type": ["null", "number"] - }, - "attachment_view_completion": { - "description": "Percentage of attachment views that are completed.", - "type": ["null", "number"] - }, - "avg_screen_time_millis": { - "description": "Average time in milliseconds users spent on the screen.", - "type": ["null", "number"] - }, - "avg_view_time_millis": { - "description": "Average time in milliseconds viewers spend on viewing content.", - "type": ["null", "number"] - }, - "impressions": { - "description": "Total number of impressions recorded.", - "type": ["null", "number"] - }, - "ios_installs": { - "description": "Total number of installs on iOS devices.", - "type": ["null", "number"] - }, - "quartile_1": { - "description": "First quartile of viewing time.", - "type": ["null", "number"] - }, - "quartile_2": { - "description": "Second quartile of viewing time.", - "type": ["null", "number"] - }, - "quartile_3": { - "description": "Third quartile of viewing time.", - "type": ["null", "number"] - }, - "screen_time_millis": { - "description": "Total time in milliseconds users spend on the screen.", - "type": ["null", "number"] - }, - "spend": { - "description": "Total spend for marketing activities.", - "type": ["null", "number"] - }, - "swipe_up_percent": { - "description": "Percentage of users who swiped up.", - "type": ["null", "number"] - }, - "swipes": { - "description": "Total number of swipe gestures.", - "type": ["null", "number"] - }, - "total_installs": { - "description": "Total number of installations.", - "type": ["null", "number"] - }, - "video_views": { - "description": "Total number of video views.", - "type": ["null", "number"] - }, - "video_views_time_based": { - "description": "Video views based on time duration.", - "type": ["null", "number"] - }, - "video_views_15s": { - "description": "Total number of video views that last at least 15 seconds.", - "type": ["null", "number"] - }, - "view_completion": { - "description": "Percentage of views that are completed.", - "type": ["null", "number"] - }, - "view_time_millis": { - "description": "Total time viewers spend on viewing content in milliseconds.", - "type": ["null", "number"] - }, - "paid_impressions": { - "description": "Total number of impressions from paid sources.", - "type": ["null", "number"] - }, - "earned_impressions": { - "description": "Imprressions earned through engagements.", - "type": ["null", "number"] - }, - "total_impressions": { - "description": "Total number of impressions including earned and paid.", - "type": ["null", "number"] - }, - "play_time_millis": { - "description": "Total time in milliseconds users spend playing content.", - "type": ["null", "number"] - }, - "shares": { - "description": "Total number of content shares.", - "type": ["null", "number"] - }, - "saves": { - "description": "Number of saves performed.", - "type": ["null", "number"] - }, - "native_leads": { - "description": "Number of leads generated natively.", - "type": ["null", "number"] - }, - "conversion_purchases": { - "description": "Number of total purchases made.", - "type": ["null", "number"] - }, - "conversion_purchases_value": { - "description": "Total value of purchases made.", - "type": ["null", "number"] - }, - "conversion_save": { - "description": "Number of saves performed.", - "type": ["null", "number"] - }, - "conversion_start_checkout": { - "description": "Number of times checkout process is initiated.", - "type": ["null", "number"] - }, - "conversion_add_cart": { - "description": "Conversion events where items are added to cart.", - "type": ["null", "number"] - }, - "conversion_view_content": { - "description": "Number of content views.", - "type": ["null", "number"] - }, - "conversion_add_billing": { - "description": "Conversion events where billing information is added.", - "type": ["null", "number"] - }, - "conversion_searches": { - "description": "Number of search queries made.", - "type": ["null", "number"] - }, - "conversion_level_completes": { - "description": "Number of game levels completed.", - "type": ["null", "number"] - }, - "conversion_app_opens": { - "description": "Number of times the app is opened.", - "type": ["null", "number"] - }, - "conversion_page_views": { - "description": "Total number of page views.", - "type": ["null", "number"] - }, - "conversion_subscribe": { - "description": "Number of subscriptions made.", - "type": ["null", "number"] - }, - "conversion_ad_click": { - "description": "Number of ad clicks recorded.", - "type": ["null", "number"] - }, - "conversion_ad_view": { - "description": "Number of ad views.", - "type": ["null", "number"] - }, - "conversion_complete_tutorial": { - "description": "Number of completed tutorials.", - "type": ["null", "number"] - }, - "conversion_invite": { - "description": "Number of invitations sent by users.", - "type": ["null", "number"] - }, - "conversion_login": { - "description": "Number of logins recorded.", - "type": ["null", "number"] - }, - "conversion_share": { - "description": "Number of shares made.", - "type": ["null", "number"] - }, - "conversion_reserve": { - "description": "Reserve conversion events.", - "type": ["null", "number"] - }, - "conversion_achievement_unlocked": { - "description": "Number of times achievements are unlocked.", - "type": ["null", "number"] - }, - "conversion_add_to_wishlist": { - "description": "Conversion events where items are added to the wishlist.", - "type": ["null", "number"] - }, - "conversion_spend_credits": { - "description": "Total credits spent during conversion events.", - "type": ["null", "number"] - }, - "conversion_rate": { - "description": "Rate of conversion events.", - "type": ["null", "number"] - }, - "conversion_start_trial": { - "description": "Number of trials started.", - "type": ["null", "number"] - }, - "conversion_list_view": { - "description": "Number of views on the list content.", - "type": ["null", "number"] - }, - "custom_event_1": { - "description": "Custom event tracking 1.", - "type": ["null", "number"] - }, - "custom_event_2": { - "description": "Custom event tracking 2.", - "type": ["null", "number"] - }, - "custom_event_3": { - "description": "Custom event tracking 3.", - "type": ["null", "number"] - }, - "custom_event_4": { - "description": "Custom event tracking 4.", - "type": ["null", "number"] - }, - "custom_event_5": { - "description": "Custom event tracking 5.", - "type": ["null", "number"] - }, - "attachment_frequency": { - "description": "Frequency of attachment views.", - "type": ["null", "number"] - }, - "attachment_uniques": { - "description": "Unique viewers of attachments.", - "type": ["null", "number"] - }, - "frequency": { - "description": "Frequency of events tracked.", - "type": ["null", "number"] - }, - "uniques": { - "description": "Total unique events or users.", - "type": ["null", "number"] - }, - "total_reach": { - "description": "Total reach including earned and paid.", - "type": ["null", "number"] - }, - "earned_reach": { - "description": "Total reach earned through engagements.", - "type": ["null", "number"] - }, - "story_opens": { - "description": "Total number of story opens.", - "type": ["null", "number"] - }, - "story_completes": { - "description": "Total number of completed stories.", - "type": ["null", "number"] - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/campaigns.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/campaigns.json deleted file mode 100644 index 3fd986eb77c8..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/campaigns.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "The unique identifier of the campaign.", - "type": ["null", "string"] - }, - "updated_at": { - "description": "The timestamp indicating when the campaign was last updated.", - "type": ["null", "string"] - }, - "created_at": { - "description": "The timestamp indicating when the campaign was created.", - "type": ["null", "string"] - }, - "name": { - "description": "The name assigned to the campaign for easy identification.", - "type": ["null", "string"] - }, - "ad_account_id": { - "description": "The unique identifier of the advertising account associated with the campaign.", - "type": ["null", "string"] - }, - "status": { - "description": "The current status of the campaign (e.g., paused, active).", - "type": ["null", "string"] - }, - "objective": { - "description": "The primary goal or objective of the campaign.", - "type": ["null", "string"] - }, - "start_time": { - "description": "The starting date and time for the campaign to begin running.", - "type": ["null", "string"] - }, - "buy_model": { - "description": "The purchasing model used for the campaign (e.g., auction, reach and frequency).", - "type": ["null", "string"] - }, - "delivery_status": { - "description": "The status of campaign delivery.", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "creation_state": { - "description": "The current state of the campaign creation process.", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/creatives.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/creatives.json deleted file mode 100644 index 90084599c23b..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/creatives.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "The unique identifier of the creative", - "type": ["null", "string"] - }, - "updated_at": { - "description": "The timestamp indicating when the creative was last updated", - "type": ["null", "string"] - }, - "created_at": { - "description": "The timestamp indicating when the creative was created", - "type": ["null", "string"] - }, - "name": { - "description": "The name or title given to the creative", - "type": ["null", "string"] - }, - "ad_account_id": { - "description": "The ID of the advertising account linked to the creative", - "type": ["null", "string"] - }, - "type": { - "description": "The type or category of the creative", - "type": ["null", "string"] - }, - "packaging_status": { - "description": "The status of the packaging for the creative", - "type": ["null", "string"] - }, - "review_status": { - "description": "The overall review status of the creative", - "type": ["null", "string"] - }, - "review_status_details": { - "description": "Additional details related to the review status", - "type": ["null", "string"] - }, - "shareable": { - "description": "Indicates if the creative is shareable", - "type": ["null", "boolean"] - }, - "forced_view_eligibility": { - "description": "Whether the creative is eligible for forced views", - "type": ["null", "string"] - }, - "headline": { - "description": "The headline or title of the creative", - "type": ["null", "string"] - }, - "brand_name": { - "description": "The name of the brand associated with the creative", - "type": ["null", "string"] - }, - "call_to_action": { - "description": "The call to action prompt for the creative", - "type": ["null", "string"] - }, - "render_type": { - "description": "The type of rendering used for the creative", - "type": ["null", "string"] - }, - "top_snap_media_id": { - "description": "The ID of the media file for the top snap", - "type": ["null", "string"] - }, - "top_snap_crop_position": { - "description": "The crop position for the top snap of the creative", - "type": ["null", "string"] - }, - "web_view_properties": { - "description": "Properties related to the web view functionality when a user interacts with the ad.", - "type": ["null", "object"], - "properties": { - "url": { - "description": "The URL to be loaded in the web view", - "type": ["null", "string"] - }, - "allow_snap_javascript_sdk": { - "description": "Whether to allow the use of Snap JavaScript SDK in the web view", - "type": ["null", "boolean"] - }, - "use_immersive_mode": { - "description": "Whether to use immersive mode in the web view", - "type": ["null", "boolean"] - }, - "deep_link_urls": { - "description": "List of deep link URLs associated with the web view", - "type": ["null", "array"] - }, - "block_preload": { - "description": "Indicates if preloading is blocked for the web view", - "type": ["null", "boolean"] - } - } - }, - "ad_product": { - "description": "The type of Snapchat advertising product used for the creative", - "type": ["null", "string"] - }, - "ad_to_place_properties": { - "description": "Properties related to the ad to be displayed within the Snapchat interface.", - "type": ["null", "object"], - "properties": { - "place_id": { - "description": "The ID of the place where the ad is to be displayed", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/media.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/media.json deleted file mode 100644 index 1e247cd5c063..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/media.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "The unique identifier of the media.", - "type": ["null", "string"] - }, - "updated_at": { - "description": "The timestamp when the media was last updated.", - "type": ["null", "string"] - }, - "created_at": { - "description": "The timestamp when the media was created.", - "type": ["null", "string"] - }, - "ad_account_id": { - "description": "The unique identifier of the ad account associated with the media.", - "type": ["null", "string"] - }, - "type": { - "description": "The type of media (e.g., image, video).", - "type": ["null", "string"] - }, - "media_status": { - "description": "The status of the media (e.g., active, inactive).", - "type": ["null", "string"] - }, - "media_usages": { - "description": "Items associated with the media usage.", - "type": ["null", "array"], - "items": { - "description": "Details of the media usage.", - "type": ["null", "string"] - } - }, - "file_name": { - "description": "The name of the media file.", - "type": ["null", "string"] - }, - "download_link": { - "description": "The link to download the media file.", - "type": ["null", "string"] - }, - "duration_in_seconds": { - "description": "The duration of the media in seconds.", - "type": ["null", "number"], - "multipleOf": 1e-10 - }, - "image_metadata": { - "description": "Additional metadata related to the image media.", - "type": ["null", "object"] - }, - "video_metadata": { - "description": "Additional metadata related to the video media.", - "type": ["null", "object"] - }, - "file_size_in_bytes": { - "description": "The file size of the media in bytes.", - "type": ["null", "integer"] - }, - "is_demo_media": { - "description": "Indicates if the media is a demo or not.", - "type": ["null", "boolean"] - }, - "hash": { - "description": "The hash value of the media file for identification purposes.", - "type": ["null", "string"] - }, - "visibility": { - "description": "The visibility settings of the media (e.g., public, private).", - "type": ["null", "string"] - }, - "name": { - "description": "The name or title of the media.", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/organizations.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/organizations.json deleted file mode 100644 index 17ff5e6cba0a..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/organizations.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "Unique identifier for the organization record.", - "type": ["null", "string"] - }, - "updated_at": { - "description": "Timestamp indicating the last update date of the organization record.", - "type": ["null", "string"] - }, - "created_at": { - "description": "Timestamp indicating the creation date of the organization record.", - "type": ["null", "string"] - }, - "name": { - "description": "The name of the organization.", - "type": ["null", "string"] - }, - "country": { - "description": "The country where the organization is located.", - "type": ["null", "string"] - }, - "postal_code": { - "description": "The postal code of the organization's location.", - "type": ["null", "string"] - }, - "locality": { - "description": "The locality or city where the organization is situated.", - "type": ["null", "string"] - }, - "contact_name": { - "description": "The name of the contact person within the organization.", - "type": ["null", "string"] - }, - "contact_email": { - "description": "The email address used for contact purposes.", - "type": ["null", "string"] - }, - "contact_phone": { - "description": "The phone number of the contact person within the organization.", - "type": ["null", "string"] - }, - "address_line_1": { - "description": "The first line of the organization's address.", - "type": ["null", "string"] - }, - "administrative_district_level_1": { - "description": "The first-level administrative district of the organization's location.", - "type": ["null", "string"] - }, - "accepted_term_version": { - "description": "The version of the terms and conditions accepted by the organization.", - "type": ["null", "string"] - }, - "contact_phone_optin": { - "description": "Flag indicating if contact person has opted in for phone contact.", - "type": ["null", "boolean"] - }, - "configuration_settings": { - "description": "Settings related to organization configurations", - "type": ["null", "object"], - "properties": { - "notifications_enabled": { - "description": "Flag indicating if notifications are enabled for the organization.", - "type": ["null", "boolean"] - } - } - }, - "type": { - "description": "The type or category of the organization.", - "type": ["null", "string"] - }, - "state": { - "description": "The state or region where the organization is located.", - "type": ["null", "string"] - }, - "roles": { - "description": "List of roles assigned to the organization.", - "type": ["null", "array"], - "items": { - "description": "A specific role assigned to the organization.", - "type": ["null", "string"] - } - }, - "my_display_name": { - "description": "The public display name of the organization.", - "type": ["null", "string"] - }, - "my_invited_email": { - "description": "Email address of the member invited to the organization.", - "type": ["null", "string"] - }, - "my_member_id": { - "description": "Unique identifier of the member within the organization.", - "type": ["null", "string"] - }, - "createdByCaller": { - "description": "Record of the creator of the organization within the system.", - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/segments.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/segments.json deleted file mode 100644 index 9f897da87e76..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/schemas/segments.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "id": { - "description": "Unique identifier for the segment", - "type": ["null", "string"] - }, - "updated_at": { - "description": "The date and time when the segment was last updated", - "type": ["null", "string"] - }, - "created_at": { - "description": "The date and time when the segment was created", - "type": ["null", "string"] - }, - "name": { - "description": "Name or title of the segment", - "type": ["null", "string"] - }, - "ad_account_id": { - "description": "The unique identifier for the ad account associated with the segment", - "type": ["null", "string"] - }, - "organization_id": { - "description": "Unique identifier for the organization to which the segment belongs", - "type": ["null", "string"] - }, - "description": { - "description": "Brief description of the segment", - "type": ["null", "string"] - }, - "status": { - "description": "Current status of the segment (e.g., active, inactive)", - "type": ["null", "string"] - }, - "targetable_status": { - "description": "Status indicating whether the segment can be targeted in marketing campaigns", - "type": ["null", "string"] - }, - "upload_status": { - "description": "Status of the segment upload process (e.g., pending, completed)", - "type": ["null", "string"] - }, - "source_type": { - "description": "Type of the source data used to create the segment", - "type": ["null", "string"] - }, - "retention_in_days": { - "description": "Number of days for which the segment data is retained", - "type": ["null", "integer"] - }, - "approximate_number_users": { - "description": "Approximate number of users within the segment", - "type": ["null", "integer"] - }, - "visible_to": { - "description": "Visibility setting for the segment, determining who can access it", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py index 1b2d79253ef5..5ce78c1e5785 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/source.py @@ -2,826 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from abc import ABC, abstractmethod -from enum import Enum -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qsl, urlparse +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import backoff -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.core import IncrementalMixin, package_name_from_class -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -# https://marketingapi.snapchat.com/docs/#core-metrics -# https://marketingapi.snapchat.com/docs/#metrics-and-supported-granularities -METRICS = [ - "android_installs", - "attachment_avg_view_time_millis", - "attachment_impressions", - "attachment_quartile_1", - "attachment_quartile_2", - "attachment_quartile_3", - "attachment_total_view_time_millis", - "attachment_view_completion", - "avg_screen_time_millis", - "avg_view_time_millis", - "impressions", - "ios_installs", - "quartile_1", - "quartile_2", - "quartile_3", - "screen_time_millis", - "spend", - "swipe_up_percent", - "swipes", - "total_installs", - "video_views", - "video_views_time_based", - "video_views_15s", - "view_completion", - "view_time_millis", - "paid_impressions", - "earned_impressions", - "total_impressions", - "play_time_millis", - "shares", - "saves", - "native_leads", - "conversion_purchases", - "conversion_purchases_value", - "conversion_save", - "conversion_start_checkout", - "conversion_add_cart", - "conversion_view_content", - "conversion_add_billing", - # 'conversion_signups', # Invalid query parameters in request URL - "conversion_searches", - "conversion_level_completes", - "conversion_app_opens", - "conversion_page_views", - "conversion_subscribe", - "conversion_ad_click", - "conversion_ad_view", - "conversion_complete_tutorial", - "conversion_invite", - "conversion_login", - "conversion_share", - "conversion_reserve", - "conversion_achievement_unlocked", - "conversion_add_to_wishlist", - "conversion_spend_credits", - "conversion_rate", - "conversion_start_trial", - "conversion_list_view", - "custom_event_1", - "custom_event_2", - "custom_event_3", - "custom_event_4", - "custom_event_5", - "story_opens", - "story_completes", -] +WARNING: Do not modify this file. +""" -METRICS_NOT_HOURLY = [ - "attachment_frequency", - "attachment_uniques", - "frequency", - "uniques", - "total_reach", - "earned_reach", -] -logger = logging.getLogger("airbyte") - - -class GranularityType(Enum): - HOUR = "HOUR" - DAY = "DAY" - LIFETIME = "LIFETIME" - - -# See the get_parent_ids function explanation -# Long story short - used as cache for ids of higher level streams, that is used -# as path variables in other streams. And to avoid requesting those ids for every stream -# we put them to a dict and check if they exist on stream calling -# The structure be like: -# { -# 'Organizations': [ -# {'organization_id': '7f064d90-52a1-some-uuid'} -# ], -# 'Adaccounts': [ -# {'ad_account_id': '04214c00-3aa5-some-uuid'}, -# {'ad_account_id': 'e4cd371b-8de8-some-uuid'} -# ] -# } -# -auxiliary_id_map = {} - - -class SnapchatMarketingException(Exception): - """Just for formatting the exception as SnapchatMarketing""" - - -def get_parent_ids(parent) -> List: - """Return record ids from stream full_refresh sync, example: - [{'id': record_1['id']}, {'id': record_2['id']}, ... {'id': record_N['id']}] - - Function uses global cache 'auxiliary_id_map' to save results of parent syncs. - It prevents multiple actual syncs of the same basic streams like organizations, adacounts etc - - :param parent: instance of stream class from what we need to retrieve ids - :returns: empty list in case no ids of the stream was found or list with {'id': id} - """ - - # return cached data if it has been already extracted - if parent.name in auxiliary_id_map: - return auxiliary_id_map[parent.name] - - parent_ids = [] - - # Run actual sync for parent stream - parent_stream_slices = parent.stream_slices(sync_mode=SyncMode.full_refresh) - for parent_stream_slice in parent_stream_slices: - records = parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=parent_stream_slice) - for record in records: - parent_ids.append({"id": record["id"]}) - - # add data to cache to prevent further actual syncs of the same streams. (they always return the same data) - auxiliary_id_map[parent.name] = parent_ids - - return parent_ids - - -class SnapchatMarketingStream(HttpStream, ABC): - url_base = "https://adsapi.snapchat.com/v1/" - primary_key = "id" - raise_on_http_errors = True - - def __init__(self, start_date, end_date, action_report_time, swipe_up_attribution_window, view_attribution_window, **kwargs): - super().__init__(**kwargs) - self.start_date = start_date - self.end_date = end_date - self.action_report_time = action_report_time - self.swipe_up_attribution_window = swipe_up_attribution_window - self.view_attribution_window = view_attribution_window - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - next_page_cursor = response.json().get("paging", False) - if next_page_cursor: - return {"cursor": dict(parse_qsl(urlparse(next_page_cursor["next_link"]).query))["cursor"]} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return next_page_token or {} - - @property - def response_root_name(self): - """Using the class name in lower to set the root node for response parsing""" - return self.name - - @property - def response_item_name(self): - """Remove last 's' from response_root_name, see example in parse_response function""" - return self.response_root_name[:-1] - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """Response json came like - { - "organizations": [ <-- response_root_name - { - "organization": { <-- response_item_name - "id": "some uuid", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", - ... some_other_json_fields ... - } - } - ] - } - So the response_root_name will be "organizations", and the response_item_name will be "organization" - Also, the client side filtering for incremental sync is used - """ - - json_response = response.json().get(self.response_root_name, []) - for resp in json_response: - if self.response_item_name not in resp: - error_text = f"stream {self.name}: field named '{self.response_item_name}' is absent in the response: {resp}" - self.logger.error(error_text) - raise SnapchatMarketingException(error_text) - yield resp.get(self.response_item_name) - - def should_retry(self, response: requests.Response) -> bool: - if response.status_code == 403: - setattr(self, "raise_on_http_errors", False) - self.logger.warning(f"Got permission error when accessing URL {response.request.url}. " f"Skipping {self.name} stream.") - return False - return super().should_retry(response) - - -class IncrementalSnapchatMarketingStream(SnapchatMarketingStream, ABC): - cursor_field = "updated_at" - - last_slice = None - current_slice = None - initial_state = None - max_state = None - - @property - @abstractmethod - def parent(self) -> SnapchatMarketingStream: - """Stream Class to extract entity ids from""" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"adaccounts/{stream_slice['id']}/{self.response_root_name}" - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - - stream_state = kwargs.get("stream_state") - self.initial_state = stream_state.get(self.cursor_field) if stream_state else self.start_date - self.max_state = self.initial_state - - parent_stream = self.parent( - authenticator=self._session.auth, - start_date=self.start_date, - end_date=self.end_date, - action_report_time=self.action_report_time, - swipe_up_attribution_window=self.swipe_up_attribution_window, - view_attribution_window=self.view_attribution_window, - ) - stream_slices = get_parent_ids(parent_stream) - - if stream_slices: - self.last_slice = stream_slices[-1] - - self.logger.info(f"{self.name} stream slices: {stream_slices}") - - return stream_slices - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Handle state value for streams with random order of cursor_field values. - State with max value should be release only in last slice otherwise records - from intermediate slices can be filtered out in 'read_records' - """ - - # store max state value across all records of all slices (parent ids) because - # cursor_field values are random (not sorted) even within one slice (parent id) - self.max_state = max(self.max_state, latest_record[self.cursor_field]) - - if self.current_slice == self.last_slice: - # return max found state for last slice only - return {self.cursor_field: self.max_state} - else: - # return release initial state until last slice is processed - return {self.cursor_field: self.initial_state} - - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - """ - This structure is used to set the class variable current_slice to the current stream slice for the - purposes described above. - Then all the retrieved records if the stream_state is present are filtered by the cursor_field value compared to the stream state - This makes the incremental magic works - """ - self.current_slice = stream_slice - records = super().read_records(stream_slice=stream_slice, stream_state=stream_state, **kwargs) - if stream_state: - for record in records: - if record[self.cursor_field] > stream_state.get(self.cursor_field): - yield record - else: - yield from records - - -class Organizations(SnapchatMarketingStream): - """Docs: https://marketingapi.snapchat.com/docs/#organizations""" - - def path(self, **kwargs) -> str: - return "me/organizations" - - -class Adaccounts(IncrementalSnapchatMarketingStream): - """Docs: https://marketingapi.snapchat.com/docs/#get-all-ad-accounts""" - - parent = Organizations - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"organizations/{stream_slice['id']}/adaccounts" - - -class Creatives(IncrementalSnapchatMarketingStream): - """Docs: https://marketingapi.snapchat.com/docs/#get-all-creatives""" - - parent = Adaccounts - - -class Media(IncrementalSnapchatMarketingStream): - """Docs: https://marketingapi.snapchat.com/docs/#get-all-media""" - - parent = Adaccounts - - @property - def response_item_name(self): - return self.response_root_name - - -class Campaigns(IncrementalSnapchatMarketingStream): - """Docs: https://marketingapi.snapchat.com/docs/#get-all-campaigns""" - - parent = Adaccounts - - -class Ads(IncrementalSnapchatMarketingStream): - """Docs: https://marketingapi.snapchat.com/docs/#get-all-ads-under-an-ad-account""" - - parent = Adaccounts - - -class Adsquads(IncrementalSnapchatMarketingStream): - """Docs: https://marketingapi.snapchat.com/docs/#get-all-ad-squads-under-an-ad-account""" - - parent = Adaccounts - - -class Segments(IncrementalSnapchatMarketingStream): - """Docs: https://marketingapi.snapchat.com/docs/#get-all-audience-segments""" - - parent = Adaccounts - - -class Stats(SnapchatMarketingStream, ABC): - """Stats streams for LIFETIME granularity. - Docs: https://marketingapi.snapchat.com/docs/#measurement - """ - - primary_key = ["id", "granularity"] - schema_name = "basic_stats" - parent_name: str = "" # name of parent class - response_root_name = "lifetime_stats" - granularity = GranularityType.LIFETIME - - @property - @abstractmethod - def parent(self) -> SnapchatMarketingStream: - """Stream Class to extract entity ids from""" - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """Each stream slice represents each entity id from parent stream""" - - parent_stream = self.parent( - authenticator=self._session.auth, - start_date=self.start_date, - end_date=self.end_date, - action_report_time=self.action_report_time, - swipe_up_attribution_window=self.swipe_up_attribution_window, - view_attribution_window=self.view_attribution_window, - ) - self.parent_name = parent_stream.name - stream_slices = get_parent_ids(parent_stream) - - if not stream_slices: - self.logger.error(f"No {'id'}s found. {self.name} cannot be extracted without {'id'}.") - - self.logger.info(f"Stats: stream_slices:{stream_slices}") - - return stream_slices - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"{self.parent_name}/{stream_slice['id']}/stats" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - params["granularity"] = self.granularity.value - params["action_report_time"] = self.action_report_time - params["swipe_up_attribution_window"] = self.swipe_up_attribution_window - params["view_attribution_window"] = self.view_attribution_window - if self.metrics: - params["fields"] = ",".join(self.metrics) - - return params - - def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - """Customized by adding stream state setting""" - - for record in super().parse_response( - response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token - ): - # move all 'stats' metrics to root level - record.update(record.pop("stats", {})) - - yield record - - def get_json_schema(self) -> Mapping[str, Any]: - """All stats streams have same schema""" - return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema(self.schema_name) - - -class StatsIncremental(Stats, IncrementalMixin): - """Incremental Stats class for Daily and Hourly streams which requires start/end date param""" - - primary_key = ["id", "granularity", "start_time"] - cursor_field: str = "start_time" - slice_period: int = 7 # days https://marketingapi.snapchat.com/docs/#metrics-and-supported-granularities - number_of_parent_ids: int = 0 - number_of_last_records: int = 0 - response_root_name = "timeseries_stats" - response_subitem_name = "timeseries" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self._state = {} - - def date_slices(self, stream_state=None) -> Iterable[Optional[Mapping[str, Any]]]: - date_slices = [] - # use start_date from state or from config - start_date_str = stream_state.get(self.cursor_field) if stream_state else self.start_date - slice_start_date = pendulum.parse(start_date_str) - end_date = pendulum.parse(self.end_date) - - while slice_start_date < end_date: - slice_end_date_next = slice_start_date + pendulum.duration(days=self.slice_period) - slice_end_date = min(slice_end_date_next, end_date) - date_slices.append({"start_time": slice_start_date.to_date_string(), "end_time": slice_end_date.to_date_string()}) - slice_start_date = slice_end_date - - self.logger.info(f"{self.name} stream date slices: {date_slices}.") - - return date_slices - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """Incremental stream slices is a combinations of date and parent id slices""" - - stream_slices = [] - - parent_ids = super().stream_slices() - - # save a number of parent ids we need to get stats for - self.number_of_parent_ids = len(parent_ids) - - # within each date period extract data for all parent_ids - # it allows incremental sync implementation with slice period granularity - for date_slice in self.date_slices(stream_state=stream_state): - for parent_id in parent_ids: - stream_slices.append({**date_slice, **parent_id}) - - self.logger.info(f"{self.name} stream slices: {stream_slices}") - - return stream_slices - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """start/end date param should be set for Daily and Hourly streams""" - - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - params["start_time"] = stream_slice["start_time"] - params["end_time"] = stream_slice["end_time"] - return params - - def update_state_after_last_record(self, record): - """Update state if last record has been read""" - record_end_date = record.get("end_time", "").split("T")[0] # "2022-07-06T00:00:00.000-07:00" --> "2022-07-06" - if record_end_date == self.end_date: - self.number_of_last_records += 1 - # Update state if 'last' records for all dependant entities have been read - if self.number_of_parent_ids == self.number_of_last_records: - self.state = {self.cursor_field: record_end_date} - - @property - def state(self): - return self._state - - @state.setter - def state(self, value): - self._state = value - - def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - """Customized by adding stream state setting""" - - # Update state for each date slice (start_date), it ensures that previous date slices have been read - # and can be skipped in next incremental sync - self.state = {self.cursor_field: stream_slice[self.cursor_field]} - - for record in super().parse_response( - response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token - ): - - record_identifiers = { - "id": record["id"], - "type": record["type"], - "granularity": record["granularity"], - } - - # Hourly/Daily stats contains nested structure (under self.response_subitem_name) with actual stats items - for stat_item in record.get(self.response_subitem_name, []): - # add common record identifiers - stat_item.update(record_identifiers) - # move all 'stats' metrics to root level - stat_item.update(stat_item.pop("stats", {})) - - # Update state for last record in the stream - self.update_state_after_last_record(stat_item) - - yield stat_item - - -class Granularity: - granularity: GranularityType - metrics = METRICS + METRICS_NOT_HOURLY - - -class Lifetime(Granularity): - """Example of raw response: - { - "request_status": "SUCCESS", - "request_id": "d0cb395f-c39d-480d-b62c-24878c7d0b76", - "lifetime_stats": [{ <-- response_root_name - "sub_request_status": "SUCCESS", - "lifetime_stat": { <-- response_item_name - "id": "96e5549f-4065-490e-93dd-ffb9f7973b77", - "type": "AD", - "granularity": "LIFETIME", - "stats": { - "impressions": 0, - "swipes": 0, - "quartile_1": 0, - "quartile_2": 0, - }, - "start_time": "2016-09-26T00:00:00.000-07:00", - "end_time": "2022-07-01T07:00:00.000-07:00", - "finalized_data_end_time": "2022-07-01T07:00:00.000-07:00", - "conversion_data_processed_end_time": "2022-07-01T00:00:00.000Z" - } - } - ] - } - """ - - granularity = GranularityType.LIFETIME - - -class Hourly(Granularity): - """Example of raw response: - { - "request_status": "SUCCESS", - "request_id": "c106d757-7c07-4415-b62f-78108d9dc668", - "timeseries_stats": [{ <-- response_root_name - "sub_request_status": "SUCCESS", - "timeseries_stat": { <-- response_item_name - "id": "417d0269-80fb-496a-b5f3-ec0bac665144", - "type": "AD", - "granularity": "HOUR", - "start_time": "2022-06-24T17:00:00.000-07:00", - "end_time": "2022-06-28T17:00:00.000-07:00", - "finalized_data_end_time": "2022-06-30T11:00:00.000-07:00", - "conversion_data_processed_end_time": "2022-06-30T00:00:00.000Z", - "timeseries": [{ <-- response_subitem_name - "start_time": "2022-06-24T17:00:00.000-07:00", - "end_time": "2022-06-24T18:00:00.000-07:00", - "stats": { - "impressions": 0, - "swipes": 0, - "quartile_1": 0, - "quartile_2": 0, - "quartile_3": 0, - "view_completion": 0, - } - }, { - "start_time": "2022-06-24T18:00:00.000-07:00", - "end_time": "2022-06-24T19:00:00.000-07:00", - "stats": { - "impressions": 0, - "swipes": 0, - } - } - ] - } - } - ] - } - """ - - granularity = GranularityType.HOUR - metrics = METRICS - slice_period = 7 # days https://marketingapi.snapchat.com/docs/#metrics-and-supported-granularities - - -class Daily(Granularity): - """Example of raw response: - { - "request_status": "SUCCESS", - "request_id": "f2cba857-e246-43bf-b644-1a0a540e1f92", - "timeseries_stats": [{ <-- response_root_name - "sub_request_status": "SUCCESS", - "timeseries_stat": { <-- response_item_name - "id": "417d0269-80fb-496a-b5f3-ec0bac665144", - "type": "AD", - "granularity": "DAY", - "start_time": "2022-06-25T00:00:00.000-07:00", - "end_time": "2022-06-29T00:00:00.000-07:00", - "finalized_data_end_time": "2022-06-30T00:00:00.000-07:00", - "conversion_data_processed_end_time": "2022-06-30T00:00:00.000Z", - "timeseries": [{ <-- response_subitem_name - "start_time": "2022-06-25T00:00:00.000-07:00", - "end_time": "2022-06-26T00:00:00.000-07:00", - "stats": { - "impressions": 0, - "swipes": 0, - "quartile_1": 0, - "quartile_2": 0, - "quartile_3": 0, - } - }, { - "start_time": "2022-06-26T00:00:00.000-07:00", - "end_time": "2022-06-27T00:00:00.000-07:00", - "stats": { - "impressions": 0, - "swipes": 0, - "quartile_1": 0, - "quartile_2": 0, - "quartile_3": 0, - }, - }, - - ] - } - } - ] - } - - """ - - granularity = GranularityType.DAY - slice_period = 31 # days https://marketingapi.snapchat.com/docs/#metrics-and-supported-granularities - - -class AdaccountsStatsHourly(Hourly, StatsIncremental): - """Adaccounts stats with Hourly granularity: https://marketingapi.snapchat.com/docs/#get-ad-account-stats""" - - parent = Adaccounts - metrics = ["spend"] - - -class AdaccountsStatsDaily(Daily, StatsIncremental): - """Adaccounts stats with Daily granularity: https://marketingapi.snapchat.com/docs/#get-ad-account-stats""" - - parent = Adaccounts - metrics = ["spend"] - - -class AdaccountsStatsLifetime(Lifetime, Stats): - """Adaccounts stats with Lifetime granularity: https://marketingapi.snapchat.com/docs/#get-ad-account-stats""" - - parent = Adaccounts - metrics = ["spend"] - - -class AdsStatsHourly(Hourly, StatsIncremental): - """Ads stats with Hourly granularity: https://marketingapi.snapchat.com/docs/#get-ad-stats""" - - parent = Ads - - -class AdsStatsDaily(Daily, StatsIncremental): - """Ads stats with Daily granularity: https://marketingapi.snapchat.com/docs/#get-ad-stats""" - - parent = Ads - - -class AdsStatsLifetime(Lifetime, Stats): - """Ads stats with Lifetime granularity: https://marketingapi.snapchat.com/docs/#get-ad-stats""" - - parent = Ads - - -class AdsquadsStatsHourly(Hourly, StatsIncremental): - """Adsquads stats with Hourly granularity: https://marketingapi.snapchat.com/docs/#get-ad-squad-stats""" - - parent = Adsquads - - -class AdsquadsStatsDaily(Daily, StatsIncremental): - """Adsquads stats with Daily granularity: https://marketingapi.snapchat.com/docs/#get-ad-squad-stats""" - - parent = Adsquads - - -class AdsquadsStatsLifetime(Lifetime, Stats): - """Adsquads stats with Lifetime granularity: https://marketingapi.snapchat.com/docs/#get-ad-squad-stats""" - - parent = Adsquads - - -class CampaignsStatsHourly(Hourly, StatsIncremental): - """Campaigns stats with Hourly granularity: https://marketingapi.snapchat.com/docs/#get-campaign-stats""" - - parent = Campaigns - - -class CampaignsStatsDaily(Daily, StatsIncremental): - """Campaigns stats with Daily granularity: https://marketingapi.snapchat.com/docs/#get-campaign-stats""" - - parent = Campaigns - - -class CampaignsStatsLifetime(Lifetime, Stats): - """Campaigns stats with Lifetime granularity: https://marketingapi.snapchat.com/docs/#get-campaign-stats""" - - parent = Campaigns - - -# Source -class SourceSnapchatMarketing(AbstractSource): - """Source Snapchat Marketing helps to retrieve the different Ad data from Snapchat business account""" - - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - auth = Oauth2Authenticator( - token_refresh_endpoint="https://accounts.snapchat.com/login/oauth2/access_token", - client_id=config["client_id"], - client_secret=config["client_secret"], - refresh_token=config["refresh_token"], - ) - token = auth.get_access_token() - url = f"{SnapchatMarketingStream.url_base}me" - - session = requests.get(url, headers={"Authorization": "Bearer {}".format(token)}) - session.raise_for_status() - return True, None - - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - # https://marketingapi.snapchat.com/docs/#core-metrics - # IMPORTANT: Metrics are finalized 48 hours after the end of the day in the Ad Account’s timezone. - DELAYED_DAYS = 2 - - # Start/End dates should better be in YYYY-MM-DD format: - # 1. minutes are not supported - # 2. when timezone is not specified, default account's timezone will be used automatically - default_end_date = pendulum.now().subtract(days=DELAYED_DAYS).to_date_string() - kwargs = { - "authenticator": Oauth2Authenticator( - token_refresh_endpoint="https://accounts.snapchat.com/login/oauth2/access_token", - client_id=config["client_id"], - client_secret=config["client_secret"], - refresh_token=config["refresh_token"], - ), - "start_date": config["start_date"], - "end_date": config.get("end_date", default_end_date), - "action_report_time": config.get("action_report_time", "conversion"), - "swipe_up_attribution_window": config.get("swipe_up_attribution_window", "28_DAY"), - "view_attribution_window": config.get("view_attribution_window", "1_DAY"), - } - - return [ - # Base streams: - Adaccounts(**kwargs), - Ads(**kwargs), - Adsquads(**kwargs), - Campaigns(**kwargs), - Creatives(**kwargs), - Media(**kwargs), - Organizations(**kwargs), - Segments(**kwargs), - # Stats streams: - AdaccountsStatsHourly(**kwargs), - AdaccountsStatsDaily(**kwargs), - AdaccountsStatsLifetime(**kwargs), - AdsStatsHourly(**kwargs), - AdsStatsDaily(**kwargs), - AdsStatsLifetime(**kwargs), - AdsquadsStatsHourly(**kwargs), - AdsquadsStatsDaily(**kwargs), - AdsquadsStatsLifetime(**kwargs), - CampaignsStatsHourly(**kwargs), - CampaignsStatsDaily(**kwargs), - CampaignsStatsLifetime(**kwargs), - ] +# Declarative Source +class SourceSnapchatMarketing(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/conftest.py new file mode 100644 index 000000000000..bbbf8b4c6df4 --- /dev/null +++ b/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/conftest.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + +from source_snapchat_marketing.source import SourceSnapchatMarketing + + +def find_stream(stream_name, config): + streams = SourceSnapchatMarketing().streams(config=config) + + # cache should be disabled once this issue is fixed https://github.com/airbytehq/airbyte-internal-issues/issues/6513 + for stream in streams: + stream.retriever.requester.use_cache = True + + # find by name + for stream in streams: + if stream.name == stream_name: + return stream + raise ValueError(f"Stream {stream_name} not found") diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py index 44ee246a5277..146ce25294b2 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py @@ -1,105 +1,63 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # -# from unittest.mock import MagicMock, PropertyMock, patch +import json +import logging import pytest -import source_snapchat_marketing from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.auth import NoAuth -from source_snapchat_marketing.source import ( - Adaccounts, - AdaccountsStatsDaily, - Ads, - AdsStatsDaily, - AdsStatsLifetime, - Oauth2Authenticator, - Organizations, - SourceSnapchatMarketing, -) +from airbyte_cdk.sources.declarative.types import StreamSlice +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException +from conftest import find_stream +from source_snapchat_marketing.source import SourceSnapchatMarketing + +logger = logging.getLogger("airbyte_logger") config_mock = { - "authenticator": NoAuth(), - "start_date": "2000-01-01", - "end_date": "2000-02-10", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + "start_date": "2024-01-01", + "end_date": "2024-02-10", "action_report_time": "impression", "swipe_up_attribution_window": "7_DAY", "view_attribution_window": "1_DAY", } -stats_stream = AdaccountsStatsDaily(**config_mock) - - -@pytest.mark.parametrize( - "slice_period,expected_date_slices", - [ - # for Hourly streams - ( - 7, - [ - {"start_time": "2000-01-01", "end_time": "2000-01-08"}, - {"start_time": "2000-01-08", "end_time": "2000-01-15"}, - {"start_time": "2000-01-15", "end_time": "2000-01-22"}, - {"start_time": "2000-01-22", "end_time": "2000-01-29"}, - {"start_time": "2000-01-29", "end_time": "2000-02-05"}, - {"start_time": "2000-02-05", "end_time": "2000-02-10"}, - ], - ), - # for Daily streams - ( - 31, - [ - {"start_time": "2000-01-01", "end_time": "2000-02-01"}, - {"start_time": "2000-02-01", "end_time": "2000-02-10"}, - ], - ), - # when start-end period == slice period - ( - 40, - [ - {"start_time": "2000-01-01", "end_time": "2000-02-10"}, - ], - ), - # when start-end period > slice period - ( - 100, - [ - {"start_time": "2000-01-01", "end_time": "2000-02-10"}, - ], - ), - ], -) -def test_date_slices(slice_period, expected_date_slices): - stats_stream.slice_period = slice_period - date_slices = stats_stream.date_slices() - assert date_slices == expected_date_slices - response_organizations = { "organizations": [ { "organization": { "id": "organization_id_1", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", } } ] } -def test_organizations(requests_mock): +def test_should_retry_403_error(requests_mock): + requests_mock.post("https://accounts.snapchat.com/login/oauth2/access_token", json={"access_token": "XXX", "expires_in": 3600}) + requests_mock.register_uri( + "GET", "https://adsapi.snapchat.com/v1/me/organizations", [{"status_code": 403, "json": {"organizations": []}}] + ) + stream = find_stream("organizations", config_mock) + with pytest.raises(DefaultBackoffException): + list(stream.read_records(sync_mode=SyncMode.full_refresh)) + +def test_organizations(requests_mock): + requests_mock.post("https://accounts.snapchat.com/login/oauth2/access_token", json={"access_token": "XXX", "expires_in": 3600}) requests_mock.get("https://adsapi.snapchat.com/v1/me/organizations", json=response_organizations) - stream = Organizations(**config_mock) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - assert list(records) == [ - { - "id": "organization_id_1", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", - } - ] + stream = find_stream("organizations", config_mock) + records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=StreamSlice(partition={}, + cursor_slice={"start_time": "2024-01-01", + "end_time": "2024-01-01"}), + stream_state=None) + assert json.dumps(next(records).data, sort_keys=True) == json.dumps( + {"id": "organization_id_1", "updated_at": "2024-02-05T22:35:17.819Z", "created_at": "2024-02-05T11:13:03.910Z"}, sort_keys=True) response_adaccounts = { @@ -107,15 +65,15 @@ def test_organizations(requests_mock): { "adaccount": { "id": "adaccount_id_1", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", } }, { "adaccount": { "id": "adaccount_id_2", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", } }, ] @@ -129,22 +87,24 @@ def run_stream(stream): def test_accounts(requests_mock): + requests_mock.post("https://accounts.snapchat.com/login/oauth2/access_token", json={"access_token": "XXX", "expires_in": 3600}) requests_mock.get("https://adsapi.snapchat.com/v1/me/organizations", json=response_organizations) requests_mock.get("https://adsapi.snapchat.com/v1/organizations/organization_id_1/adaccounts", json=response_adaccounts) - stream = Adaccounts(**config_mock) + stream = find_stream("adaccounts", config_mock) records = run_stream(stream) - assert list(records) == [ + list_records = [record.data for record in records] + assert list_records == [ { "id": "adaccount_id_1", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", }, { "id": "adaccount_id_2", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", }, ] @@ -154,15 +114,15 @@ def test_accounts(requests_mock): { "ad": { "id": "ad_id_1", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", } }, { "ad": { "id": "ad_id_2", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", } }, ] @@ -170,24 +130,26 @@ def test_accounts(requests_mock): def test_ads(requests_mock): + requests_mock.post("https://accounts.snapchat.com/login/oauth2/access_token", json={"access_token": "XXX", "expires_in": 3600}) requests_mock.get("https://adsapi.snapchat.com/v1/me/organizations", json=response_organizations) requests_mock.get("https://adsapi.snapchat.com/v1/organizations/organization_id_1/adaccounts", json=response_adaccounts) requests_mock.get("https://adsapi.snapchat.com/v1/adaccounts/adaccount_id_1/ads", json={"ads": []}) requests_mock.get("https://adsapi.snapchat.com/v1/adaccounts/adaccount_id_2/ads", json=response_ads) - stream = Ads(**config_mock) + stream = find_stream("ads", config_mock) records = run_stream(stream) - assert list(records) == [ + list_records = [record.data for record in records] + assert list_records == [ { "id": "ad_id_1", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", }, { "id": "ad_id_2", - "updated_at": "2020-12-15T22:35:17.819Z", - "created_at": "2020-12-15T11:13:03.910Z", + "updated_at": "2024-02-05T22:35:17.819Z", + "created_at": "2024-02-05T11:13:03.910Z", }, ] @@ -215,7 +177,6 @@ def test_ads(requests_mock): ], } - response_ads_stats_lifetime_2 = { "request_status": "SUCCESS", "request_id": "d0cb395f-c39d-480d-b62c-24878c7d0b76", @@ -241,6 +202,7 @@ def test_ads(requests_mock): def test_ads_stats_lifetime(requests_mock): + requests_mock.post("https://accounts.snapchat.com/login/oauth2/access_token", json={"access_token": "XXX", "expires_in": 3600}) requests_mock.get("https://adsapi.snapchat.com/v1/me/organizations", json=response_organizations) requests_mock.get("https://adsapi.snapchat.com/v1/organizations/organization_id_1/adaccounts", json=response_adaccounts) @@ -249,31 +211,166 @@ def test_ads_stats_lifetime(requests_mock): requests_mock.get("https://adsapi.snapchat.com/v1/ads/ad_id_1/stats", json=response_ads_stats_lifetime_1) requests_mock.get("https://adsapi.snapchat.com/v1/ads/ad_id_2/stats", json=response_ads_stats_lifetime_2) - stream = AdsStatsLifetime(**config_mock) + stream = find_stream("ads_stats_lifetime", config_mock) records = run_stream(stream) - assert list(records) == [ + list_records = [record.data for record in records] + assert list_records == [ { - "conversion_data_processed_end_time": "2022-07-01T00:00:00.000Z", + "id": "ad_id_1", + "type": "AD", + "granularity": "LIFETIME", + "start_time": "2016-09-26T00:00:00.000-07:00", "end_time": "2022-07-01T07:00:00.000-07:00", "finalized_data_end_time": "2022-07-01T07:00:00.000-07:00", - "granularity": "LIFETIME", - "id": "ad_id_1", + "conversion_data_processed_end_time": "2022-07-01T00:00:00.000Z", + "android_installs": None, + "attachment_avg_view_time_millis": None, + "attachment_impressions": None, + "attachment_quartile_1": None, + "attachment_quartile_2": None, + "attachment_quartile_3": None, + "attachment_total_view_time_millis": None, + "attachment_view_completion": None, + "avg_screen_time_millis": None, + "avg_view_time_millis": None, "impressions": 0, - "start_time": "2016-09-26T00:00:00.000-07:00", + "ios_installs": None, + "quartile_1": None, + "quartile_2": None, + "quartile_3": None, + "screen_time_millis": None, + "swipe_up_percent": None, "swipes": 0, - "type": "AD", + "total_installs": None, + "video_views": None, + "video_views_time_based": None, + "video_views_15s": None, + "view_completion": None, + "view_time_millis": None, + "paid_impressions": None, + "earned_impressions": None, + "total_impressions": None, + "play_time_millis": None, + "shares": None, + "saves": None, + "native_leads": None, + "conversion_purchases": None, + "conversion_purchases_value": None, + "conversion_save": None, + "conversion_start_checkout": None, + "conversion_add_cart": None, + "conversion_view_content": None, + "conversion_add_billing": None, + "conversion_searches": None, + "conversion_level_completes": None, + "conversion_app_opens": None, + "conversion_page_views": None, + "conversion_subscribe": None, + "conversion_ad_click": None, + "conversion_ad_view": None, + "conversion_complete_tutorial": None, + "conversion_invite": None, + "conversion_login": None, + "conversion_share": None, + "conversion_reserve": None, + "conversion_achievement_unlocked": None, + "conversion_add_to_wishlist": None, + "conversion_spend_credits": None, + "conversion_rate": None, + "conversion_start_trial": None, + "conversion_list_view": None, + "custom_event_1": None, + "custom_event_2": None, + "custom_event_3": None, + "custom_event_4": None, + "custom_event_5": None, + "story_opens": None, + "story_completes": None, + "attachment_frequency": None, + "attachment_uniques": None, + "frequency": None, + "uniques": None, + "total_reach": None, + "earned_reach": None }, { - "conversion_data_processed_end_time": "2022-07-01T00:00:00.000Z", + "id": "ad_id_2", + "type": "AD", + "granularity": "LIFETIME", + "start_time": "2016-09-26T00:00:00.000-07:00", "end_time": "2022-07-01T07:00:00.000-07:00", "finalized_data_end_time": "2022-07-01T07:00:00.000-07:00", - "granularity": "LIFETIME", - "id": "ad_id_2", + "conversion_data_processed_end_time": "2022-07-01T00:00:00.000Z", + "android_installs": None, + "attachment_avg_view_time_millis": None, + "attachment_impressions": None, + "attachment_quartile_1": None, + "attachment_quartile_2": None, + "attachment_quartile_3": None, + "attachment_total_view_time_millis": None, + "attachment_view_completion": None, + "avg_screen_time_millis": None, + "avg_view_time_millis": None, "impressions": 0, - "start_time": "2016-09-26T00:00:00.000-07:00", + "ios_installs": None, + "quartile_1": None, + "quartile_2": None, + "quartile_3": None, + "screen_time_millis": None, + "swipe_up_percent": None, "swipes": 0, - "type": "AD", - }, + "total_installs": None, + "video_views": None, + "video_views_time_based": None, + "video_views_15s": None, + "view_completion": None, + "view_time_millis": None, + "paid_impressions": None, + "earned_impressions": None, + "total_impressions": None, + "play_time_millis": None, + "shares": None, + "saves": None, + "native_leads": None, + "conversion_purchases": None, + "conversion_purchases_value": None, + "conversion_save": None, + "conversion_start_checkout": None, + "conversion_add_cart": None, + "conversion_view_content": None, + "conversion_add_billing": None, + "conversion_searches": None, + "conversion_level_completes": None, + "conversion_app_opens": None, + "conversion_page_views": None, + "conversion_subscribe": None, + "conversion_ad_click": None, + "conversion_ad_view": None, + "conversion_complete_tutorial": None, + "conversion_invite": None, + "conversion_login": None, + "conversion_share": None, + "conversion_reserve": None, + "conversion_achievement_unlocked": None, + "conversion_add_to_wishlist": None, + "conversion_spend_credits": None, + "conversion_rate": None, + "conversion_start_trial": None, + "conversion_list_view": None, + "custom_event_1": None, + "custom_event_2": None, + "custom_event_3": None, + "custom_event_4": None, + "custom_event_5": None, + "story_opens": None, + "story_completes": None, + "attachment_frequency": None, + "attachment_uniques": None, + "frequency": None, + "uniques": None, + "total_reach": None, + "earned_reach": None + } ] @@ -287,14 +384,14 @@ def test_ads_stats_lifetime(requests_mock): "id": "417d0269-80fb-496a-b5f3-ec0bac665144", "type": "AD", "granularity": "DAY", - "start_time": "2022-06-25T00:00:00.000-07:00", - "end_time": "2022-06-29T00:00:00.000-07:00", + "start_time": "2024-02-05T00:00:00.000-07:00", + "end_time": "2024-02-08T00:00:00.000-07:00", "finalized_data_end_time": "2022-06-30T00:00:00.000-07:00", "conversion_data_processed_end_time": "2022-06-30T00:00:00.000Z", "timeseries": [ { - "start_time": "2022-06-25T00:00:00.000-07:00", - "end_time": "2022-06-26T00:00:00.000-07:00", + "start_time": "2024-02-05T00:00:00.000-07:00", + "end_time": "2024-02-06T00:00:00.000-07:00", "stats": { "impressions": 0, "swipes": 0, @@ -304,8 +401,8 @@ def test_ads_stats_lifetime(requests_mock): }, }, { - "start_time": "2022-06-26T00:00:00.000-07:00", - "end_time": "2022-06-27T00:00:00.000-07:00", + "start_time": "2024-02-07T00:00:00.000-07:00", + "end_time": "2024-02-08T00:00:00.000-07:00", "stats": { "impressions": 0, "swipes": 0, @@ -322,6 +419,7 @@ def test_ads_stats_lifetime(requests_mock): def test_ads_stats_daily(requests_mock): + requests_mock.post("https://accounts.snapchat.com/login/oauth2/access_token", json={"access_token": "XXX", "expires_in": 3600}) requests_mock.get("https://adsapi.snapchat.com/v1/me/organizations", json=response_organizations) requests_mock.get("https://adsapi.snapchat.com/v1/organizations/organization_id_1/adaccounts", json=response_adaccounts) @@ -330,36 +428,11 @@ def test_ads_stats_daily(requests_mock): requests_mock.get("https://adsapi.snapchat.com/v1/ads/ad_id_1/stats", json={"timeseries_stats": []}) requests_mock.get("https://adsapi.snapchat.com/v1/ads/ad_id_2/stats", json=response_ads_stats_daily_1) - stream = AdsStatsDaily(**config_mock) + stream = find_stream("ads_stats_daily", config_mock) records = run_stream(stream) assert len(list(records)) == 4 # 2 records for each of 2 slices -def test_get_parent_ids(requests_mock): - """Test cache usage in get_parent_ids""" - # nonlocal auxiliary_id_map - - requests_mock.get("https://adsapi.snapchat.com/v1/me/organizations", json=response_organizations) - requests_mock.get("https://adsapi.snapchat.com/v1/organizations/organization_id_1/adaccounts", json=response_adaccounts) - - stream = Adaccounts(**config_mock) - - # reset cache, it can be filled by calls in previous tests - source_snapchat_marketing.source.auxiliary_id_map = {} - - # 1st call - list(run_stream(stream)) - # 1st request to organizations - # 2nd request to adaccounts - assert len(requests_mock.request_history) == 2 - - # 2nd call - list(run_stream(stream)) - # request to organizations is skipped due to cache - # 3rd request to adaccounts - assert len(requests_mock.request_history) == 3 - - def test_source_streams(): source_config = {"client_id": "XXX", "client_secret": "XXX", "refresh_token": "XXX", "start_date": "2022-05-25"} streams = SourceSnapchatMarketing().streams(config=source_config) @@ -369,34 +442,7 @@ def test_source_streams(): def test_source_check_connection(requests_mock): source_config = {"client_id": "XXX", "client_secret": "XXX", "refresh_token": "XXX", "start_date": "2022-05-25"} requests_mock.post("https://accounts.snapchat.com/login/oauth2/access_token", json={"access_token": "XXX", "expires_in": 3600}) - requests_mock.get("https://adsapi.snapchat.com/v1/me", json={}) + requests_mock.get("https://adsapi.snapchat.com/v1/me/organizations", json={}) - results = SourceSnapchatMarketing().check_connection(logger=None, config=source_config) + results = SourceSnapchatMarketing().check_connection(logger=logger, config=source_config) assert results == (True, None) - - -def test_retry_get_access_token(requests_mock): - requests_mock.register_uri( - "POST", - "https://accounts.snapchat.com/login/oauth2/access_token", - [{"status_code": 429}, {"status_code": 429}, {"status_code": 200, "json": {"access_token": "token", "expires_in": 3600}}], - ) - auth = Oauth2Authenticator( - token_refresh_endpoint="https://accounts.snapchat.com/login/oauth2/access_token", - client_id="client_id", - client_secret="client_secret", - refresh_token="refresh_token", - ) - token = auth.get_access_token() - assert len(requests_mock.request_history) == 3 - assert token == "token" - - -def test_should_retry_403_error(requests_mock): - requests_mock.register_uri( - "GET", "https://adsapi.snapchat.com/v1/me/organizations", [{"status_code": 403, "json": {"organizations": []}}] - ) - stream = Organizations(**config_mock) - records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) - - assert not records diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/.coveragerc b/airbyte-integrations/connectors/source-tiktok-marketing/.coveragerc new file mode 100644 index 000000000000..92d753c7c6ef --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_tiktok_marketing/run.py diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json index 38b317323132..3b7690cfe7aa 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json @@ -6,7 +6,17 @@ "name": "ad_group_audience_reports_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -17,7 +27,17 @@ "name": "ads_audience_reports_by_province_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -28,7 +48,17 @@ "name": "ad_groups_reports_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -39,7 +69,26 @@ "name": "ad_groups" }, "stream_state": { - "modify_time": "2099-01-01 01:00:00" + "states": [ + { + "partition": { + "advertiser_id": "7001035076276387841", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01 01:01:01" + } + }, + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01 01:01:01" + } + } + ] } } }, @@ -50,7 +99,17 @@ "name": "ads_audience_reports_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -61,7 +120,17 @@ "name": "ads_reports_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -72,7 +141,26 @@ "name": "ads" }, "stream_state": { - "modify_time": "2099-01-01 01:00:0" + "states": [ + { + "partition": { + "advertiser_id": "7001035076276387841", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01 01:01:01" + } + }, + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01 01:01:01" + } + } + ] } } }, @@ -83,7 +171,17 @@ "name": "advertisers_audience_reports_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -94,7 +192,17 @@ "name": "advertisers_reports_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -105,7 +213,17 @@ "name": "campaigns_audience_reports_by_country_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -116,7 +234,17 @@ "name": "campaigns_reports_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -127,7 +255,26 @@ "name": "campaigns" }, "stream_state": { - "modify_time": "2099-01-01 01:00:0" + "states": [ + { + "partition": { + "advertiser_id": "7001035076276387841", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01 01:01:01" + } + }, + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01 01:01:01" + } + } + ] } } }, @@ -138,7 +285,26 @@ "name": "creative_assets_images" }, "stream_state": { - "modify_time": "2099-01-01 01:00:0" + "states": [ + { + "partition": { + "advertiser_id": "7001035076276387841", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01T01:01:01Z" + } + }, + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01T01:01:01Z" + } + } + ] } } }, @@ -149,7 +315,26 @@ "name": "creative_assets_videos" }, "stream_state": { - "modify_time": "2099-01-01 01:00:0" + "states": [ + { + "partition": { + "advertiser_id": "7001035076276387841", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01T01:01:01Z" + } + }, + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "modify_time": "2222-01-01T01:01:01Z" + } + } + ] } } }, @@ -160,7 +345,17 @@ "name": "ad_group_audience_reports_by_platform_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -171,7 +366,17 @@ "name": "campaigns_audience_reports_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -182,7 +387,17 @@ "name": "campaigns_audience_reports_by_platform_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -193,7 +408,17 @@ "name": "ad_group_audience_reports_by_country_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -204,7 +429,17 @@ "name": "advertisers_audience_reports_by_platform_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -215,7 +450,17 @@ "name": "ads_audience_reports_by_country_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -226,7 +471,17 @@ "name": "ads_audience_reports_by_platform_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } }, @@ -237,7 +492,17 @@ "name": "advertisers_audience_reports_by_country_daily" }, "stream_state": { - "stat_time_day": "2099-01-01" + "states": [ + { + "partition": { + "advertiser_id": "7002238017842757633", + "parent_slice": {} + }, + "cursor": { + "stat_time_day": "2222-01-01 00:00:00" + } + } + ] } } } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl index 0efab7420dd8..3721d0552c72 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl @@ -16,9 +16,9 @@ {"stream": "creative_assets_videos", "data": {"size": 13605359, "signature": "ded8dcda6363d0219764ba5246a673ad", "file_name": "Video16974675945951_Whale, sea, electronica(859574)", "width": 720, "create_time": "2023-10-16T14:46:50Z", "bit_rate": 3626524, "allow_download": true, "modify_time": "2023-10-16T14:46:50Z", "height": 1280, "material_id": "7290567851409981441", "preview_url_expire_time": "2023-10-23 17:48:23", "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/oEaaNUsAlIQwDnBGmBLecBzDQGnACMA0Xgbfb5~tplv-noop.image?x-expires=1698083303&x-signature=%2Fq08ZTEll8KgfYXlg%2B1r8oqCgBk%3D", "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkplrc77u30n4dkdb0", "preview_url": "http://v16m-default.akamaized.net/4a8d0298528bfaa92661ed8b31e822b7/6536b1e7/video/tos/alisg/tos-alisg-ve-0051c001-sg/oQABgVatYClIbB2FGEnU0QXmQLTNDzegfWGDs5/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1630&bt=815&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7T8KqFZiyq8Z&mime_type=video_mp4&qs=0&rc=ZTw5M2c4ZDhoODllM2VmNkBpM2dncTY6Zm9ubjMzODYzNEAxXy40YDUyXzAxYmMwMGItYSNfaDEtcjRvbWpgLS1kMC1zcw%3D%3D&l=202310231147525B364A25545CD403DD31&btag=e00088000", "duration": 30.013, "displayable": true, "format": "mp4"}, "emitted_at": 1698061673747} {"stream": "creative_assets_videos", "data": {"size": 2150312, "signature": "61c2035644dbcea75ef315af73a120ea", "file_name": "Optimized Version 3_202203281449", "width": 720, "create_time": "2022-03-28T11:49:10Z", "bit_rate": 3430892, "allow_download": true, "modify_time": "2022-03-28T11:49:12Z", "height": 1280, "material_id": "7080116206691549186", "preview_url_expire_time": "2023-10-23 17:47:58", "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/v0201/7f371ff6f0764f8b8ef4f37d7b980d50~tplv-noop.image?x-expires=1698083278&x-signature=1Gq16fQpsQPQsQwk5nzMZoQ5GcY%3D", "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000c90q1d3c77ub6e96fvo0", "preview_url": "http://v16m-default.akamaized.net/47236ddfa95a07a9b22fc7b37b96568b/6536b1ce/video/tos/alisg/tos-alisg-v-0000/8968c64a5dc6489a89fe324a6156634d/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1664&bt=832&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7T8KqFZiyq8Z&mime_type=video_mp4&qs=0&rc=NjY6aDpoNmY0aTRnNmczOEBpM3k5aGU6Zmd0PDMzODYzNEAvYzZiYTAuNTYxMzUxYDZjYSNsYzYzcjQwLi1gLS1kMC1zcw%3D%3D&l=202310231147525B364A25545CD403DD31&btag=e00088000", "duration": 5.014, "displayable": true, "format": "mp4"}, "emitted_at": 1698061673748} {"stream": "creative_assets_videos", "data": {"size": 2338817, "signature": "f95bfbd2c3d6f6c16d3bd048447b3b73", "file_name": "7021053237617754114", "width": 720, "create_time": "2021-10-20T08:04:10Z", "bit_rate": 1867878, "allow_download": false, "modify_time": "2022-03-28T12:08:49Z", "height": 1280, "material_id": "7021053237617754114", "preview_url_expire_time": "2023-10-23 17:48:03", "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/v0201/8f77082a1f3c40c586f8282356490c58~tplv-noop.image?x-expires=1698083283&x-signature=LCEIxKOuPrFQFqfrpoGE2K6tK00%3D", "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000c5nsqcbc77ubdn136b70", "preview_url": "http://v16m-default.akamaized.net/cb4fc19071c911726067bfea11fe06ec/6536b1d3/video/tos/alisg/tos-alisg-v-0000/43f1d52d089a4e428e31dcf356d66906/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1642&bt=821&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7T8KqFZiyq8Z&mime_type=video_mp4&qs=0&rc=OzpoOWc2OzdpN2hnOjtoOEBpM2U2cWU6ZmZ2ODMzODYzNEAzLTMtNGAuNl8xMmE2MGA1YSM0My5hcjRfbmtgLS1kMC1zcw%3D%3D&l=202310231147525B364A25545CD403DD31&btag=e00088000", "duration": 10.017, "displayable": true, "format": "mp4"}, "emitted_at": 1698061673749} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7001035076276387841, "advertiser_name": "Airbyte0827"}, "emitted_at": 1698061674290} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7001040009704833026, "advertiser_name": "Airbyte08270"}, "emitted_at": 1698061674291} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7002238017842757633, "advertiser_name": "Airbyte0830"}, "emitted_at": 1698061674292} +{"stream": "advertiser_ids", "data": {"advertiser_id": "7001035076276387841", "advertiser_name": "Airbyte0827"}, "emitted_at": 1698061674290} +{"stream": "advertiser_ids", "data": {"advertiser_id": "7001040009704833026", "advertiser_name": "Airbyte08270"}, "emitted_at": 1698061674291} +{"stream": "advertiser_ids", "data": {"advertiser_id": "7002238017842757633", "advertiser_name": "Airbyte0830"}, "emitted_at": 1698061674292} {"stream": "ads_reports_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-25 00:00:00"}, "metrics": {"cpm": "3.43", "shares": 0, "real_time_cost_per_result": "0.290", "video_views_p75": 140, "follows": 0, "comments": 0, "mobile_app_id": "0", "tt_app_id": 0, "video_watched_6s": 180, "cost_per_result": "0.290", "average_video_play_per_user": 1.64, "cta_purchase": "0", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "video_views_p50": 214, "cost_per_secondary_goal_result": null, "ctr": "1.18", "real_time_result_rate": "1.18", "real_time_app_install_cost": 0, "impressions": "5830", "conversion": "0", "cta_conversion": "0", "placement_type": "Automatic Placement", "profile_visits": 0, "result": "69", "cost_per_1000_reached": "4.16", "video_views_p25": 513, "campaign_id": 1714125042508817, "vta_purchase": "0", "tt_app_name": "0", "onsite_shopping": "0", "total_pageview": "0", "cpc": "0.29", "complete_payment": "0", "dpa_target_audience_type": null, "total_onsite_shopping_value": "0.00", "vta_conversion": "0", "spend": "20.00", "real_time_result": "69", "secondary_goal_result_rate": null, "conversion_rate": "0.00", "secondary_goal_result": null, "adgroup_name": "Ad Group20211020010107", "total_purchase_value": "0.00", "result_rate": "1.18", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "likes": 36, "video_watched_2s": 686, "real_time_app_install": 0, "reach": "4806", "total_complete_payment_rate": "0.00", "clicks": "69", "cost_per_conversion": "0.00", "app_install": 0, "real_time_conversion_rate": "0.00", "video_play_actions": 5173, "value_per_complete_payment": "0.00", "frequency": "1.21", "average_video_play": 1.52, "video_views_p100": 92, "clicks_on_music_disc": 0, "adgroup_id": 1714125049901106, "campaign_name": "Website Traffic20211020010104"}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1698062442958} {"stream": "ads_reports_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"cpm": "5.31", "shares": 0, "real_time_cost_per_result": "0.377", "video_views_p75": 74, "follows": 0, "comments": 1, "mobile_app_id": "0", "tt_app_id": 0, "video_watched_6s": 106, "cost_per_result": "0.377", "average_video_play_per_user": 1.55, "cta_purchase": "0", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "video_views_p50": 130, "cost_per_secondary_goal_result": null, "ctr": "1.41", "real_time_result_rate": "1.41", "real_time_app_install_cost": 0, "impressions": "3765", "conversion": "0", "cta_conversion": "0", "placement_type": "Automatic Placement", "profile_visits": 0, "result": "53", "cost_per_1000_reached": "6.38", "video_views_p25": 295, "campaign_id": 1714125042508817, "vta_purchase": "0", "tt_app_name": "0", "onsite_shopping": "0", "total_pageview": "0", "cpc": "0.38", "complete_payment": "0", "dpa_target_audience_type": null, "total_onsite_shopping_value": "0.00", "vta_conversion": "0", "spend": "20.00", "real_time_result": "53", "secondary_goal_result_rate": null, "conversion_rate": "0.00", "secondary_goal_result": null, "adgroup_name": "Ad Group20211020010107", "total_purchase_value": "0.00", "result_rate": "1.41", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "likes": 36, "video_watched_2s": 408, "real_time_app_install": 0, "reach": "3134", "total_complete_payment_rate": "0.00", "clicks": "53", "cost_per_conversion": "0.00", "app_install": 0, "real_time_conversion_rate": "0.00", "video_play_actions": 3344, "value_per_complete_payment": "0.00", "frequency": "1.20", "average_video_play": 1.45, "video_views_p100": 52, "clicks_on_music_disc": 0, "adgroup_id": 1714125049901106, "campaign_name": "Website Traffic20211020010104"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1698062442963} {"stream": "ads_reports_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-26 00:00:00"}, "metrics": {"cpm": "5.33", "shares": 0, "real_time_cost_per_result": "0.435", "video_views_p75": 90, "follows": 0, "comments": 1, "mobile_app_id": "0", "tt_app_id": 0, "video_watched_6s": 112, "cost_per_result": "0.435", "average_video_play_per_user": 1.61, "cta_purchase": "0", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "video_views_p50": 142, "cost_per_secondary_goal_result": null, "ctr": "1.23", "real_time_result_rate": "1.23", "real_time_app_install_cost": 0, "impressions": "3750", "conversion": "0", "cta_conversion": "0", "placement_type": "Automatic Placement", "profile_visits": 0, "result": "46", "cost_per_1000_reached": "6.41", "video_views_p25": 297, "campaign_id": 1714125042508817, "vta_purchase": "0", "tt_app_name": "0", "onsite_shopping": "0", "total_pageview": "0", "cpc": "0.43", "complete_payment": "0", "dpa_target_audience_type": null, "total_onsite_shopping_value": "0.00", "vta_conversion": "0", "spend": "20.00", "real_time_result": "46", "secondary_goal_result_rate": null, "conversion_rate": "0.00", "secondary_goal_result": null, "adgroup_name": "Ad Group20211020010107", "total_purchase_value": "0.00", "result_rate": "1.23", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "likes": 25, "video_watched_2s": 413, "real_time_app_install": 0, "reach": "3119", "total_complete_payment_rate": "0.00", "clicks": "46", "cost_per_conversion": "0.00", "app_install": 0, "real_time_conversion_rate": "0.00", "video_play_actions": 3344, "value_per_complete_payment": "0.00", "frequency": "1.20", "average_video_play": 1.5, "video_views_p100": 71, "clicks_on_music_disc": 0, "adgroup_id": 1714125049901106, "campaign_name": "Website Traffic20211020010104"}, "stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1698062442968} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl index c01f830dbef7..820755928b66 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl @@ -30,3 +30,29 @@ {"stream": "advertisers_audience_reports", "data": {"metrics": {"cpm": "6.75", "ctr": "1.79", "impressions": "335", "spend": "2.26", "clicks": "6", "cpc": "0.38"}, "dimensions": {"age": "AGE_18_24", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_18_24"}, "emitted_at": 1698066026190} {"stream": "advertisers_audience_reports", "data": {"metrics": {"cpm": "0.00", "ctr": "2.86", "impressions": "35", "spend": "0.00", "clicks": "1", "cpc": "0.00"}, "dimensions": {"age": "AGE_35_44", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698066026193} {"stream": "advertisers_audience_reports", "data": {"metrics": {"cpm": "3.88", "ctr": "1.35", "impressions": "2146", "spend": "8.32", "clicks": "29", "cpc": "0.29"}, "dimensions": {"age": "AGE_13_17", "gender": "FEMALE", "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1698066026196} +{"stream": "campaigns_audience_reports", "data": {"dimensions": {"age": "AGE_18_24", "gender": "MALE", "campaign_id": 1779923887578145, "stat_time_day": "2023-10-16 00:00:00"}, "metrics": {"spend": "0.00", "clicks": "0", "campaign_name": "UTM_PARAMSTraffic20231016173112", "ctr": "0.00", "cpm": "0.00", "cpc": "0.00", "impressions": "31"}, "stat_time_day": "2023-10-16 00:00:00", "campaign_id": 1779923887578145, "gender": "MALE", "age": "AGE_18_24"}, "emitted_at": 1716548552147} +{"stream": "campaigns_audience_reports", "data": {"dimensions": {"age": "AGE_55_100", "gender": "FEMALE", "campaign_id": 1779923887578145, "stat_time_day": "2023-10-16 00:00:00"}, "metrics": {"spend": "0.00", "clicks": "0", "campaign_name": "UTM_PARAMSTraffic20231016173112", "ctr": "0.00", "cpm": "0.00", "cpc": "0.00", "impressions": "5"}, "stat_time_day": "2023-10-16 00:00:00", "campaign_id": 1779923887578145, "gender": "FEMALE", "age": "AGE_55_100"}, "emitted_at": 1716548552153} +{"stream": "campaigns_audience_reports", "data": {"dimensions": {"age": "AGE_45_54", "gender": "MALE", "campaign_id": 1779923887578145, "stat_time_day": "2023-10-16 00:00:00"}, "metrics": {"spend": "0.00", "clicks": "0", "campaign_name": "UTM_PARAMSTraffic20231016173112", "ctr": "0.00", "cpm": "0.00", "cpc": "0.00", "impressions": "1"}, "stat_time_day": "2023-10-16 00:00:00", "campaign_id": 1779923887578145, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1716548552159} +{"stream": "ad_group_audience_reports_by_platform", "data": {"metrics": {"campaign_id": 1779923887578145, "real_time_result": "61", "spend": "9.09", "adgroup_name": "Ad group 20231016073545", "real_time_result_rate": "2.40", "tt_app_name": "0", "cpc": "0.15", "dpa_target_audience_type": "-", "result": "61", "result_rate": "2.40", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.00", "cost_per_result": "0.15", "tt_app_id": "0", "impressions": "2546", "placement_type": "Automatic Placement", "promotion_type": "Website", "campaign_name": "UTM_PARAMSTraffic20231016173112", "real_time_cost_per_result": "0.15", "mobile_app_id": "0", "clicks": "61", "real_time_conversion": "0", "conversion": "0", "cost_per_conversion": "0.00", "cpm": "3.57", "ctr": "2.40", "real_time_conversion_rate": "0.00"}, "dimensions": {"adgroup_id": 1779923881029666, "platform": "IPHONE", "stat_time_day": "2023-10-16 00:00:00"}, "stat_time_day": "2023-10-16 00:00:00", "adgroup_id": 1779923881029666, "platform": "IPHONE"}, "emitted_at": 1716548748341} +{"stream": "ad_group_audience_reports_by_platform", "data": {"metrics": {"campaign_id": 1779923887578145, "real_time_result": "0", "spend": "0.00", "adgroup_name": "Ad group 20231016073545", "real_time_result_rate": "0.00", "tt_app_name": "0", "cpc": "0.00", "dpa_target_audience_type": "-", "result": "0", "result_rate": "0.00", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.00", "cost_per_result": "0.00", "tt_app_id": "0", "impressions": "7", "placement_type": "Automatic Placement", "promotion_type": "Website", "campaign_name": "UTM_PARAMSTraffic20231016173112", "real_time_cost_per_result": "0.00", "mobile_app_id": "0", "clicks": "0", "real_time_conversion": "0", "conversion": "0", "cost_per_conversion": "0.00", "cpm": "0.00", "ctr": "0.00", "real_time_conversion_rate": "0.00"}, "dimensions": {"adgroup_id": 1779923881029666, "platform": "ANDROID", "stat_time_day": "2023-10-16 00:00:00"}, "stat_time_day": "2023-10-16 00:00:00", "adgroup_id": 1779923881029666, "platform": "ANDROID"}, "emitted_at": 1716548748346} +{"stream": "ad_group_audience_reports_by_platform", "data": {"metrics": {"campaign_id": 1779923887578145, "real_time_result": "5", "spend": "0.91", "adgroup_name": "Ad group 20231016073545", "real_time_result_rate": "2.91", "tt_app_name": "0", "cpc": "0.18", "dpa_target_audience_type": "-", "result": "5", "result_rate": "2.91", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.00", "cost_per_result": "0.18", "tt_app_id": "0", "impressions": "172", "placement_type": "Automatic Placement", "promotion_type": "Website", "campaign_name": "UTM_PARAMSTraffic20231016173112", "real_time_cost_per_result": "0.18", "mobile_app_id": "0", "clicks": "5", "real_time_conversion": "0", "conversion": "0", "cost_per_conversion": "0.00", "cpm": "5.29", "ctr": "2.91", "real_time_conversion_rate": "0.00"}, "dimensions": {"adgroup_id": 1779923881029666, "platform": "IPAD", "stat_time_day": "2023-10-16 00:00:00"}, "stat_time_day": "2023-10-16 00:00:00", "adgroup_id": 1779923881029666, "platform": "IPAD"}, "emitted_at": 1716548748351} +{"stream": "ad_group_audience_reports_by_country", "data": {"dimensions": {"stat_time_day": "2022-03-29 00:00:00", "country_code": "US", "adgroup_id": 1728545385226289}, "metrics": {"cpc": "0.35", "campaign_name": "CampaignVadimTraffic", "promotion_type": "Website", "campaign_id": 1728545382536225, "real_time_result": "57", "mobile_app_id": "0", "real_time_cost_per_conversion": "0.00", "real_time_conversion_rate": "0.00", "cost_per_result": "0.35", "spend": "20.00", "real_time_cost_per_result": "0.35", "ctr": "0.93", "conversion_rate": "0.00", "conversion": "0", "tt_app_name": "0", "dpa_target_audience_type": "-", "real_time_result_rate": "0.93", "clicks": "57", "cpm": "3.26", "placement_type": "Automatic Placement", "cost_per_conversion": "0.00", "result_rate": "0.93", "tt_app_id": "0", "impressions": "6137", "adgroup_name": "AdGroupVadim", "real_time_conversion": "0", "result": "57"}, "stat_time_day": "2022-03-29 00:00:00", "adgroup_id": 1728545385226289, "country_code": "US"}, "emitted_at": 1716549438311} +{"stream": "ad_group_audience_reports_by_country", "data": {"metrics": {"promotion_type": "Website", "tt_app_id": "0", "real_time_conversion_rate": "0.00", "cost_per_conversion": "0.00", "mobile_app_id": "0", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "campaign_id": 1714125042508817, "result_rate": "0.00", "cpc": "0.00", "clicks": "0", "real_time_cost_per_result": "0.00", "dpa_target_audience_type": "-", "spend": "0.00", "result": "0", "real_time_result_rate": "0.00", "real_time_result": "0", "impressions": "0", "cost_per_result": "0.00", "campaign_name": "Website Traffic20211020010104", "tt_app_name": "0", "conversion": "0", "adgroup_name": "Ad Group20211020010107", "cpm": "0.00", "real_time_cost_per_conversion": "0.00", "real_time_conversion": "0", "ctr": "0.00"}, "dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2022-05-18 00:00:00", "country_code": "US"}, "stat_time_day": "2022-05-18 00:00:00", "adgroup_id": 1714125049901106, "country_code": "US"}, "emitted_at": 1716549439431} +{"stream": "ad_group_audience_reports_by_country", "data": {"metrics": {"real_time_conversion_rate": "0.00", "cpm": "3.67", "real_time_cost_per_conversion": "0.00", "cpc": "0.15", "placement_type": "Automatic Placement", "campaign_id": 1779923887578145, "mobile_app_id": "0", "tt_app_id": "0", "real_time_result_rate": "2.42", "dpa_target_audience_type": "-", "impressions": "2725", "cost_per_result": "0.15", "ctr": "2.42", "real_time_result": "66", "cost_per_conversion": "0.00", "campaign_name": "UTM_PARAMSTraffic20231016173112", "tt_app_name": "0", "adgroup_name": "Ad group 20231016073545", "result_rate": "2.42", "spend": "10.00", "result": "66", "promotion_type": "Website", "conversion_rate": "0.00", "clicks": "66", "conversion": "0", "real_time_cost_per_result": "0.15", "real_time_conversion": "0"}, "dimensions": {"country_code": "US", "stat_time_day": "2023-10-16 00:00:00", "adgroup_id": 1779923881029666}, "stat_time_day": "2023-10-16 00:00:00", "adgroup_id": 1779923881029666, "country_code": "US"}, "emitted_at": 1716549458277} +{"stream": "ads_audience_reports_by_country", "data": {"dimensions": {"ad_id": 1779923894506609, "country_code": "US", "stat_time_day": "2023-10-16 00:00:00"}, "metrics": {"cost_per_result": "0.15", "result_rate": "2.70", "impressions": "2335", "conversion_rate": "0.00", "cpm": "3.97", "real_time_conversion_rate": "0.00", "tt_app_id": "0", "mobile_app_id": "0", "clicks": "63", "real_time_cost_per_result": "0.15", "ctr": "2.70", "conversion": "0", "real_time_result_rate": "2.70", "adgroup_name": "Ad group 20231016073545", "tt_app_name": "0", "adgroup_id": 1779923881029666, "ad_text": "airbyte", "placement_type": "Automatic Placement", "spend": "9.26", "cost_per_conversion": "0.00", "real_time_result": "63", "cpc": "0.15", "campaign_name": "UTM_PARAMSTraffic20231016173112", "result": "63", "campaign_id": 1779923887578145, "ad_name": "Video16974675945951_Whale, sea, electronica(859574)_2023-10-16 07:46:35", "dpa_target_audience_type": "-", "real_time_conversion": "0", "promotion_type": "Website", "real_time_cost_per_conversion": "0.00"}, "stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894506609, "country_code": "US"}, "emitted_at": 1716549916816} +{"stream": "ads_audience_reports_by_country", "data": {"dimensions": {"ad_id": 1779923894511665, "country_code": "US", "stat_time_day": "2023-10-16 00:00:00"}, "metrics": {"cost_per_result": "0.25", "result_rate": "0.77", "impressions": "390", "conversion_rate": "0.00", "cpm": "1.90", "real_time_conversion_rate": "0.00", "tt_app_id": "0", "mobile_app_id": "0", "clicks": "3", "real_time_cost_per_result": "0.25", "ctr": "0.77", "conversion": "0", "real_time_result_rate": "0.77", "adgroup_name": "Ad group 20231016073545", "tt_app_name": "0", "adgroup_id": 1779923881029666, "ad_text": "airbyte", "placement_type": "Automatic Placement", "spend": "0.74", "cost_per_conversion": "0.00", "real_time_result": "3", "cpc": "0.25", "campaign_name": "UTM_PARAMSTraffic20231016173112", "result": "3", "campaign_id": 1779923887578145, "ad_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)_2023-10-16 07:46:35", "dpa_target_audience_type": "-", "real_time_conversion": "0", "promotion_type": "Website", "real_time_cost_per_conversion": "0.00"}, "stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894511665, "country_code": "US"}, "emitted_at": 1716549916821} +{"stream": "advertisers_audience_reports_by_country", "data": {"metrics": {"cpc": "0.00", "cpm": "0.00", "spend": "0.00", "ctr": "0.00", "impressions": "0", "clicks": "0"}, "dimensions": {"stat_time_day": "2022-04-02 00:00:00", "advertiser_id": 7002238017842757633, "country_code": "US"}, "stat_time_day": "2022-04-02 00:00:00", "advertiser_id": 7002238017842757633, "country_code": "US"}, "emitted_at": 1716550397762} +{"stream": "advertisers_audience_reports_by_country", "data": {"metrics": {"cpm": "0.00", "clicks": "0", "cpc": "0.00", "impressions": "0", "ctr": "0.00", "spend": "0.00"}, "dimensions": {"stat_time_day": "2022-05-18 00:00:00", "advertiser_id": 7002238017842757633, "country_code": "US"}, "stat_time_day": "2022-05-18 00:00:00", "advertiser_id": 7002238017842757633, "country_code": "US"}, "emitted_at": 1716550398276} +{"stream": "advertisers_audience_reports_by_country", "data": {"dimensions": {"country_code": "US", "stat_time_day": "2023-10-16 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"ctr": "2.42", "clicks": "66", "spend": "10.00", "impressions": "2725", "cpm": "3.67", "cpc": "0.15"}, "stat_time_day": "2023-10-16 00:00:00", "advertiser_id": 7002238017842757633, "country_code": "US"}, "emitted_at": 1716550407969} +{"stream": "campaigns_audience_reports_by_platform", "data": {"metrics": {"clicks": "5", "ctr": "2.91", "cpc": "0.18", "spend": "0.91", "impressions": "172", "campaign_name": "UTM_PARAMSTraffic20231016173112", "cpm": "5.29"}, "dimensions": {"campaign_id": 1779923887578145, "platform": "IPAD", "stat_time_day": "2023-10-16 00:00:00"}, "stat_time_day": "2023-10-16 00:00:00", "campaign_id": 1779923887578145, "platform": "IPAD"}, "emitted_at": 1716550603828} +{"stream": "campaigns_audience_reports_by_platform", "data": {"metrics": {"clicks": "0", "ctr": "0.00", "cpc": "0.00", "spend": "0.00", "impressions": "7", "campaign_name": "UTM_PARAMSTraffic20231016173112", "cpm": "0.00"}, "dimensions": {"campaign_id": 1779923887578145, "platform": "ANDROID", "stat_time_day": "2023-10-16 00:00:00"}, "stat_time_day": "2023-10-16 00:00:00", "campaign_id": 1779923887578145, "platform": "ANDROID"}, "emitted_at": 1716550603833} +{"stream": "campaigns_audience_reports_by_platform", "data": {"metrics": {"clicks": "61", "ctr": "2.40", "cpc": "0.15", "spend": "9.09", "impressions": "2546", "campaign_name": "UTM_PARAMSTraffic20231016173112", "cpm": "3.57"}, "dimensions": {"campaign_id": 1779923887578145, "platform": "IPHONE", "stat_time_day": "2023-10-16 00:00:00"}, "stat_time_day": "2023-10-16 00:00:00", "campaign_id": 1779923887578145, "platform": "IPHONE"}, "emitted_at": 1716550603838} +{"stream": "advertisers_audience_reports_by_platform", "data": {"metrics": {"ctr": "2.91", "cpc": "0.18", "spend": "0.91", "impressions": "172", "clicks": "5", "cpm": "5.29"}, "dimensions": {"platform": "IPAD", "stat_time_day": "2023-10-16 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2023-10-16 00:00:00", "advertiser_id": 7002238017842757633, "platform": "IPAD"}, "emitted_at": 1716550796752} +{"stream": "advertisers_audience_reports_by_platform", "data": {"metrics": {"ctr": "0.00", "cpc": "0.00", "spend": "0.00", "impressions": "7", "clicks": "0", "cpm": "0.00"}, "dimensions": {"platform": "ANDROID", "stat_time_day": "2023-10-16 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2023-10-16 00:00:00", "advertiser_id": 7002238017842757633, "platform": "ANDROID"}, "emitted_at": 1716550796757} +{"stream": "advertisers_audience_reports_by_platform", "data": {"metrics": {"ctr": "2.40", "cpc": "0.15", "spend": "9.09", "impressions": "2546", "clicks": "61", "cpm": "3.57"}, "dimensions": {"platform": "IPHONE", "stat_time_day": "2023-10-16 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2023-10-16 00:00:00", "advertiser_id": 7002238017842757633, "platform": "IPHONE"}, "emitted_at": 1716550796762} +{"stream": "ads_audience_reports_by_platform", "data": {"metrics": {"real_time_conversion_rate": "0.00", "cpm": "0.00", "real_time_cost_per_conversion": "0.00", "cpc": "0.00", "placement_type": "Automatic Placement", "campaign_id": 1779923887578145, "mobile_app_id": "0", "tt_app_id": "0", "real_time_result_rate": "0.00", "dpa_target_audience_type": "-", "impressions": "3", "cost_per_result": "0.00", "ctr": "0.00", "real_time_result": "0", "cost_per_conversion": "0.00", "campaign_name": "UTM_PARAMSTraffic20231016173112", "tt_app_name": "0", "adgroup_name": "Ad group 20231016073545", "result_rate": "0.00", "spend": "0.00", "result": "0", "ad_name": "Video16974675945951_Whale, sea, electronica(859574)_2023-10-16 07:46:35", "promotion_type": "Website", "ad_text": "airbyte", "conversion_rate": "0.00", "clicks": "0", "conversion": "0", "adgroup_id": 1779923881029666, "real_time_cost_per_result": "0.00", "real_time_conversion": "0"}, "dimensions": {"stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894506609, "platform": "ANDROID"}, "stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894506609, "platform": "ANDROID"}, "emitted_at": 1716550966449} +{"stream": "ads_audience_reports_by_platform", "data": {"metrics": {"real_time_conversion_rate": "0.00", "cpm": "3.95", "real_time_cost_per_conversion": "0.00", "cpc": "0.15", "placement_type": "Automatic Placement", "campaign_id": 1779923887578145, "mobile_app_id": "0", "tt_app_id": "0", "real_time_result_rate": "2.69", "dpa_target_audience_type": "-", "impressions": "2197", "cost_per_result": "0.15", "ctr": "2.69", "real_time_result": "59", "cost_per_conversion": "0.00", "campaign_name": "UTM_PARAMSTraffic20231016173112", "tt_app_name": "0", "adgroup_name": "Ad group 20231016073545", "result_rate": "2.69", "spend": "8.67", "result": "59", "ad_name": "Video16974675945951_Whale, sea, electronica(859574)_2023-10-16 07:46:35", "promotion_type": "Website", "ad_text": "airbyte", "conversion_rate": "0.00", "clicks": "59", "conversion": "0", "adgroup_id": 1779923881029666, "real_time_cost_per_result": "0.15", "real_time_conversion": "0"}, "dimensions": {"stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894506609, "platform": "IPHONE"}, "stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894506609, "platform": "IPHONE"}, "emitted_at": 1716550966454} +{"stream": "ads_audience_reports_by_platform", "data": {"metrics": {"real_time_conversion_rate": "0.00", "cpm": "4.37", "real_time_cost_per_conversion": "0.00", "cpc": "0.15", "placement_type": "Automatic Placement", "campaign_id": 1779923887578145, "mobile_app_id": "0", "tt_app_id": "0", "real_time_result_rate": "2.96", "dpa_target_audience_type": "-", "impressions": "135", "cost_per_result": "0.15", "ctr": "2.96", "real_time_result": "4", "cost_per_conversion": "0.00", "campaign_name": "UTM_PARAMSTraffic20231016173112", "tt_app_name": "0", "adgroup_name": "Ad group 20231016073545", "result_rate": "2.96", "spend": "0.59", "result": "4", "ad_name": "Video16974675945951_Whale, sea, electronica(859574)_2023-10-16 07:46:35", "promotion_type": "Website", "ad_text": "airbyte", "conversion_rate": "0.00", "clicks": "4", "conversion": "0", "adgroup_id": 1779923881029666, "real_time_cost_per_result": "0.15", "real_time_conversion": "0"}, "dimensions": {"stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894506609, "platform": "IPAD"}, "stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894506609, "platform": "IPAD"}, "emitted_at": 1716550966459} +{"stream": "ads_audience_reports_by_province", "data": {"metrics": {"conversion_rate": "0.00", "campaign_id": 1779923887578145, "promotion_type": "Website", "real_time_conversion": "0", "ad_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)_2023-10-16 07:46:35", "dpa_target_audience_type": "-", "adgroup_name": "Ad group 20231016073545", "mobile_app_id": "0", "spend": "0.00", "ad_text": "airbyte", "cpc": "0.00", "impressions": "1", "real_time_conversion_rate": "0.00", "placement_type": "Automatic Placement", "real_time_result": "0", "tt_app_id": "0", "real_time_cost_per_result": "0.00", "conversion": "0", "cost_per_result": "0.00", "real_time_cost_per_conversion": "0.00", "ctr": "0.00", "adgroup_id": 1779923881029666, "cost_per_conversion": "0.00", "clicks": "0", "campaign_name": "UTM_PARAMSTraffic20231016173112", "real_time_result_rate": "0.00", "result": "0", "result_rate": "0.00", "cpm": "0.00", "tt_app_name": "0"}, "dimensions": {"ad_id": 1779923894511665, "stat_time_day": "2023-10-16 00:00:00", "province_id": "4099753"}, "stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894511665, "province_id": "4099753"}, "emitted_at": 1716551128258} +{"stream": "ads_audience_reports_by_province", "data": {"metrics": {"conversion_rate": "0.00", "campaign_id": 1779923887578145, "promotion_type": "Website", "real_time_conversion": "0", "ad_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)_2023-10-16 07:46:35", "dpa_target_audience_type": "-", "adgroup_name": "Ad group 20231016073545", "mobile_app_id": "0", "spend": "0.00", "ad_text": "airbyte", "cpc": "0.00", "impressions": "5", "real_time_conversion_rate": "0.00", "placement_type": "Automatic Placement", "real_time_result": "0", "tt_app_id": "0", "real_time_cost_per_result": "0.00", "conversion": "0", "cost_per_result": "0.00", "real_time_cost_per_conversion": "0.00", "ctr": "0.00", "adgroup_id": 1779923881029666, "cost_per_conversion": "0.00", "clicks": "0", "campaign_name": "UTM_PARAMSTraffic20231016173112", "real_time_result_rate": "0.00", "result": "0", "result_rate": "0.00", "cpm": "0.00", "tt_app_name": "0"}, "dimensions": {"ad_id": 1779923894511665, "stat_time_day": "2023-10-16 00:00:00", "province_id": "6254926"}, "stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894511665, "province_id": "6254926"}, "emitted_at": 1716551128263} +{"stream": "ads_audience_reports_by_province", "data": {"metrics": {"conversion_rate": "0.00", "campaign_id": 1779923887578145, "promotion_type": "Website", "real_time_conversion": "0", "ad_name": "Video16974675945951_Whale, sea, electronica(859574)_2023-10-16 07:46:35", "dpa_target_audience_type": "-", "adgroup_name": "Ad group 20231016073545", "mobile_app_id": "0", "spend": "0.17", "ad_text": "airbyte", "cpc": "0.17", "impressions": "14", "real_time_conversion_rate": "0.00", "placement_type": "Automatic Placement", "real_time_result": "1", "tt_app_id": "0", "real_time_cost_per_result": "0.17", "conversion": "0", "cost_per_result": "0.17", "real_time_cost_per_conversion": "0.00", "ctr": "7.14", "adgroup_id": 1779923881029666, "cost_per_conversion": "0.00", "clicks": "1", "campaign_name": "UTM_PARAMSTraffic20231016173112", "real_time_result_rate": "7.14", "result": "1", "result_rate": "7.14", "cpm": "12.14", "tt_app_name": "0"}, "dimensions": {"ad_id": 1779923894506609, "stat_time_day": "2023-10-16 00:00:00", "province_id": "4099753"}, "stat_time_day": "2023-10-16 00:00:00", "ad_id": 1779923894506609, "province_id": "4099753"}, "emitted_at": 1716551128268} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml index a4e7151baff8..e6857389b3d3 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 - dockerImageTag: 3.9.10 + dockerImageTag: 4.0.0 dockerRepository: airbyte/source-tiktok-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing githubIssueLabel: source-tiktok-marketing @@ -24,6 +24,50 @@ data: enabled: true oss: enabled: true + releases: + breakingChanges: + 4.0.0: + message: + The source TikTok Marketing connector is being migrated from the Python CDK + to our declarative low-code CDK. Due to changes in the handling of state + format for incremental substreams, this migration constitutes a breaking + change for the following streams - ad_groups, ads, campaigns, creative_assets_images, creative_assets_videos, + *_reports_daily, *_reports_hourly, *_reports_by_country_daily, *_reports_by_platform_daily. + Also the schema for advertiser_ids stream was changed to use string type of advertiser_id + field as API docs declares it. + Users will need to reset source configuration, refresh the source schema and reset the impacted streams after upgrading. + For more information, see our migration documentation for source TikTok Marketing. + upgradeDeadline: "2024-07-15" + scopedImpact: + - scopeType: stream + impactedScopes: + - "advertiser_ids" + - "ad_group_audience_reports_by_country_daily" + - "ad_group_audience_reports_by_platform_daily" + - "ad_group_audience_reports_daily" + - "ad_groups" + - "ad_groups_reports_daily" + - "ad_groups_reports_hourly" + - "ads" + - "ads_audience_reports_by_country_daily" + - "ads_audience_reports_by_platform_daily" + - "ads_audience_reports_by_province_daily" + - "ads_audience_reports_daily" + - "ads_reports_daily" + - "ads_reports_hourly" + - "advertisers_audience_reports_by_country_daily" + - "advertisers_audience_reports_by_platform_daily" + - "advertisers_audience_reports_daily" + - "advertisers_reports_daily" + - "advertisers_reports_hourly" + - "campaigns" + - "campaigns_audience_reports_by_country_daily" + - "campaigns_audience_reports_by_platform_daily" + - "campaigns_audience_reports_daily" + - "campaigns_reports_daily" + - "campaigns_reports_hourly" + - "creative_assets_images" + - "creative_assets_videos" releaseStage: generally_available remoteRegistries: pypi: @@ -41,7 +85,7 @@ data: supportLevel: certified tags: - language:python - - cdk:python + - cdk:low-code connectorTestSuitesOptions: - suite: unitTests testSecrets: diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/poetry.lock b/airbyte-integrations/connectors/source-tiktok-marketing/poetry.lock index cede7db42349..b60e9274b31c 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/poetry.lock +++ b/airbyte-integrations/connectors/source-tiktok-marketing/poetry.lock @@ -1,31 +1,35 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" -version = "0.80.0" +version = "1.8.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-0.80.0-py3-none-any.whl", hash = "sha256:060e92323a73674fa4e9e2e4a1eb312b9b9d072c9bbe5fa28f54ef21cb4974f3"}, - {file = "airbyte_cdk-0.80.0.tar.gz", hash = "sha256:1383512a83917fecca5b24cea4c72aa5c561cf96dd464485fbcefda48fe574c5"}, + {file = "airbyte_cdk-1.8.0-py3-none-any.whl", hash = "sha256:ca23d7877005fe87ffc4a3a3de29ee55eed625d874eb59b49664b156f9ae9ee2"}, + {file = "airbyte_cdk-1.8.0.tar.gz", hash = "sha256:ac82fbfd6b650b7ed015900748e30fdd2a4c574caa54d1bcc03cb584a17f1533"}, ] [package.dependencies] -airbyte-protocol-models = "0.5.1" +airbyte-protocol-models = ">=0.9.0,<1.0" backoff = "*" cachetools = "*" +cryptography = ">=42.0.5,<43.0.0" Deprecated = ">=1.2,<1.3" -dpath = ">=2.0.1,<2.1.0" +dpath = ">=2.1.6,<3.0.0" genson = "1.2.2" isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" jsonref = ">=0.2,<0.3" jsonschema = ">=3.2.0,<3.3.0" +langchain_core = "0.1.42" pendulum = "<3.0.0" pydantic = ">=1.10.8,<2.0.0" +pyjwt = ">=2.8.0,<3.0.0" pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" +pytz = "2024.1" PyYAML = ">=6.0.1,<7.0.0" requests = "*" requests_cache = "*" @@ -34,17 +38,17 @@ wcmatch = "8.4" [package.extras] file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] -vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] name = "airbyte-protocol-models" -version = "0.5.1" +version = "0.12.2" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, - {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, + {file = "airbyte_protocol_models-0.12.2-py3-none-any.whl", hash = "sha256:1780db5b26285865b858d26502933def8e11919c9436ccf7b8b9cb0170b07c2a"}, + {file = "airbyte_protocol_models-0.12.2.tar.gz", hash = "sha256:b7c4d9a7c32c0691601c2b9416af090a858e126666e2c8c880d7a1798eb519f0"}, ] [package.dependencies] @@ -148,6 +152,70 @@ files = [ {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -258,6 +326,60 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "cryptography" +version = "42.0.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "deprecated" version = "1.2.14" @@ -277,13 +399,13 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "dpath" -version = "2.0.8" +version = "2.2.0" description = "Filesystem-like pathing and searching for dictionaries" optional = false python-versions = ">=3.7" files = [ - {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, - {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, + {file = "dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576"}, + {file = "dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e"}, ] [[package]] @@ -363,6 +485,31 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpointer" +version = "3.0.0" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"}, + {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"}, +] + [[package]] name = "jsonref" version = "0.2" @@ -395,6 +542,44 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] +[[package]] +name = "langchain-core" +version = "0.1.42" +description = "Building applications with LLMs through composability" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langchain_core-0.1.42-py3-none-any.whl", hash = "sha256:c5653ffa08a44f740295c157a24c0def4a753333f6a2c41f76bf431cd00be8b5"}, + {file = "langchain_core-0.1.42.tar.gz", hash = "sha256:40751bf60ea5d8e2b2efe65290db434717ee3834870c002e40e2811f09d814e6"}, +] + +[package.dependencies] +jsonpatch = ">=1.33,<2.0" +langsmith = ">=0.1.0,<0.2.0" +packaging = ">=23.2,<24.0" +pydantic = ">=1,<3" +PyYAML = ">=5.3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +extended-testing = ["jinja2 (>=3,<4)"] + +[[package]] +name = "langsmith" +version = "0.1.82" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.82-py3-none-any.whl", hash = "sha256:9b3653e7d316036b0c60bf0bc3e280662d660f485a4ebd8e5c9d84f9831ae79c"}, + {file = "langsmith-0.1.82.tar.gz", hash = "sha256:c02e2bbc488c10c13b52c69d271eb40bd38da078d37b6ae7ae04a18bd48140be"}, +] + +[package.dependencies] +orjson = ">=3.9.14,<4.0.0" +pydantic = {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""} +requests = ">=2,<3" + [[package]] name = "markupsafe" version = "2.1.5" @@ -464,15 +649,70 @@ files = [ {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] +[[package]] +name = "orjson" +version = "3.10.5" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b"}, + {file = "orjson-3.10.5-cp310-none-win32.whl", hash = "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2"}, + {file = "orjson-3.10.5-cp310-none-win_amd64.whl", hash = "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228"}, + {file = "orjson-3.10.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa"}, + {file = "orjson-3.10.5-cp311-none-win32.whl", hash = "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04"}, + {file = "orjson-3.10.5-cp311-none-win_amd64.whl", hash = "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c"}, + {file = "orjson-3.10.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3"}, + {file = "orjson-3.10.5-cp312-none-win32.whl", hash = "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2"}, + {file = "orjson-3.10.5-cp312-none-win_amd64.whl", hash = "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5"}, + {file = "orjson-3.10.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b"}, + {file = "orjson-3.10.5-cp38-none-win32.whl", hash = "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4"}, + {file = "orjson-3.10.5-cp38-none-win_amd64.whl", hash = "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09"}, + {file = "orjson-3.10.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86"}, + {file = "orjson-3.10.5-cp39-none-win32.whl", hash = "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47"}, + {file = "orjson-3.10.5-cp39-none-win_amd64.whl", hash = "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7"}, + {file = "orjson-3.10.5.tar.gz", hash = "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d"}, +] + [[package]] name = "packaging" -version = "24.1" +version = "23.2" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -551,6 +791,17 @@ files = [ {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + [[package]] name = "pydantic" version = "1.10.17" @@ -610,6 +861,23 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pyrate-limiter" version = "3.1.1" @@ -721,6 +989,17 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + [[package]] name = "pytzdata" version = "2020.1" @@ -888,6 +1167,21 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "tenacity" +version = "8.4.2" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"}, + {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "timeout-decorator" version = "0.5.0" @@ -1047,4 +1341,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "dae736ca2caa9569937a51240b46694cf4689a734092af252200e68ac2ea37a4" +content-hash = "13edf8f801099dcc8ac226445c7dd0b7e123abe1769efad9bcd729bb0a994d44" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/pyproject.toml b/airbyte-integrations/connectors/source-tiktok-marketing/pyproject.toml index 5b740d69fe4d..6b5c4064aacf 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/pyproject.toml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "3.9.10" +version = "4.0.0" name = "source-tiktok-marketing" description = "Source implementation for Tiktok Marketing." authors = [ "Airbyte ",] @@ -17,7 +17,7 @@ include = "source_tiktok_marketing" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "0.80.0" +airbyte-cdk = "^1" [tool.poetry.scripts] source-tiktok-marketing = "source_tiktok_marketing.run:run" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py new file mode 100644 index 000000000000..f74710d7d2ff --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import json +from typing import Any, Iterable, Mapping + +import dpath.util +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter +from airbyte_cdk.sources.declarative.types import StreamSlice + + +class MultipleAdvertiserIdsPerPartition(SubstreamPartitionRouter): + """ + Custom AdvertiserIdsPartitionRouter and AdvertiserIdPartitionRouter partition routers are used to get advertiser_ids + as slices for streams where it uses as request param. + + When using a sandbox account, it's impossible to get advertiser_ids via API. + In this case user need to provide advertiser_id in a config and connector need to use provided ids + and do not make requests to get this id. + + When advertiser_id not provided components get slices as usual. + Main difference between AdvertiserIdsPartitionRouter and AdvertiserIdPartitionRouter is + that MultipleAdvertiserIdsPerPartition returns multiple advertiser_ids in a one slice when id is not provided, + e.g. {"advertiser_ids": '["11111111", "22222222"]', "parent_slice": {}}. + And SingleAdvertiserIdPerPartition returns single slice for every advertiser_id as usual. + + MultipleAdvertiserIdsPerPartition is used by Advertisers stream, which is full refresh only, where advertiser_ids is required param, + this approach also helps make less amount of requests. + Advertisers docs: https://business-api.tiktok.com/portal/docs?id=1739593083610113. + + path_in_config: List[List[str]]: path to value in the config in priority order. + partition_field: str: field to insert partition value. + """ + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + super().__post_init__(parameters) + self._path_to_partition_in_config = self._parameters["path_in_config"] + self._partition_field = self._parameters["partition_field"] + + def get_partition_value_from_config(self) -> str: + for path in self._path_to_partition_in_config: + config_value = dpath.util.get(self.config, path, default=None) + if config_value: + return config_value + + def stream_slices(self) -> Iterable[StreamSlice]: + partition_value_in_config = self.get_partition_value_from_config() + if partition_value_in_config: + slices = [partition_value_in_config] + else: + slices = [_id.partition[self._partition_field] for _id in super().stream_slices()] + + start, end, step = 0, len(slices), 100 + + for i in range(start, end, step): + yield StreamSlice(partition={"advertiser_ids": json.dumps(slices[i : min(end, i + step)]), "parent_slice": {}}, cursor_slice={}) + + +class SingleAdvertiserIdPerPartition(MultipleAdvertiserIdsPerPartition): + """ + SingleAdvertiserIdPerPartition returns single slice for every advertiser_id in the parent stream + or takes value for advertiser_id from a config and skips reading slices. + + path_in_config: List[List[str]]: path to value in the config in priority order. + partition_field: str: field to insert partition value. + """ + + def stream_slices(self) -> Iterable[StreamSlice]: + partition_value_in_config = self.get_partition_value_from_config() + + if partition_value_in_config: + yield StreamSlice(partition={self._partition_field: partition_value_in_config, "parent_slice": {}}, cursor_slice={}) + else: + yield from super(MultipleAdvertiserIdsPerPartition, self).stream_slices() diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/hourly_datetime_based_cursor.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/hourly_datetime_based_cursor.py new file mode 100644 index 000000000000..27ad24de6061 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/hourly_datetime_based_cursor.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +from typing import Iterable + +from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.types import StreamSlice + + +class HourlyDatetimeBasedCursor(DatetimeBasedCursor): + """ + We need to overwrite stream_slices to replace hour=0, minute=0, second=0, microsecond=0 in start value in slices + because it is used as request params. + TikTok Marketing API doesn't allow date range more than one day. + In case when start date 2024-01-01 10:00:00 with step P1D slices look like {'start_time': '2024-01-01', 'end_time': '2024-01-02'}, + so API returns the following error: + 'code': 40002, 'message': 'max time span is 1 day when use stat_time_hour'. + To avoid such cases we replace start hours/minutes/seconds to 0 to have correct request params. + """ + + def stream_slices(self) -> Iterable[StreamSlice]: + """ + Partition the daterange into slices of size = step. + + The start of the window is the minimum datetime between start_datetime - lookback_window and the stream_state's datetime + The end of the window is the minimum datetime between the start of the window and end_datetime. + + :return: + """ + end_datetime = self.select_best_end_datetime() + start_datetime = self._calculate_earliest_possible_value(self.select_best_end_datetime()).replace(hour=0, minute=0, second=0) + return self._partition_daterange(start_datetime, end_datetime, self._step) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py new file mode 100644 index 000000000000..628a57a7a89f --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py @@ -0,0 +1,47 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +from typing import Any, Iterable, Mapping, Optional + +from airbyte_cdk.sources.declarative.extractors import RecordFilter +from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState + + +class PerPartitionRecordFilter(RecordFilter): + + """ + Prepares per partition stream state to be used in the Record Filter condition. + Gets current stream state cursor value for stream slice and passes it to condition. + + From + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]} + To + {"start_time": "2023-12-31"}. + + partition_field:str: Used to get partition value from current slice. + """ + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + super().__post_init__(parameters) + self._partition_field = parameters["partition_field"] + + def filter_records( + self, + records: Iterable[Mapping[str, Any]], + stream_state: StreamState, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Iterable[Mapping[str, Any]]: + stream_state = next( + ( + p["cursor"] + for p in stream_state.get("states", []) + if p["partition"][self._partition_field] == stream_slice[self._partition_field] + ), + {}, + ) + + kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token} + + for record in records: + if self._filter_interpolator.eval(self.config, record=record, **kwargs): + yield record diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py new file mode 100644 index 000000000000..0b6883e6d13d --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py @@ -0,0 +1,26 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +from dataclasses import InitVar, dataclass +from typing import Any, Mapping, Optional + +from airbyte_cdk.sources.declarative.transformations import RecordTransformation +from airbyte_cdk.sources.declarative.types import Config, FieldPointer, StreamSlice, StreamState + + +@dataclass +class TransformEmptyMetrics(RecordTransformation): + empty_value = "-" + + def transform( + self, + record: Mapping[str, Any], + config: Optional[Config] = None, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Mapping[str, Any]: + + for metric_key, metric_value in record.get("metrics", {}).items(): + if metric_value == self.empty_value: + record["metrics"][metric_key] = None + + return record diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/manifest.yaml b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/manifest.yaml new file mode 100644 index 000000000000..25d2e0eefc39 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/manifest.yaml @@ -0,0 +1,4267 @@ +version: 1.1.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - advertisers + +definitions: + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['credentials']['access_token'] if config.get('credentials') else config['access_token'] }}" + inject_into: + type: RequestOption + inject_into: header + field_name: Access-Token + + requester: + type: HttpRequester + url_base: '"https://{{ "sandbox-ads" if config.get(''credentials'', {}).get(''auth_type'', "") == "sandbox_access_token" else "business-api" }}.tiktok.com/open_api/v1.3/"' + path: "{{ parameters['path'] }}" + http_method: GET + error_handler: + type: DefaultErrorHandler + response_filters: + - predicate: "{{ response.get('code') != 0 }}" + action: FAIL + error_message: "{{ response['message'] }}" + authenticator: + $ref: "#/definitions/authenticator" + request_body_json: {} + + record_selector: + type: RecordSelector + schema_normalization: Default + extractor: + type: DpathExtractor + field_path: ["data", "list"] + + record_selector_with_filter_by_modify_time: + $ref: "#/definitions/record_selector" + record_filter: + type: CustomRecordFilter + class_name: "source_tiktok_marketing.components.semi_incremental_record_filter.PerPartitionRecordFilter" + condition: "{{ record['modify_time'] >= stream_state.get('modify_time', config.get('start_date', '')) }}" + $parameters: + partition_field: advertiser_id + + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_headers: {} + authenticator: + $ref: "#/definitions/authenticator" + request_body_json: {} + record_selector: + $ref: "#/definitions/requester" + paginator: + type: NoPagination + partition_router: [] + + paginator_page_increment: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page_size" + pagination_strategy: + type: "PageIncrement" + page_size: 100 + start_from_page: 1 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + + incremental_sync: + type: DatetimeBasedCursor + cursor_field: "modify_time" + cursor_datetime_formats: + - "%Y-%m-%d %H:%M:%S" + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%d %H:%M:%SZ" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('start_date', '2016-09-01') }}" + datetime_format: "%Y-%m-%d" + + single_id_partition_router: + - class_name: "source_tiktok_marketing.components.advertiser_ids_partition_router.SingleAdvertiserIdPerPartition" + $parameters: + path_in_config: + - ["credentials", "advertiser_id"] + - ["environment", "advertiser_id"] + partition_field: advertiser_id + parent_stream_configs: + - type: ParentStreamConfig + parent_key: advertiser_id + request_option: + inject_into: request_parameter + type: RequestOption + field_name: advertiser_id + partition_field: advertiser_id + stream: + $ref: "#/definitions/advertiser_ids_stream" + + multiple_id_partition_router: + - class_name: "source_tiktok_marketing.components.advertiser_ids_partition_router.MultipleAdvertiserIdsPerPartition" + $parameters: + path_in_config: + - ["credentials", "advertiser_id"] + - ["environment", "advertiser_id"] + partition_field: advertiser_ids + parent_stream_configs: + - type: ParentStreamConfig + parent_key: advertiser_id + request_option: + inject_into: request_parameter + type: RequestOption + field_name: advertiser_ids + partition_field: advertiser_ids + stream: + $ref: "#/definitions/advertiser_ids_stream" + + incremental_stream: + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/record_selector_with_filter_by_modify_time" + paginator: + $ref: "#/definitions/paginator_page_increment" + pagination_strategy: + type: "PageIncrement" + page_size: '{{ parameters.get("page_size", 1000) }}' + start_from_page: 1 + partition_router: + $ref: "#/definitions/single_id_partition_router" + incremental_sync: + $ref: "#/definitions/incremental_sync" + + advertiser_ids_stream: + type: DeclarativeStream + name: advertiser_ids + $parameters: + path: "oauth2/advertiser/get/" + primary_key: + - advertiser_id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/advertiser_ids" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_parameters: + secret: "{{ config.get('credentials', config.get('environment', {})).get('secret') }}" + app_id: "{{ config.get('credentials', config.get('environment', {})).get('app_id', 0) }}" + request_headers: {} + record_selector: + $ref: "#/definitions/record_selector" + paginator: + type: NoPagination + partition_router: [] + + advertisers_stream: + type: DeclarativeStream + name: advertisers + $parameters: + path: "advertiser/info/" + primary_key: + - advertiser_id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/advertisers" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/record_selector" + paginator: + $ref: "#/definitions/paginator_page_increment" + partition_router: + $ref: "#/definitions/multiple_id_partition_router" + + audiences_stream: + type: DeclarativeStream + name: audiences + $parameters: + path: "dmp/custom_audience/list/" + primary_key: + - audience_id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/audiences" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/record_selector" + paginator: + $ref: "#/definitions/paginator_page_increment" + partition_router: + $ref: "#/definitions/single_id_partition_router" + + creative_assets_music_stream: + type: DeclarativeStream + name: creative_assets_music + $parameters: + path: "file/music/get/" + primary_key: + - music_id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/creative_assets_music" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + type: RecordSelector + schema_normalization: Default + extractor: + type: DpathExtractor + field_path: ["data", "musics"] + paginator: + $ref: "#/definitions/paginator_page_increment" + partition_router: + $ref: "#/definitions/single_id_partition_router" + + creative_assets_portfolios_stream: + type: DeclarativeStream + name: creative_assets_portfolios + $parameters: + path: "creative/portfolio/list/" + primary_key: + - creative_portfolio_id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/creative_assets_portfolios" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + type: RecordSelector + schema_normalization: Default + extractor: + type: DpathExtractor + field_path: ["data", "creative_portfolios"] + paginator: + $ref: "#/definitions/paginator_page_increment" + partition_router: + $ref: "#/definitions/single_id_partition_router" + + campaigns_stream: + type: DeclarativeStream + name: campaigns + $parameters: + path: "campaign/get/" + primary_key: + - campaign_id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/campaigns" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_parameters: + filtering: '{{ {"secondary_status": "CAMPAIGN_STATUS_ALL"}|string if config.get("include_deleted", False) }}' + record_selector: + $ref: "#/definitions/record_selector_with_filter_by_modify_time" + paginator: + $ref: "#/definitions/incremental_stream/retriever/paginator" + partition_router: + $ref: "#/definitions/single_id_partition_router" + incremental_sync: + $ref: "#/definitions/incremental_sync" + + ad_groups_stream: + type: DeclarativeStream + name: ad_groups + $parameters: + path: "adgroup/get/" + primary_key: + - adgroup_id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/ad_groups" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_parameters: + filtering: '{{ {"secondary_status": "ADGROUP_STATUS_ALL"}|string if config.get("include_deleted", False) }}' + record_selector: + $ref: "#/definitions/record_selector_with_filter_by_modify_time" + paginator: + $ref: "#/definitions/incremental_stream/retriever/paginator" + partition_router: + $ref: "#/definitions/single_id_partition_router" + incremental_sync: + $ref: "#/definitions/incremental_sync" + + ads_stream: + type: DeclarativeStream + name: ads + $parameters: + path: "ad/get/" + primary_key: + - ad_id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/ads" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_parameters: + filtering: '{{ {"secondary_status": "AD_STATUS_ALL"}|string if config.get("include_deleted", False) }}' + record_selector: + $ref: "#/definitions/record_selector_with_filter_by_modify_time" + paginator: + $ref: "#/definitions/incremental_stream/retriever/paginator" + partition_router: + $ref: "#/definitions/single_id_partition_router" + incremental_sync: + $ref: "#/definitions/incremental_sync" + + creative_assets_images_stream: + type: DeclarativeStream + name: creative_assets_images + $parameters: + path: "file/image/ad/search/" + page_size: 100 + primary_key: + - image_id + $ref: "#/definitions/incremental_stream" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/creative_assets_images" + + creative_assets_videos_stream: + type: DeclarativeStream + name: creative_assets_videos + $parameters: + path: "file/video/ad/search/" + page_size: 100 + primary_key: + - video_id + $ref: "#/definitions/incremental_stream" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/creative_assets_videos" + + record_selector_for_daily_reports_streams: + $ref: "#/definitions/record_selector" + record_filter: + type: CustomRecordFilter + class_name: "source_tiktok_marketing.components.semi_incremental_record_filter.PerPartitionRecordFilter" + condition: "{{ record['dimensions']['stat_time_day'] >= stream_state.get('stat_time_day', config.get('start_date', '')) }}" + $parameters: + partition_field: advertiser_id + + record_selector_for_hourly_reports_streams: + $ref: "#/definitions/record_selector" + record_filter: + type: CustomRecordFilter + class_name: "source_tiktok_marketing.components.semi_incremental_record_filter.PerPartitionRecordFilter" + condition: "{{ record['dimensions']['stat_time_hour'] >= stream_state.get('stat_time_hour', config.get('start_date', '')) }}" + $parameters: + partition_field: advertiser_id + + base_report_retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_parameters: + service_type: "AUCTION" + report_type: "BASIC" + data_level: '{{ parameters["data_level"] }}' + dimensions: '{{ parameters["dimensions"] | string }}' + metrics: '{{ (parameters.get("report_metrics", []) + ["spend", "cpc", "cpm", "impressions", "clicks", "ctr", "reach", "cost_per_1000_reached", "frequency", "video_play_actions", "video_watched_2s", "video_watched_6s", "average_video_play", "average_video_play_per_user", "video_views_p25", "video_views_p50", "video_views_p75", "video_views_p100", "profile_visits", "likes", "comments", "shares", "follows", "clicks_on_music_disc", "real_time_app_install", "real_time_app_install_cost", "app_install"]) | string }}' + start_date: "{{ stream_interval['start_time'] }}" + end_date: "{{ stream_interval['end_time'] }}" + filters: '{{ [ + {"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"}, + ] | string if config.get("include_deleted", False)}}' + authenticator: + $ref: "#/definitions/authenticator" + request_body_json: {} + record_selector: + $ref: "#/definitions/record_selector_for_daily_reports_streams" + paginator: + $ref: "#/definitions/paginator_page_increment" + pagination_strategy: + type: "PageIncrement" + page_size: 1000 + start_from_page: 1 + partition_router: + $ref: "#/definitions/single_id_partition_router" + + report_daily_incremental_sync: + type: DatetimeBasedCursor + cursor_field: "stat_time_day" + lookback_window: "P{{ config.get('attribution_window', 0) }}D" + cursor_granularity: "P1D" + step: P30D + cursor_datetime_formats: + - "%Y-%m-%d %H:%M:%S" + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('start_date', '2016-09-01') }}" + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', today_utc()) }}" + datetime_format: "%Y-%m-%d" + + report_hourly_incremental_sync: + type: CustomIncrementalSync + class_name: source_tiktok_marketing.components.hourly_datetime_based_cursor.HourlyDatetimeBasedCursor + cursor_field: "stat_time_hour" + lookback_window: "P{{ config.get('attribution_window', 0) }}D" + cursor_granularity: "PT1H" + step: P1D + cursor_datetime_formats: + - "%Y-%m-%d %H:%M:%S" + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('start_date', '2016-09-01') }}" + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ config.get('end_date', today_utc()) }}" + datetime_format: "%Y-%m-%d" + + base_report_daily: + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/base_report" + retriever: + $ref: "#/definitions/base_report_retriever" + record_selector: + $ref: "#/definitions/record_selector_for_daily_reports_streams" + incremental_sync: + $ref: "#/definitions/report_daily_incremental_sync" + + base_report_hourly: + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/base_report" + retriever: + $ref: "#/definitions/base_report_retriever" + record_selector: + $ref: "#/definitions/record_selector_for_hourly_reports_streams" + incremental_sync: + $ref: "#/definitions/report_hourly_incremental_sync" + + base_report_lifetime: + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/base_report" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_parameters: + service_type: "AUCTION" + report_type: "BASIC" + data_level: '{{ parameters["data_level"] }}' + dimensions: '{{ parameters["dimensions"] | string }}' + metrics: '{{ (parameters.get("report_metrics", []) + ["spend", "cpc", "cpm", "impressions", "clicks", "ctr", "reach", "cost_per_1000_reached", "frequency", "video_play_actions", "video_watched_2s", "video_watched_6s", "average_video_play", "average_video_play_per_user", "video_views_p25", "video_views_p50", "video_views_p75", "video_views_p100", "profile_visits", "likes", "comments", "shares", "follows", "clicks_on_music_disc", "real_time_app_install", "real_time_app_install_cost", "app_install"]) | string }}' + query_lifetime: "true" + filters: '{{ [ + {"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"}, + ] | string if config.get("include_deleted", False)}}' + paginator: + $ref: "#/definitions/paginator_page_increment" + pagination_strategy: + type: "PageIncrement" + page_size: 1000 + start_from_page: 1 + record_selector: + $ref: "#/definitions/record_selector" + partition_router: + $ref: "#/definitions/single_id_partition_router" + + ads_reports_daily_stream: + type: DeclarativeStream + name: ads_reports_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_AD" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "secondary_goal_result", + "cost_per_secondary_goal_result", + "secondary_goal_result_rate", + "adgroup_id", + "ad_name", + "ad_text", + "total_purchase_value", + "total_onsite_shopping_value", + "onsite_shopping", + "vta_purchase", + "vta_conversion", + "cta_purchase", + "cta_conversion", + "total_pageview", + "complete_payment", + "value_per_complete_payment", + "total_complete_payment_rate", + ] + dimensions: ["ad_id", "stat_time_day"] + primary_key: + - ad_id + - stat_time_day + $ref: "#/definitions/base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["ad_id"] + value: "{{ record.dimensions.ad_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ad_groups_reports_daily_stream: + type: DeclarativeStream + name: ad_groups_reports_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADGROUP" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "secondary_goal_result", + "cost_per_secondary_goal_result", + "secondary_goal_result_rate", + ] + dimensions: ["adgroup_id", "stat_time_day"] + primary_key: + - adgroup_id + - stat_time_day + $ref: "#/definitions/base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["adgroup_id"] + value: "{{ record.dimensions.adgroup_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + advertisers_reports_daily_stream: + type: DeclarativeStream + name: advertisers_reports_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADVERTISER" + report_metrics: ["cash_spend", "voucher_spend"] + dimensions: ["advertiser_id", "stat_time_day"] + primary_key: + - advertiser_id + - stat_time_day + $ref: "#/definitions/base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["advertiser_id"] + value: "{{ record.dimensions.advertiser_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + campaigns_reports_daily_stream: + type: DeclarativeStream + name: campaigns_reports_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_CAMPAIGN" + report_metrics: ["campaign_name"] + dimensions: ["campaign_id", "stat_time_day"] + primary_key: + - campaign_id + - stat_time_day + $ref: "#/definitions/base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["campaign_id"] + value: "{{ record.dimensions.campaign_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + audience_base_report_retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_parameters: + service_type: "AUCTION" + report_type: "AUDIENCE" + data_level: '{{ parameters["data_level"] }}' + dimensions: '{{ parameters["dimensions"] | string }}' + metrics: '{{ (parameters.get("report_metrics", []) + ["spend", "cpc", "cpm", "impressions", "clicks", "ctr"]) | string }}' + start_date: "{{ stream_interval['start_time'] }}" + end_date: "{{ stream_interval['end_time'] }}" + filters: '{{ [ + {"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"}, + ] | string if config.get("include_deleted", False)}}' + authenticator: + $ref: "#/definitions/authenticator" + request_body_json: {} + record_selector: + $ref: "#/definitions/record_selector_for_daily_reports_streams" + paginator: + $ref: "#/definitions/paginator_page_increment" + pagination_strategy: + type: "PageIncrement" + page_size: 1000 + start_from_page: 1 + partition_router: + $ref: "#/definitions/single_id_partition_router" + + audience_base_report_daily: + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/audience_report" + retriever: + $ref: "#/definitions/audience_base_report_retriever" + incremental_sync: + $ref: "#/definitions/report_daily_incremental_sync" + + campaigns_audience_reports_daily_stream: + type: DeclarativeStream + name: campaigns_audience_reports_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_CAMPAIGN" + report_metrics: ["campaign_name"] + dimensions: ["campaign_id", "stat_time_day", "gender", "age"] + primary_key: + - campaign_id + - stat_time_day + - gender + - age + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["campaign_id"] + value: "{{ record.dimensions.campaign_id }}" + - path: ["gender"] + value: "{{ record.dimensions.gender }}" + - path: ["age"] + value: "{{ record.dimensions.age }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ad_group_audience_reports_daily_stream: + type: DeclarativeStream + name: ad_group_audience_reports_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADGROUP" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + ] + dimensions: ["adgroup_id", "stat_time_day", "gender", "age"] + primary_key: + - adgroup_id + - stat_time_day + - gender + - age + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["adgroup_id"] + value: "{{ record.dimensions.adgroup_id }}" + - path: ["gender"] + value: "{{ record.dimensions.gender }}" + - path: ["age"] + value: "{{ record.dimensions.age }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ads_audience_reports_daily_stream: + type: DeclarativeStream + name: ads_audience_reports_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_AD" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "adgroup_id", + "ad_name", + "ad_text", + ] + dimensions: ["ad_id", "stat_time_day", "gender", "age"] + primary_key: + - ad_id + - stat_time_day + - gender + - age + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["ad_id"] + value: "{{ record.dimensions.ad_id }}" + - path: ["gender"] + value: "{{ record.dimensions.gender }}" + - path: ["age"] + value: "{{ record.dimensions.age }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + advertisers_audience_reports_daily_stream: + type: DeclarativeStream + name: advertisers_audience_reports_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADVERTISER" + dimensions: ["advertiser_id", "stat_time_day", "gender", "age"] + primary_key: + - advertiser_id + - stat_time_day + - gender + - age + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["advertiser_id"] + value: "{{ record.dimensions.advertiser_id }}" + - path: ["gender"] + value: "{{ record.dimensions.gender }}" + - path: ["age"] + value: "{{ record.dimensions.age }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + campaigns_audience_reports_by_country_daily_stream: + type: DeclarativeStream + name: campaigns_audience_reports_by_country_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_CAMPAIGN" + report_metrics: ["campaign_name"] + dimensions: ["campaign_id", "stat_time_day", "country_code"] + primary_key: + - campaign_id + - stat_time_day + - country_code + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["campaign_id"] + value: "{{ record.dimensions.campaign_id }}" + - path: ["country_code"] + value_type: "string" + value: "{{ record.dimensions.country_code }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ad_group_audience_reports_by_country_daily_stream: + type: DeclarativeStream + name: ad_group_audience_reports_by_country_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADGROUP" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + ] + dimensions: ["adgroup_id", "stat_time_day", "country_code"] + primary_key: + - adgroup_id + - stat_time_day + - country_code + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["adgroup_id"] + value: "{{ record.dimensions.adgroup_id }}" + - path: ["country_code"] + value_type: "string" + value: "{{ record.dimensions.country_code }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ads_audience_reports_by_country_daily_stream: + type: DeclarativeStream + name: ads_audience_reports_by_country_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_AD" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "adgroup_id", + "ad_name", + "ad_text", + ] + dimensions: ["ad_id", "stat_time_day", "country_code"] + primary_key: + - ad_id + - stat_time_day + - country_code + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["ad_id"] + value: "{{ record.dimensions.ad_id }}" + - path: ["country_code"] + value_type: "string" + value: "{{ record.dimensions.country_code }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + advertisers_audience_reports_by_country_daily_stream: + type: DeclarativeStream + name: advertisers_audience_reports_by_country_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADVERTISER" + dimensions: ["advertiser_id", "stat_time_day", "country_code"] + primary_key: + - advertiser_id + - stat_time_day + - country_code + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["advertiser_id"] + value: "{{ record.dimensions.advertiser_id }}" + - path: ["country_code"] + value_type: "string" + value: "{{ record.dimensions.country_code }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + campaigns_audience_reports_by_platform_daily_stream: + type: DeclarativeStream + name: campaigns_audience_reports_by_platform_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_CAMPAIGN" + report_metrics: ["campaign_name"] + dimensions: ["campaign_id", "stat_time_day", "platform"] + primary_key: + - campaign_id + - stat_time_day + - platform + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["campaign_id"] + value: "{{ record.dimensions.campaign_id }}" + - path: ["platform"] + value_type: "string" + value: "{{ record.dimensions.platform }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ad_group_audience_reports_by_platform_daily_stream: + type: DeclarativeStream + name: ad_group_audience_reports_by_platform_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADGROUP" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + ] + dimensions: ["adgroup_id", "stat_time_day", "platform"] + primary_key: + - adgroup_id + - stat_time_day + - platform + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["adgroup_id"] + value: "{{ record.dimensions.adgroup_id }}" + - path: ["platform"] + value_type: "string" + value: "{{ record.dimensions.platform }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ads_audience_reports_by_platform_daily_stream: + type: DeclarativeStream + name: ads_audience_reports_by_platform_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_AD" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "adgroup_id", + "ad_name", + "ad_text", + ] + dimensions: ["ad_id", "stat_time_day", "platform"] + primary_key: + - ad_id + - stat_time_day + - platform + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["ad_id"] + value: "{{ record.dimensions.ad_id }}" + - path: ["platform"] + value_type: "string" + value: "{{ record.dimensions.platform }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + advertisers_audience_reports_by_platform_daily_stream: + type: DeclarativeStream + name: advertisers_audience_reports_by_platform_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADVERTISER" + dimensions: ["advertiser_id", "stat_time_day", "platform"] + primary_key: + - advertiser_id + - stat_time_day + - platform + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["advertiser_id"] + value: "{{ record.dimensions.advertiser_id }}" + - path: ["platform"] + value_type: "string" + value: "{{ record.dimensions.platform }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ads_audience_reports_by_province_daily_stream: + type: DeclarativeStream + name: ads_audience_reports_by_province_daily + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_AD" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "adgroup_id", + "ad_name", + "ad_text", + ] + dimensions: ["ad_id", "stat_time_day", "province_id"] + primary_key: + - ad_id + - stat_time_day + - province_id + $ref: "#/definitions/audience_base_report_daily" + transformations: + - type: AddFields + fields: + - path: ["stat_time_day"] + value: "{{ record.dimensions.stat_time_day }}" + - path: ["ad_id"] + value: "{{ record.dimensions.ad_id }}" + - path: ["province_id"] + value: "{{ record.dimensions.province_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ads_reports_hourly_stream: + type: DeclarativeStream + name: ads_reports_hourly + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_AD" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "secondary_goal_result", + "cost_per_secondary_goal_result", + "secondary_goal_result_rate", + "adgroup_id", + "ad_name", + "ad_text", + "total_purchase_value", + "total_onsite_shopping_value", + "onsite_shopping", + "vta_purchase", + "vta_conversion", + "cta_purchase", + "cta_conversion", + "total_pageview", + "complete_payment", + "value_per_complete_payment", + "total_complete_payment_rate", + ] + dimensions: ["ad_id", "stat_time_hour"] + primary_key: + - ad_id + - stat_time_hour + $ref: "#/definitions/base_report_hourly" + transformations: + - type: AddFields + fields: + - path: ["stat_time_hour"] + value: "{{ record.dimensions.stat_time_hour }}" + - path: ["ad_id"] + value: "{{ record.dimensions.ad_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + advertisers_reports_hourly_stream: + type: DeclarativeStream + name: advertisers_reports_hourly + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADVERTISER" + dimensions: ["advertiser_id", "stat_time_hour"] + primary_key: + - advertiser_id + - stat_time_hour + $ref: "#/definitions/base_report_hourly" + transformations: + - type: AddFields + fields: + - path: ["stat_time_hour"] + value: "{{ record.dimensions.stat_time_hour }}" + - path: ["advertiser_id"] + value: "{{ record.dimensions.advertiser_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + campaigns_reports_hourly_stream: + type: DeclarativeStream + name: campaigns_reports_hourly + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_CAMPAIGN" + report_metrics: ["campaign_name"] + dimensions: ["campaign_id", "stat_time_hour"] + primary_key: + - campaign_id + - stat_time_hour + $ref: "#/definitions/base_report_hourly" + transformations: + - type: AddFields + fields: + - path: ["stat_time_hour"] + value: "{{ record.dimensions.stat_time_hour }}" + - path: ["campaign_id"] + value: "{{ record.dimensions.campaign_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ad_groups_reports_hourly_stream: + type: DeclarativeStream + name: ad_groups_reports_hourly + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADGROUP" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "secondary_goal_result", + "cost_per_secondary_goal_result", + "secondary_goal_result_rate", + ] + dimensions: ["adgroup_id", "stat_time_hour"] + primary_key: + - adgroup_id + - stat_time_hour + $ref: "#/definitions/base_report_hourly" + transformations: + - type: AddFields + fields: + - path: ["stat_time_hour"] + value: "{{ record.dimensions.stat_time_hour }}" + - path: ["adgroup_id"] + value: "{{ record.dimensions.adgroup_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ads_reports_lifetime_stream: + type: DeclarativeStream + name: ads_reports_lifetime + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_AD" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "secondary_goal_result", + "cost_per_secondary_goal_result", + "secondary_goal_result_rate", + "adgroup_id", + "ad_name", + "ad_text", + "total_purchase_value", + "total_onsite_shopping_value", + "onsite_shopping", + "vta_purchase", + "vta_conversion", + "cta_purchase", + "cta_conversion", + "total_pageview", + "complete_payment", + "value_per_complete_payment", + "total_complete_payment_rate", + ] + dimensions: ["ad_id"] + primary_key: + - ad_id + $ref: "#/definitions/base_report_lifetime" + transformations: + - type: AddFields + fields: + - path: ["ad_id"] + value: "{{ record.dimensions.ad_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + advertisers_reports_lifetime_stream: + type: DeclarativeStream + name: advertisers_reports_lifetime + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADVERTISER" + dimensions: ["advertiser_id"] + primary_key: + - advertiser_id + $ref: "#/definitions/base_report_lifetime" + transformations: + - type: AddFields + fields: + - path: ["advertiser_id"] + value: "{{ record.dimensions.advertiser_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + campaigns_reports_lifetime_stream: + type: DeclarativeStream + name: campaigns_reports_lifetime + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_CAMPAIGN" + report_metrics: ["campaign_name"] + dimensions: ["campaign_id"] + primary_key: + - campaign_id + $ref: "#/definitions/base_report_lifetime" + transformations: + - type: AddFields + fields: + - path: ["campaign_id"] + value: "{{ record.dimensions.campaign_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + ad_groups_reports_lifetime_stream: + type: DeclarativeStream + name: ad_groups_reports_lifetime + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADGROUP" + report_metrics: + [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "secondary_goal_result", + "cost_per_secondary_goal_result", + "secondary_goal_result_rate", + ] + dimensions: ["adgroup_id"] + primary_key: + - adgroup_id + $ref: "#/definitions/base_report_lifetime" + transformations: + - type: AddFields + fields: + - path: ["adgroup_id"] + value: "{{ record.dimensions.adgroup_id }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + audience_base_report_lifetime: + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/definitions/schemas/audience_report" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + request_parameters: + service_type: "AUCTION" + report_type: "AUDIENCE" + data_level: '{{ parameters["data_level"] }}' + dimensions: '{{ parameters["dimensions"] | string }}' + metrics: '{{ (parameters.get("report_metrics", []) + ["spend", "cpc", "cpm", "impressions", "clicks", "ctr"]) | string }}' + start_date: '{{ day_delta(-365, "%Y-%m-%d") if config.get("start_date", "2016-09-01") < day_delta(-365, "%Y-%m-%d") else config["start_date"] }}' + end_date: "{{ today_utc() }}" + lifetime: "true" + filters: '{{ [ + {"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"}, + ] | string if config.get("include_deleted", False)}}' + record_selector: + $ref: "#/definitions/record_selector" + paginator: + $ref: "#/definitions/paginator_page_increment" + pagination_strategy: + type: "PageIncrement" + page_size: 1000 + start_from_page: 1 + partition_router: + $ref: "#/definitions/single_id_partition_router" + + advertisers_audience_reports_lifetime_stream: + type: DeclarativeStream + name: advertisers_audience_reports_lifetime + $parameters: + path: "report/integrated/get/" + data_level: "AUCTION_ADVERTISER" + dimensions: ["advertiser_id", "gender", "age"] + primary_key: + - advertiser_id + - gender + - age + $ref: "#/definitions/audience_base_report_lifetime" + transformations: + - type: AddFields + fields: + - path: ["advertiser_id"] + value: "{{ record.dimensions.advertiser_id }}" + - path: ["gender"] + value: "{{ record.dimensions.gender }}" + - path: ["age"] + value: "{{ record.dimensions.age }}" + - type: CustomTransformation + class_name: "source_tiktok_marketing.components.transformations.TransformEmptyMetrics" + + schemas: + advertiser_ids: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + advertiser_id: + description: + The unique identifier for each advertiser in the TikTok marketing + platform. + type: string + advertiser_name: + description: The name of the advertiser registered in the TikTok marketing platform. + type: string + advertisers: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + advertiser_id: + description: Unique identifier for the advertiser. + type: integer + name: + description: The name of the advertiser or company. + type: string + address: + description: The physical address of the advertiser. + type: + - "null" + - string + company: + description: The name of the company associated with the advertiser. + type: + - "null" + - string + contacter: + description: The contact person for the advertiser. + type: + - "null" + - string + country: + description: The country where the advertiser is located. + type: + - "null" + - string + currency: + description: The currency used for transactions in the account. + type: + - "null" + - string + description: + description: A brief description or bio of the advertiser or company. + type: + - "null" + - string + email: + description: The email address associated with the advertiser. + type: + - "null" + - string + industry: + description: The industry or sector the advertiser operates in. + type: + - "null" + - string + language: + description: The preferred language of communication for the advertiser. + type: + - "null" + - string + license_no: + description: The license number of the advertiser. + type: + - "null" + - string + license_url: + description: The URL link to the advertiser's license documentation. + type: + - "null" + - string + cellphone_number: + description: The cellphone number of the advertiser. + type: + - "null" + - string + promotion_area: + description: The specific area or region where the advertiser focuses promotion. + type: + - "null" + - string + rejection_reason: + description: Reason for any advertisement rejection by the platform. + type: + - "null" + - string + role: + description: The role or position of the advertiser within the company. + type: + - "null" + - string + status: + description: The current status of the advertiser's account. + type: + - "null" + - string + timezone: + description: The timezone setting for the advertiser's activities. + type: + - "null" + - string + balance: + description: The current balance in the advertiser's account. + type: number + create_time: + description: The timestamp when the advertiser account was created. + type: integer + telephone_number: + description: The telephone number of the advertiser. + type: + - "null" + - string + display_timezone: + description: The timezone for display purposes. + type: + - "null" + - string + promotion_center_province: + description: + The province or state at the center of the advertiser's promotion + activities. + type: + - "null" + - string + advertiser_account_type: + description: The type of advertiser's account (e.g., individual, business). + type: + - "null" + - string + license_city: + description: The city where the advertiser's license is registered. + type: + - "null" + - string + brand: + description: The brand name associated with the advertiser. + type: + - "null" + - string + license_province: + description: The province or state where the advertiser's license is registered. + type: + - "null" + - string + promotion_center_city: + description: The city at the center of the advertiser's promotion activities. + type: + - "null" + - string + audiences: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + shared: + description: Flag indicating if the audience is shared with others + type: + - "null" + - boolean + is_creator: + description: Flag indicating if the audience creator is the user + type: + - "null" + - boolean + audience_id: + description: Unique identifier for the audience + type: + - "null" + - string + cover_num: + description: Number of audience members covered + type: + - "null" + - integer + create_time: + description: Timestamp indicating when the audience was created + type: + - "null" + - string + format: date-time + is_valid: + description: Flag indicating if the audience data is valid + type: + - "null" + - boolean + is_expiring: + description: Flag indicating if the audience data is expiring soon + type: + - "null" + - boolean + expired_time: + description: Timestamp indicating when the audience data expires + type: + - "null" + - string + format: date-time + name: + description: Name of the audience + type: + - "null" + - string + audience_type: + description: Type of audience (e.g., demographic, interest-based) + type: + - "null" + - string + calculate_type: + description: Method used to calculate audience data + type: + - "null" + - string + creative_assets_music: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + music_id: + description: The unique identifier for the music asset. + type: + - "null" + - string + material_id: + description: The unique ID assigned to the music asset. + type: + - "null" + - string + sources: + description: + The list of different sources or versions available for the music + asset. + type: + - "null" + - array + items: + type: + - "null" + - string + author: + description: The author of the music asset. + type: + - "null" + - string + liked: + description: The number of likes received by the music asset. + type: + - "null" + - boolean + cover_url: + description: The URL to the cover image associated with the music asset. + type: + - "null" + - string + url: + description: The URL to access or play the music asset. + type: + - "null" + - string + duration: + description: The duration of the music asset in seconds. + type: + - "null" + - number + style: + description: The style or genre of the music asset. + type: + - "null" + - string + signature: + description: The digital signature associated with the music asset. + type: + - "null" + - string + name: + description: The name or title of the music asset. + type: + - "null" + - string + file_name: + description: The file name of the music asset. + type: + - "null" + - string + copyright: + description: The copyright information related to the music asset. + type: + - "null" + - string + create_time: + description: The timestamp indicating when the music asset was created. + type: + - "null" + - string + format: date-time + modify_time: + description: The timestamp indicating when the music asset was last modified. + type: + - "null" + - string + format: date-time + creative_assets_portfolios: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + creative_portfolio_id: + description: The unique identifier for the creative portfolio. + type: + - "null" + - string + creative_portfolio_type: + description: The type of the creative portfolio, such as image, video, or carousel. + type: + - "null" + - string + creative_portfolio_preview_url: + description: The URL pointing to a preview image or video of the creative portfolio. + type: + - "null" + - string + campaigns: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + campaign_id: + description: The unique identifier of the campaign + type: integer + campaign_name: + description: Name of the campaign for easy identification + type: string + campaign_type: + description: Type of campaign (e.g., awareness, conversion) + type: string + advertiser_id: + description: The unique identifier of the advertiser associated with the campaign + type: integer + budget: + description: Total budget allocated for the campaign + type: number + budget_mode: + description: Mode in which the budget is being managed (e.g., daily, lifetime) + type: string + secondary_status: + description: Additional status information of the campaign + type: string + operation_status: + description: Current operational status of the campaign (e.g., active, paused) + type: + - "null" + - string + objective: + description: The objective or goal of the campaign + type: + - "null" + - string + objective_type: + description: + Type of objective selected for the campaign (e.g., brand awareness, + app installs) + type: + - "null" + - string + budget_optimize_on: + description: The metric or event that the budget optimization is based on + type: + - "null" + - boolean + bid_type: + description: Type of bid strategy being used in the campaign + type: + - "null" + - string + deep_bid_type: + description: Advanced bid type used for campaign optimization + type: + - "null" + - string + optimization_goal: + description: Specific goal to be optimized for in the campaign + type: + - "null" + - string + split_test_variable: + description: Variable being tested in a split test campaign + type: + - "null" + - string + is_new_structure: + description: Flag indicating if the campaign utilizes a new campaign structure + type: boolean + create_time: + description: Timestamp when the campaign was created + type: string + format: date-time + airbyte_type: timestamp_without_timezone + modify_time: + description: Timestamp when the campaign was last modified + type: string + format: date-time + airbyte_type: timestamp_without_timezone + roas_bid: + description: Return on ad spend goal set for the campaign + type: + - "null" + - number + is_smart_performance_campaign: + description: Flag indicating if the campaign uses smart performance optimization + type: + - "null" + - boolean + is_search_campaign: + description: Flag indicating if the campaign is a search campaign + type: + - "null" + - boolean + app_promotion_type: + description: Type of app promotion being used in the campaign + type: + - "null" + - string + rf_campaign_type: + description: Type of RF (reach and frequency) campaign being run + type: + - "null" + - string + ad_groups: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + adgroup_id: + description: The unique identifier of the ad group + type: integer + campaign_id: + description: The unique identifier of the campaign + type: integer + advertiser_id: + description: The unique identifier of the advertiser + type: integer + adgroup_name: + description: The name of the ad group + type: string + placement_type: + description: The type of ad placement + type: string + enum: + - PLACEMENT_TYPE_AUTOMATIC + - PLACEMENT_TYPE_NORMAL + placements: + description: Information about the ad placements targeted + type: + - "null" + - array + items: + type: string + inventory_filter_enabled: + description: Flag indicating if inventory filter is enabled + type: + - "null" + - boolean + comment_disabled: + description: Flag indicating if comments are disabled + type: boolean + app_id: + description: The unique identifier of the app + type: + - "null" + - integer + promotion_type: + description: The type of promotion + type: string + enum: + - APP_ANDROID + - APP_IOS + - WEBSITE + - LEAD_GENERATION + - WEBSITE_OR_DISPLAY + - TIKTOK_SHOP + - VIDEO_SHOPPING + - LIVE_SHOPPING + app_download_url: + description: The URL for downloading the associated app + type: + - "null" + - string + package: + description: The package used for the ad group + type: + - "null" + - string + pixel_id: + description: The ID of the pixel used for tracking + type: + - "null" + - integer + optimization_event: + description: The event used for optimization + type: + - "null" + - string + secondary_optimization_event: + description: Additional event used for optimization + type: + - "null" + - string + creative_material_mode: + description: The mode for creative materials + type: string + modify_time: + description: The timestamp for when the ad group was last modified + type: string + format: date-time + airbyte_type: timestamp_without_timezone + create_time: + description: The timestamp for when the ad group was created + type: string + format: date-time + airbyte_type: timestamp_without_timezone + audience_ids: + description: The IDs of the targeted audience + type: array + items: + type: integer + excluded_audience_ids: + description: The IDs of excluded audiences + type: array + items: + type: integer + audience_type: + description: The type of audience being targeted + type: + - "null" + - string + location_ids: + description: The IDs of targeted locations + type: array + items: + type: integer + is_hfss: + description: Flag indicating if high-frequency short sequences are included + type: boolean + interest_category_ids: + description: The IDs of interest categories for targeting + type: array + items: + type: integer + interest_keyword_ids: + description: The IDs of interest keywords for targeting + type: array + items: + type: integer + age_groups: + description: The targeted age groups for the ad group + type: + - "null" + - array + items: + type: string + gender: + description: The targeted gender for the ad group + type: + - "null" + - string + languages: + description: The targeted languages for the ad group + type: array + items: + type: string + operating_systems: + description: The targeted operating systems + type: array + items: + type: string + network_types: + description: The types of networks targeted + type: array + items: + type: string + device_price_ranges: + description: The price ranges for devices + type: + - "null" + - array + items: + type: number + min_android_version: + description: The minimum required Android version + type: + - "null" + - string + ios14_targeting: + description: Information about iOS 14 targeting settings + type: + - "null" + - string + device_model_ids: + description: The IDs of targeted device models + type: + - "null" + - array + items: + type: integer + min_ios_version: + description: The minimum required iOS version + type: + - "null" + - string + budget_mode: + description: The mode for managing the budget + type: string + budget: + description: The allocated budget for the ad group + type: number + schedule_type: + description: The type of scheduling + type: string + schedule_start_time: + description: The start time of the scheduling + type: string + format: date-time + airbyte_type: timestamp_without_timezone + schedule_end_time: + description: The end time of the scheduling + type: string + format: date-time + airbyte_type: timestamp_without_timezone + dayparting: + description: Information about dayparting settings + type: + - "null" + - string + optimization_goal: + description: The goal set for optimization + type: string + cpv_video_duration: + description: The duration for cost-per-view video + type: + - "null" + - string + conversion_window: + description: The window for tracking conversions + type: + - "null" + - string + pacing: + description: Information about the pacing settings + type: + - "null" + - string + billing_event: + description: The event used for billing + type: + - "null" + - string + skip_learning_phase: + description: Flag indicating if the learning phase is skipped + type: integer + bid_type: + description: The type of bidding + type: + - "null" + - string + bid_price: + description: The price set for bidding + type: number + conversion_bid_price: + description: The bid price for conversions + type: number + deep_bid_type: + description: The type of deep bid strategy + type: + - "null" + - string + deep_cpa_bid: + description: The bid amount for deep cost-per-action + type: number + secondary_status: + description: The secondary status of the ad group + type: string + operation_status: + description: The status of the operation + type: string + frequency: + description: The frequency of ad display + type: + - "null" + - integer + frequency_schedule: + description: The schedule for frequency capping + type: + - "null" + - integer + statistic_type: + description: The type of statistics being tracked + type: + - "null" + - string + carrier_ids: + description: The IDs of the targeted carriers + type: + - "null" + - array + items: + type: integer + carriers: + description: Information about the targeted carriers + type: + - "null" + - array + items: + type: string + video_download_disabled: + description: Flag indicating if video downloads are disabled + type: boolean + blocked_pangle_app_ids: + description: The IDs of the blocked Pangle apps + type: + - "null" + - array + items: + type: string + action_category_ids: + description: The IDs of the action categories associated with the ad group + type: + - "null" + - array + items: + type: string + action_days: + description: The number of days the action has been performed + type: + - "null" + - integer + video_actions: + description: Information about video-specific actions + type: + - "null" + - array + items: + type: string + rf_purchased_type: + description: Type of purchased results + type: + - "null" + - string + purchased_impression: + description: Information about purchased impressions + type: + - "null" + - number + purchased_reach: + description: Information about purchased reach + type: + - "null" + - number + rf_estimated_cpr: + description: Estimated cost per result + type: + - "null" + - number + rf_estimated_frequency: + description: Estimated frequency of results + type: + - "null" + - number + included_pangle_audience_package_ids: + description: The IDs of included Pangle audience packages + type: + - "null" + - array + items: + type: number + excluded_pangle_audience_package_ids: + description: The IDs of excluded Pangle audience packages + type: + - "null" + - array + items: + type: number + is_new_structure: + description: Flag indicating if the ad group follows a new structure + type: boolean + is_smart_performance_campaign: + description: Flag indicating if the campaign is using smart performance + type: + - "null" + - boolean + catalog_id: + description: The unique identifier of the catalog + type: + - "null" + - integer + product_set_id: + description: The ID of the product set + type: + - "null" + - integer + catalog_authorized_bc_id: + description: The authorized Business Center ID for the catalog + type: + - "null" + - integer + audience_rule: + description: The rule set for targeting the audience + type: + - "null" + - object + included_custom_actions: + description: Custom actions that are included + type: + - "null" + - array + items: + type: object + excluded_custom_actions: + description: Custom actions that are excluded + type: + - "null" + - array + items: + type: object + shopping_ads_retargeting_type: + description: The type of retargeting used for shopping ads + type: + - "null" + - string + split_test_adgroup_ids: + description: The IDs of ad groups participating in split testing + type: + - "null" + - array + items: + type: number + brand_safety_type: + description: The type of brand safety measures + type: + - "null" + - string + brand_safety_partner: + description: Information about the brand safety partners + type: + - "null" + - string + promotion_website_type: + description: The type of website used for promotion + type: + - "null" + - string + ios_quota_type: + description: The type of iOS quota + type: + - "null" + - string + roas_bid: + description: The bid amount set for return on ad spend + type: + - "null" + - number + actions: + description: Information about the actions taken on the ad group + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + action_category_ids: + description: The IDs of the action categories for the specific action + type: + - "null" + - array + items: + type: integer + action_period: + description: The period during which the action was taken + type: + - "null" + - number + action_scene: + description: The scene in which the action took place + type: + - "null" + - string + video_user_actions: + description: User actions specific to video content + type: + - "null" + - array + items: + type: string + targeting_expansion: + description: Settings for targeting expansion + type: + - "null" + - object + properties: + expansion_enabled: + description: Flag indicating if targeting expansion is enabled + type: boolean + expansion_types: + description: Types of expansion enabled + type: + - "null" + - array + items: + type: string + schedule_infos: + description: Information about the scheduling arrangements + type: + - "null" + - array + items: + type: object + share_disabled: + description: Flag indicating if sharing is disabled + type: + - "null" + - boolean + auto_targeting_enabled: + description: Flag indicating if auto-targeting is enabled + type: + - "null" + - boolean + ios14_quota_type: + description: The type of iOS 14 quota + type: + - "null" + - string + campaign_name: + description: The name of the campaign + type: + - "null" + - string + bid_display_mode: + description: The display mode for bidding + type: + - "null" + - string + scheduled_budget: + description: The budget allocated for scheduling + type: + - "null" + - number + adgroup_app_profile_page_state: + description: The state of the app profile page related to the ad group + type: + - "null" + - string + keywords: + description: Keywords associated with the ad group + type: + - "null" + - string + next_day_retention: + description: Retention information for the next day + type: + - "null" + - number + category_id: + description: The ID of the category for the ad group + type: + - "null" + - integer + search_result_enabled: + description: Flag indicating if search results are enabled + type: + - "null" + - boolean + app_type: + description: The type of the associated app + type: + - "null" + - string + feed_type: + description: The type of feed used + type: + - "null" + - string + delivery_mode: + description: The mode for delivery + type: + - "null" + - string + category_exclusion_ids: + description: The IDs of the excluded categories + type: + - "null" + - array + items: + type: string + contextual_tag_ids: + description: The IDs of contextual tags for targeting + type: + - "null" + - array + items: + type: + - "null" + - string + household_income: + description: The targeted household income groups + type: + - "null" + - array + items: + type: + - "null" + - string + isp_ids: + description: The IDs of the targeted internet service providers + type: + - "null" + - array + items: + type: + - "null" + - string + spending_power: + description: Information about the spending power targeted + type: + - "null" + - string + zipcode_ids: + description: The IDs of targeted ZIP codes + type: + - "null" + - array + items: + type: + - "null" + - string + ads: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + advertiser_id: + description: The unique identifier of the advertiser + type: integer + campaign_id: + description: The unique identifier of the campaign + type: integer + campaign_name: + description: The name of the campaign + type: string + adgroup_id: + description: The unique identifier of the ad group + type: integer + adgroup_name: + description: The name of the ad group + type: string + ad_id: + description: The unique identifier of the ad + type: integer + ad_name: + description: The name of the ad + type: string + tracking_app_id: + description: The unique identifier of the tracking app + type: + - "null" + - string + tracking_offline_event_set_ids: + description: The unique identifiers of offline event sets for tracking + type: + - "null" + - array + items: + description: Unique identifier of an offline event set + type: + - "null" + - string + call_to_action: + description: The call-to-action text for the ad + type: + - "null" + - string + call_to_action_id: + description: The identifier of the call-to-action + type: + - "null" + - string + disclaimer_type: + description: The type of disclaimer displayed + type: + - "null" + - string + disclaimer_text: + description: The disclaimer text + type: + - "null" + - object + properties: + text: + description: The text of the disclaimer + type: + - "null" + - string + disclaimer_clickable_texts: + description: Clickable disclaimer texts with URLs + type: + - "null" + - object + properties: + text: + description: The disclaimer text + type: + - "null" + - string + url: + description: The URL associated with the disclaimer text + type: + - "null" + - string + card_id: + description: The identifier of the card + type: + - "null" + - integer + secondary_status: + description: The secondary status of the ad + type: string + operation_status: + description: The operational status of the ad + type: + - "null" + - string + is_aco: + description: Indicates if the ad is under Automated Creative Optimization + type: + - "null" + - boolean + image_ids: + description: The unique identifiers of images used in the ad + type: + - "null" + - array + items: + description: Unique identifier of an image + type: string + image_mode: + description: The mode of displaying images + type: + - "null" + - string + ad_format: + description: The format of the ad (e.g., image, video, carousel) + type: + - "null" + - string + ad_text: + description: The text content of the ad + type: + - "null" + - string + ad_texts: + description: The text content of the ad in various languages + type: + - "null" + - array + items: + description: Text content in a specific language + type: string + video_id: + description: The unique identifier of the video + type: + - "null" + - string + tiktok_item_id: + description: The unique identifier of the TikTok item + type: + - "null" + - string + premium_badge_id: + description: The unique identifier of the premium badge + type: + - "null" + - string + app_name: + description: The name of the mobile app where the ad is displayed + type: + - "null" + - string + landing_page_url: + description: The URL of the landing page for the ad + type: + - "null" + - string + landing_page_urls: + description: The URLs of landing pages for the ad + type: + - "null" + - array + items: + description: URL of a landing page + type: string + display_name: + description: The display name of the ad + type: + - "null" + - string + profile_image_url: + description: The URL of the profile image associated with the ad + type: + - "null" + - string + impression_tracking_url: + description: The URL for tracking ad impressions + type: + - "null" + - string + click_tracking_url: + description: The URL for tracking ad clicks + type: + - "null" + - string + tracking_pixel_id: + description: The unique identifier of the tracking pixel + type: + - "null" + - integer + deeplink: + description: The deeplink URL for the ad + type: + - "null" + - string + deeplink_type: + description: The type of deeplink used + type: + - "null" + - string + fallback_type: + description: The type of fallback used + type: + - "null" + - string + playable_url: + description: The URL for a playable ad + type: + - "null" + - string + vast_moat_enabled: + description: Indicates if VAST MOAT is enabled + type: + - "null" + - boolean + page_id: + description: The unique identifier of the page + type: + - "null" + - number + creative_authorized: + description: Indicates if the creative is authorized + type: + - "null" + - boolean + is_new_structure: + description: Indicates if the ad is part of a new structure + type: + - "null" + - boolean + create_time: + description: The timestamp when the ad was created + type: string + format: date-time + airbyte_type: timestamp_without_timezone + modify_time: + description: The timestamp when the ad was last modified + type: string + format: date-time + airbyte_type: timestamp_without_timezone + shopping_ads_fallback_type: + description: The type of fallback for shopping ads + type: + - "null" + - string + shopping_deeplink_type: + description: The type of deeplink for shopping + type: + - "null" + - string + shopping_ads_video_package_id: + description: The unique identifier of the video package for shopping ads + type: + - "null" + - string + promotional_music_disabled: + description: Indicates if promotional music is disabled + type: + - "null" + - boolean + item_duet_status: + description: The status of item duet + type: + - "null" + - string + item_stitch_status: + description: The status of item stitch + type: + - "null" + - string + avatar_icon_web_uri: + description: The URL of the avatar icon for the ad + type: + - "null" + - string + brand_safety_postbid_partner: + description: Details about post-bidding partner for brand safety + type: + - "null" + - string + brand_safety_vast_url: + description: The VAST URL for brand safety tracking + type: + - "null" + - string + creative_type: + description: The type of creative used in the ad + type: + - "null" + - string + identity_id: + description: The identifier of the identity + type: + - "null" + - string + identity_type: + description: The type of identity + type: + - "null" + - string + identity_authorized_bc_id: + description: The authorized identity for branded content + type: + - "null" + - string + phone_region_code: + description: The region code for the phone number + type: + - "null" + - string + phone_region_calling_code: + description: The calling code region for the phone number + type: + - "null" + - string + optimization_event: + description: The event used for optimization + type: + - "null" + - string + phone_number: + description: The phone number associated with the ad + type: + - "null" + - string + carousel_image_index: + description: The index of the image in a carousel ad + type: + - "null" + - integer + viewability_postbid_partner: + description: Details about post-bidding partner for viewability tracking + type: + - "null" + - string + viewability_vast_url: + description: The VAST URL for viewability tracking + type: + - "null" + - string + music_id: + description: The unique identifier of the music used in the ad + type: + - "null" + - string + utm_params: + description: UTM parameters for tracking + type: + - "null" + - array + items: + description: Key-value pair for a UTM parameter + type: + - "null" + - object + properties: + key: + description: The key of the UTM parameter + type: + - "null" + - string + value: + description: The value of the UTM parameter + type: + - "null" + - string + shopping_ads_deeplink_type: + description: The type of deeplink for shopping ads + type: + - "null" + - string + dark_post_status: + description: The status of dark post + type: + - "null" + - string + branded_content_disabled: + description: Indicates if branded content is disabled + type: + - "null" + - string + product_specific_type: + description: The specific type of product + type: + - "null" + - string + catalog_id: + description: The unique identifier of the catalog + type: + - "null" + - string + item_group_ids: + description: The unique identifiers of item groups + type: + - "null" + - array + items: + description: Unique identifier of an item group + type: + - "null" + - string + product_set_id: + description: The unique identifier of the product set + type: + - "null" + - string + sku_ids: + description: The unique identifiers of SKUs associated with the ad + type: + - "null" + - array + items: + description: Unique identifier of a SKU + type: + - "null" + - string + dynamic_format: + description: The dynamic format of the ad + type: + - "null" + - string + vertical_video_strategy: + description: The strategy for displaying vertical videos + type: + - "null" + - string + dynamic_destination: + description: The dynamic destination of the ad + type: + - "null" + - string + showcase_products: + description: Products displayed in a showcase ad + type: + - "null" + - object + properties: + item_group_id: + description: The unique identifier of the item group + type: + - "null" + - string + store_id: + description: The unique identifier of the store + type: + - "null" + - string + catalog_id: + description: The unique identifier of the catalog + type: + - "null" + - string + tiktok_page_category: + description: The category of the TikTok page + type: + - "null" + - string + creative_assets_images: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + image_id: + description: The unique identifier for the image. + type: + - "null" + - string + format: + description: The format type of the image file. + type: + - "null" + - string + image_url: + description: The URL to access the image. + type: + - "null" + - string + height: + description: The height dimension of the image. + type: + - "null" + - integer + width: + description: The width dimension of the image. + type: + - "null" + - integer + signature: + description: The signature of the image for security purposes. + type: + - "null" + - string + size: + description: The size of the image file. + type: + - "null" + - integer + material_id: + description: The ID associated with the material of the image. + type: + - "null" + - string + is_carousel_usable: + description: Flag indicating if the image can be used in a carousel. + type: + - "null" + - boolean + file_name: + description: The name of the image file. + type: + - "null" + - string + create_time: + description: The timestamp when the creative asset image was created. + type: + - "null" + - string + format: date-time + modify_time: + description: The timestamp when the creative asset image was last modified. + type: + - "null" + - string + format: date-time + displayable: + description: Flag indicating if the image is displayable or not. + type: + - "null" + - boolean + creative_assets_videos: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + video_id: + description: ID of the video. + type: + - "null" + - string + video_cover_url: + description: URL for the cover image of the video. + type: + - "null" + - string + format: + description: Format of the video file. + type: + - "null" + - string + preview_url: + description: URL for previewing the video. + type: + - "null" + - string + preview_url_expire_time: + description: Timestamp when the preview URL expires. + type: + - "null" + - string + format: date-time + airbyte_type: timestamp_without_timezone + duration: + description: Duration of the video in seconds. + type: + - "null" + - number + height: + description: Height of the video in pixels. + type: + - "null" + - integer + width: + description: Width of the video in pixels. + type: + - "null" + - integer + bit_rate: + description: The bitrate of the video. + type: + - "null" + - number + signature: + description: Signature for authenticating the video request. + type: + - "null" + - string + size: + description: Size of the video file in bytes. + type: + - "null" + - integer + material_id: + description: ID of the video material. + type: + - "null" + - string + allowed_placements: + description: List of placements where the video can be used. + type: + - "null" + - array + items: + description: Specific placement where the video is allowed. + type: + - "null" + - string + allow_download: + description: Indicates if the video can be downloaded by users. + type: + - "null" + - boolean + file_name: + description: Name of the video file. + type: + - "null" + - string + create_time: + description: Timestamp when the video was created. + type: + - "null" + - string + format: date-time + modify_time: + description: Timestamp when the video was last modified. + type: + - "null" + - string + format: date-time + displayable: + description: Indicates if the video is displayable. + type: + - "null" + - boolean + base_report: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + stat_time_day: + description: The date for which the statistical data is recorded. + type: + - "null" + - string + format: date-time + airbyte_type: timestamp_without_timezone + stat_time_hour: + description: The hour of the day for which the statistical data is recorded. + type: + - "null" + - string + format: date-time + airbyte_type: timestamp_without_timezone + campaign_id: + description: The unique identifier for a marketing campaign. + type: + - "null" + - integer + adgroup_id: + description: The unique identifier for an ad group. + type: + - "null" + - integer + ad_id: + description: The unique identifier for an advertisement. + type: + - "null" + - integer + advertiser_id: + description: The unique identifier for an advertiser. + type: + - "null" + - integer + metrics: + description: + A list of metrics for which data should be retrieved such as views, + likes, comments, or shares. + type: + - "null" + - object + properties: + campaign_name: + description: The name of the marketing campaign. + type: + - "null" + - string + campaign_id: + description: + The unique identifier for a marketing campaign within the metrics + level. + type: + - "null" + - integer + adgroup_name: + description: The name of the ad group. + type: + - "null" + - string + placement_type: + description: Type of advertisement placement. + type: + - "null" + - string + adgroup_id: + description: The unique identifier for an ad group within the metrics level. + type: + - "null" + - integer + ad_name: + description: The name of the advertisement. + type: + - "null" + - string + ad_text: + description: The content or text of the advertisement. + type: + - "null" + - string + tt_app_id: + description: The unique identifier for a TikTok application. + type: + - "null" + - integer + tt_app_name: + description: The name of the TikTok application. + type: + - "null" + - string + mobile_app_id: + description: The unique identifier for a mobile application. + type: + - "null" + - string + promotion_type: + description: Type of promotion. + type: + - "null" + - string + dpa_target_audience_type: + description: Dynamic product ad target audience type. + type: + - "null" + - string + spend: + description: Amount of money spent. + type: + - "null" + - string + cash_spend: + description: The amount of money spent in cash. + type: + - "null" + - string + voucher_spend: + description: Amount spent on vouchers. + type: + - "null" + - string + cpc: + description: Cost per click. + type: + - "null" + - string + cpm: + description: Cost per thousand impressions. + type: + - "null" + - string + impressions: + description: Number of times the advertisement is viewed. + type: + - "null" + - string + clicks: + description: The number of clicks on the advertisement. + type: + - "null" + - string + ctr: + description: Click-through rate. + type: + - "null" + - string + reach: + description: Total number of unique users reached. + type: + - "null" + - string + cost_per_1000_reached: + description: The cost per 1000 reached users. + type: + - "null" + - string + conversion: + description: The number of conversions. + type: + - "null" + - string + cost_per_conversion: + description: The cost per conversion. + type: + - "null" + - string + conversion_rate: + description: The rate of conversion. + type: + - "null" + - string + real_time_conversion: + description: Real-time conversions. + type: + - "null" + - string + real_time_cost_per_conversion: + description: Cost per conversion in real-time. + type: + - "null" + - string + real_time_conversion_rate: + description: Real-time conversion rate. + type: + - "null" + - string + result: + description: Number of results. + type: + - "null" + - string + cost_per_result: + description: The cost per result. + type: + - "null" + - string + result_rate: + description: Rate of results. + type: + - "null" + - string + real_time_result: + description: Real-time results. + type: + - "null" + - string + real_time_cost_per_result: + description: Cost per result in real-time. + type: + - "null" + - string + real_time_result_rate: + description: Real-time result rate. + type: + - "null" + - string + secondary_goal_result: + description: Results for secondary goals. + type: + - "null" + - string + cost_per_secondary_goal_result: + description: The cost per secondary goal result. + type: + - "null" + - string + secondary_goal_result_rate: + description: Rate of results for secondary goals. + type: + - "null" + - string + frequency: + description: Frequency of occurrence. + type: + - "null" + - string + total_purchase_value: + description: Total value of purchases made. + type: + - "null" + - string + total_onsite_shopping_value: + description: Total value of onsite shopping. + type: + - "null" + - string + onsite_shopping: + description: Shopping happening on the site. + type: + - "null" + - string + vta_purchase: + description: Purchase through vertical takeoff ad (VTA). + type: + - "null" + - string + cta_purchase: + description: Purchase through call-to-action. + type: + - "null" + - string + cta_conversion: + description: Conversion through call-to-action. + type: + - "null" + - string + vta_conversion: + description: Conversion through vertical takeoff ad (VTA). + type: + - "null" + - string + total_pageview: + description: Total number of page views. + type: + - "null" + - string + complete_payment: + description: The number of completed payments. + type: + - "null" + - string + value_per_complete_payment: + description: Value per completed payment. + type: + - "null" + - string + total_complete_payment_rate: + description: Rate of total completed payments. + type: + - "null" + - string + video_play_actions: + description: Actions related to video plays. + type: + - "null" + - number + video_watched_2s: + description: Number of viewers watching at least 2 seconds of the video. + type: + - "null" + - number + video_watched_6s: + description: Number of viewers watching at least 6 seconds of the video. + type: + - "null" + - number + average_video_play: + description: The average number of video plays. + type: + - "null" + - number + average_video_play_per_user: + description: The average number of video plays per user. + type: + - "null" + - number + video_views_p25: + description: Percentage of viewers watching at least 25% of the video. + type: + - "null" + - number + video_views_p50: + description: Percentage of viewers watching at least 50% of the video. + type: + - "null" + - number + video_views_p75: + description: Percentage of viewers watching at least 75% of the video. + type: + - "null" + - number + video_views_p100: + description: Percentage of viewers watching the entire video. + type: + - "null" + - number + profile_visits: + description: Number of visits to the profile. + type: + - "null" + - number + likes: + description: Number of likes received. + type: + - "null" + - number + comments: + description: The number of comments received. + type: + - "null" + - number + shares: + description: Number of shares. + type: + - "null" + - number + follows: + description: Number of follows. + type: + - "null" + - number + clicks_on_music_disc: + description: The number of clicks on the music disc. + type: + - "null" + - number + real_time_app_install: + description: Real-time app installations. + type: + - "null" + - number + real_time_app_install_cost: + description: Cost of real-time app installations. + type: + - "null" + - number + app_install: + description: The number of app installations. + type: + - "null" + - number + profile_visits_rate: + description: Rate of profile visits. + type: + - "null" + - number + purchase: + description: Number of purchases made. + type: + - "null" + - number + purchase_rate: + description: Rate of purchases. + type: + - "null" + - number + registration: + description: Number of registrations. + type: + - "null" + - number + registration_rate: + description: Rate of registrations. + type: + - "null" + - number + sales_lead: + description: Number of sales leads. + type: + - "null" + - number + sales_lead_rate: + description: Rate of sales leads. + type: + - "null" + - number + cost_per_app_install: + description: The cost per app installation. + type: + - "null" + - number + cost_per_purchase: + description: The cost per purchase. + type: + - "null" + - number + cost_per_registration: + description: The cost per registration. + type: + - "null" + - number + cost_per_sales_lead: + description: The cost per sales lead. + type: + - "null" + - number + cost_per_total_sales_lead: + description: The cost per total sales lead. + type: + - "null" + - number + cost_per_total_app_event_add_to_cart: + description: The cost per total app events adding to cart. + type: + - "null" + - number + total_app_event_add_to_cart: + description: Total app events adding items to cart. + type: + - "null" + - number + dimensions: + description: + A list of dimensions for which data should be retrieved such as time, + user demographics, or content type. + type: + - "null" + - object + properties: + stat_time_day: + description: The date for which the statistical data is recorded. + type: + - "null" + - string + format: date-time + stat_time_hour: + description: The hour of the day for which the statistical data is recorded. + type: + - "null" + - string + format: date-time + campaign_id: + description: + The unique identifier for a marketing campaign within the dimension + level. + type: + - "null" + - integer + adgroup_id: + description: The unique identifier for an ad group within the dimension level. + type: + - "null" + - integer + ad_id: + description: + The unique identifier for an advertisement within the dimension + level. + type: + - "null" + - integer + advertiser_id: + description: + The unique identifier for an advertiser within the dimension + level. + type: + - "null" + - integer + audience_report: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + type: object + properties: + advertiser_id: + description: Unique identifier for the advertiser + type: + - "null" + - integer + adgroup_id: + description: Unique identifier for the ad group + type: + - "null" + - integer + campaign_id: + description: Unique identifier for the campaign + type: + - "null" + - integer + ad_id: + description: Unique identifier for the ad + type: + - "null" + - integer + stat_time_day: + description: Day timestamp for the statistics + type: + - "null" + - string + format: date-time + airbyte_type: timestamp_without_timezone + stat_time_hour: + description: Hour timestamp for the statistics + type: + - "null" + - string + format: date-time + airbyte_type: timestamp_without_timezone + country_code: + description: Country code of the target audience + type: + - "null" + - string + platform: + description: Platform where the ad is displayed + type: + - "null" + - string + gender: + description: Gender of the target audience + type: + - "null" + - string + age: + description: Age group of the target audience + type: + - "null" + - string + province_id: + description: Province identifier of the target audience + type: + - "null" + - string + metrics: + description: + Defines the metrics or quantitative measurements of the audience + data such as number of views, engagement rate, share count, etc. + type: + - "null" + - object + properties: + campaign_name: + description: Name of the campaign + type: + - "null" + - string + campaign_id: + description: Campaign identifier within metrics + type: + - "null" + - integer + adgroup_name: + description: Name of the ad group + type: + - "null" + - string + placement_type: + description: Type of ad placement + type: + - "null" + - string + adgroup_id: + description: Unique identifier for the ad group within metrics + type: + - "null" + - integer + ad_name: + description: Name of the ad + type: + - "null" + - string + ad_text: + description: Text content of the ad + type: + - "null" + - string + tt_app_id: + description: TikTok app identifier + type: + - "null" + - string + tt_app_name: + description: TikTok app name + type: + - "null" + - string + mobile_app_id: + description: Mobile app identifier + type: + - "null" + - string + promotion_type: + description: Type of promotion + type: + - "null" + - string + dpa_target_audience_type: + description: Dynamic product ads target audience type + type: + - "null" + - string + spend: + description: Amount spent on the ad campaign + type: + - "null" + - string + cpc: + description: Cost per click + type: + - "null" + - string + cpm: + description: Cost per 1000 impressions + type: + - "null" + - string + impressions: + description: Number of times the ad was displayed + type: + - "null" + - string + clicks: + description: Number of clicks on the ad + type: + - "null" + - string + ctr: + description: Click-through rate + type: + - "null" + - string + reach: + description: Number of unique users who saw the ad + type: + - "null" + - string + cost_per_1000_reached: + description: Cost per 1000 impressions reached + type: + - "null" + - string + conversion: + description: Number of conversions from the ad + type: + - "null" + - string + cost_per_conversion: + description: Cost per conversion + type: + - "null" + - string + conversion_rate: + description: Rate of conversions from the ad + type: + - "null" + - string + real_time_conversion: + description: Real-time conversions from the ad + type: + - "null" + - string + real_time_cost_per_conversion: + description: Real-time cost per conversion + type: + - "null" + - string + real_time_conversion_rate: + description: Real-time conversion rate + type: + - "null" + - string + result: + description: Total results achieved + type: + - "null" + - string + cost_per_result: + description: Cost per result achieved + type: + - "null" + - string + result_rate: + description: Result rate + type: + - "null" + - string + real_time_result: + description: Real-time results achieved + type: + - "null" + - string + real_time_cost_per_result: + description: Real-time cost per result achieved + type: + - "null" + - string + real_time_result_rate: + description: Real-time result rate + type: + - "null" + - string + province_id: + description: Province identifier + type: + - "null" + - string + dimensions: + description: + Specifies the dimensions or attributes of the audience data being + reported such as age, gender, location, etc. + type: + - "null" + - object + properties: + stat_time_day: + description: Day timestamp for the statistics + type: + - "null" + - string + format: date-time + stat_time_hour: + description: Hour timestamp for the statistics + type: + - "null" + - string + format: date-time + country_code: + description: Country code within dimensions + type: + - "null" + - string + campaign_id: + description: Campaign identifier within dimensions + type: + - "null" + - integer + adgroup_id: + description: Unique identifier for the ad group within dimensions + type: + - "null" + - integer + ad_id: + description: Unique identifier for the ad within dimensions + type: + - "null" + - integer + advertiser_id: + description: Unique identifier for the advertiser within dimensions + type: + - "null" + - integer + gender: + description: Gender of the target audience within dimensions + type: + - "null" + - string + age: + description: Age group within dimensions + type: + - "null" + - string + ac: + description: AC description + type: + - "null" + - string + language: + description: Language of the target audience + type: + - "null" + - string + platform: + description: Platform type of the ad + type: + - "null" + - string + interest_category: + description: Interest category of the target audience + type: + - "null" + - string + placement: + description: Placement type of the ad + type: + - "null" + - string + +streams: + - $ref: "#/definitions/advertiser_ids_stream" + - $ref: "#/definitions/advertisers_stream" + - $ref: "#/definitions/audiences_stream" + - $ref: "#/definitions/creative_assets_music_stream" + - $ref: "#/definitions/creative_assets_portfolios_stream" + - $ref: "#/definitions/campaigns_stream" + - $ref: "#/definitions/ad_groups_stream" + - $ref: "#/definitions/ads_stream" + - $ref: "#/definitions/creative_assets_images_stream" + - $ref: "#/definitions/creative_assets_videos_stream" + - $ref: "#/definitions/ads_reports_daily_stream" + - $ref: "#/definitions/ad_groups_reports_daily_stream" + - $ref: "#/definitions/advertisers_reports_daily_stream" + - $ref: "#/definitions/campaigns_reports_daily_stream" + - $ref: "#/definitions/campaigns_audience_reports_daily_stream" + - $ref: "#/definitions/ad_group_audience_reports_daily_stream" + - $ref: "#/definitions/ads_audience_reports_daily_stream" + - $ref: "#/definitions/advertisers_audience_reports_daily_stream" + - $ref: "#/definitions/campaigns_audience_reports_by_country_daily_stream" + - $ref: "#/definitions/ad_group_audience_reports_by_country_daily_stream" + - $ref: "#/definitions/ads_audience_reports_by_country_daily_stream" + - $ref: "#/definitions/advertisers_audience_reports_by_country_daily_stream" + - $ref: "#/definitions/campaigns_audience_reports_by_platform_daily_stream" + - $ref: "#/definitions/ad_group_audience_reports_by_platform_daily_stream" + - $ref: "#/definitions/ads_audience_reports_by_platform_daily_stream" + - $ref: "#/definitions/advertisers_audience_reports_by_platform_daily_stream" + - $ref: "#/definitions/ads_audience_reports_by_province_daily_stream" + - $ref: "#/definitions/ads_reports_hourly_stream" + - $ref: "#/definitions/advertisers_reports_hourly_stream" + - $ref: "#/definitions/campaigns_reports_hourly_stream" + - $ref: "#/definitions/ad_groups_reports_hourly_stream" + - $ref: "#/definitions/ads_reports_lifetime_stream" + - $ref: "#/definitions/advertisers_reports_lifetime_stream" + - $ref: "#/definitions/campaigns_reports_lifetime_stream" + - $ref: "#/definitions/ad_groups_reports_lifetime_stream" + - $ref: "#/definitions/advertisers_audience_reports_lifetime_stream" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/tiktok-marketing + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: TikTok Marketing Source Spec + additionalProperties: true + type: object + properties: + credentials: + title: Authentication Method + description: Authentication method + default: {} + order: 0 + type: object + oneOf: + - title: OAuth2.0 + type: object + properties: + auth_type: + title: Auth Type + const: oauth2.0 + order: 0 + type: string + app_id: + title: App ID + description: The Developer Application App ID. + airbyte_secret: true + type: string + secret: + title: Secret + description: The Developer Application Secret. + airbyte_secret: true + type: string + access_token: + title: Access Token + description: Long-term Authorized Access Token. + airbyte_secret: true + type: string + advertiser_id: + title: Advertiser ID + description: + The Advertiser ID to filter reports and streams. Let this + empty to retrieve all. + type: string + required: + - app_id + - secret + - access_token + - title: Sandbox Access Token + type: object + properties: + auth_type: + title: Auth Type + const: sandbox_access_token + order: 0 + type: string + advertiser_id: + title: Advertiser ID + description: + The Advertiser ID which generated for the developer's Sandbox + application. + type: string + access_token: + title: Access Token + description: The long-term authorized access token. + airbyte_secret: true + type: string + required: + - advertiser_id + - access_token + start_date: + title: Replication Start Date + description: + "The Start Date in format: YYYY-MM-DD. Any data before this date + will not be replicated. If this parameter is not set, all data will be replicated." + default: "2016-09-01" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + order: 1 + type: string + format: date + end_date: + title: End Date + description: + The date until which you'd like to replicate data for all incremental + streams, in the format YYYY-MM-DD. All data generated between start_date and + this date will be replicated. Not setting this option will result in always + syncing the data till the current date. + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + order: 2 + type: string + format: date + attribution_window: + title: Attribution Window + description: The attribution window in days. + minimum: 0 + maximum: 364 + default: 3 + order: 3 + type: integer + include_deleted: + title: Include Deleted Data in Reports and Ads, Ad Groups and Campaign streams. + description: Set to active if you want to include deleted data in report based streams and Ads, Ad Groups and Campaign streams. + default: false + order: 4 + type: boolean + advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - credentials + - auth_type + predicate_value: oauth2.0 + oauth_config_specification: + complete_oauth_output_specification: + title: CompleteOauthOutputSpecification + type: object + properties: + access_token: + title: Access Token + path_in_connector_config: + - credentials + - access_token + type: string + required: + - access_token + complete_oauth_server_input_specification: + title: CompleteOauthServerInputSpecification + type: object + properties: + app_id: + title: App Id + type: string + secret: + title: Secret + type: string + required: + - app_id + - secret + complete_oauth_server_output_specification: + title: CompleteOauthServerOutputSpecification + type: object + properties: + app_id: + title: App Id + path_in_connector_config: + - credentials + - app_id + type: string + secret: + title: Secret + path_in_connector_config: + - credentials + - secret + type: string + required: + - app_id + - secret + +metadata: + autoImportSchema: + advertiser_ids: true diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json deleted file mode 100644 index 30b1ca85675f..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json +++ /dev/null @@ -1,584 +0,0 @@ -{ - "type": "object", - "properties": { - "adgroup_id": { - "description": "The unique identifier of the ad group", - "type": "integer" - }, - "campaign_id": { - "description": "The unique identifier of the campaign", - "type": "integer" - }, - "advertiser_id": { - "description": "The unique identifier of the advertiser", - "type": "integer" - }, - "adgroup_name": { - "description": "The name of the ad group", - "type": "string" - }, - "placement_type": { - "description": "The type of ad placement", - "type": "string", - "enum": ["PLACEMENT_TYPE_AUTOMATIC", "PLACEMENT_TYPE_NORMAL"] - }, - "placements": { - "description": "Information about the ad placements targeted", - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "inventory_filter_enabled": { - "description": "Flag indicating if inventory filter is enabled", - "type": ["null", "boolean"] - }, - "comment_disabled": { - "description": "Flag indicating if comments are disabled", - "type": "boolean" - }, - "app_id": { - "description": "The unique identifier of the app", - "type": ["null", "integer"] - }, - "promotion_type": { - "description": "The type of promotion", - "type": "string", - "enum": [ - "APP_ANDROID", - "APP_IOS", - "WEBSITE", - "LEAD_GENERATION", - "WEBSITE_OR_DISPLAY", - "TIKTOK_SHOP", - "VIDEO_SHOPPING", - "LIVE_SHOPPING" - ] - }, - "app_download_url": { - "description": "The URL for downloading the associated app", - "type": ["null", "string"] - }, - "package": { - "description": "The package used for the ad group", - "type": ["null", "string"] - }, - "pixel_id": { - "description": "The ID of the pixel used for tracking", - "type": ["null", "integer"] - }, - "optimization_event": { - "description": "The event used for optimization", - "type": ["null", "string"] - }, - "secondary_optimization_event": { - "description": "Additional event used for optimization", - "type": ["null", "string"] - }, - "creative_material_mode": { - "description": "The mode for creative materials", - "type": "string" - }, - "modify_time": { - "description": "The timestamp for when the ad group was last modified", - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "create_time": { - "description": "The timestamp for when the ad group was created", - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "audience_ids": { - "description": "The IDs of the targeted audience", - "type": "array", - "items": { - "type": "integer" - } - }, - "excluded_audience_ids": { - "description": "The IDs of excluded audiences", - "type": "array", - "items": { - "type": "integer" - } - }, - "audience_type": { - "description": "The type of audience being targeted", - "type": ["null", "string"] - }, - "location_ids": { - "description": "The IDs of targeted locations", - "type": "array", - "items": { - "type": "integer" - } - }, - "is_hfss": { - "description": "Flag indicating if high-frequency short sequences are included", - "type": "boolean" - }, - "interest_category_ids": { - "description": "The IDs of interest categories for targeting", - "type": "array", - "items": { - "type": "integer" - } - }, - "interest_keyword_ids": { - "description": "The IDs of interest keywords for targeting", - "type": "array", - "items": { - "type": "integer" - } - }, - "age_groups": { - "description": "The targeted age groups for the ad group", - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "gender": { - "description": "The targeted gender for the ad group", - "type": ["null", "string"] - }, - "languages": { - "description": "The targeted languages for the ad group", - "type": "array", - "items": { - "type": "string" - } - }, - "operating_systems": { - "description": "The targeted operating systems", - "type": "array", - "items": { - "type": "string" - } - }, - "network_types": { - "description": "The types of networks targeted", - "type": "array", - "items": { - "type": "string" - } - }, - "device_price_ranges": { - "description": "The price ranges for devices", - "type": ["null", "array"], - "items": { - "type": "number" - } - }, - "min_android_version": { - "description": "The minimum required Android version", - "type": ["null", "string"] - }, - "ios14_targeting": { - "description": "Information about iOS 14 targeting settings", - "type": ["null", "string"] - }, - "device_model_ids": { - "description": "The IDs of targeted device models", - "type": ["null", "array"], - "items": { - "type": "integer" - } - }, - "min_ios_version": { - "description": "The minimum required iOS version", - "type": ["null", "string"] - }, - "budget_mode": { - "description": "The mode for managing the budget", - "type": "string" - }, - "budget": { - "description": "The allocated budget for the ad group", - "type": "number" - }, - "schedule_type": { - "description": "The type of scheduling", - "type": "string" - }, - "schedule_start_time": { - "description": "The start time of the scheduling", - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "schedule_end_time": { - "description": "The end time of the scheduling", - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "dayparting": { - "description": "Information about dayparting settings", - "type": ["null", "string"] - }, - "optimization_goal": { - "description": "The goal set for optimization", - "type": "string" - }, - "cpv_video_duration": { - "description": "The duration for cost-per-view video", - "type": ["null", "string"] - }, - "conversion_window": { - "description": "The window for tracking conversions", - "type": ["null", "string"] - }, - "pacing": { - "description": "Information about the pacing settings", - "type": ["null", "string"] - }, - "billing_event": { - "description": "The event used for billing", - "type": ["null", "string"] - }, - "skip_learning_phase": { - "description": "Flag indicating if the learning phase is skipped", - "type": "integer" - }, - "bid_type": { - "description": "The type of bidding", - "type": ["null", "string"] - }, - "bid_price": { - "description": "The price set for bidding", - "type": "number" - }, - "conversion_bid_price": { - "description": "The bid price for conversions", - "type": "number" - }, - "deep_bid_type": { - "description": "The type of deep bid strategy", - "type": ["null", "string"] - }, - "deep_cpa_bid": { - "description": "The bid amount for deep cost-per-action", - "type": "number" - }, - "secondary_status": { - "description": "The secondary status of the ad group", - "type": "string" - }, - "operation_status": { - "description": "The status of the operation", - "type": "string" - }, - "frequency": { - "description": "The frequency of ad display", - "type": ["null", "integer"] - }, - "frequency_schedule": { - "description": "The schedule for frequency capping", - "type": ["null", "integer"] - }, - "statistic_type": { - "description": "The type of statistics being tracked", - "type": ["null", "string"] - }, - "carrier_ids": { - "description": "The IDs of the targeted carriers", - "type": ["null", "array"], - "items": { - "type": "integer" - } - }, - "carriers": { - "description": "Information about the targeted carriers", - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "video_download_disabled": { - "description": "Flag indicating if video downloads are disabled", - "type": "boolean" - }, - "blocked_pangle_app_ids": { - "description": "The IDs of the blocked Pangle apps", - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "action_category_ids": { - "description": "The IDs of the action categories associated with the ad group", - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "action_days": { - "description": "The number of days the action has been performed", - "type": ["null", "integer"] - }, - "video_actions": { - "description": "Information about video-specific actions", - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "rf_purchased_type": { - "description": "Type of purchased results", - "type": ["null", "string"] - }, - "purchased_impression": { - "description": "Information about purchased impressions", - "type": ["null", "number"] - }, - "purchased_reach": { - "description": "Information about purchased reach", - "type": ["null", "number"] - }, - "rf_estimated_cpr": { - "description": "Estimated cost per result", - "type": ["null", "number"] - }, - "rf_estimated_frequency": { - "description": "Estimated frequency of results", - "type": ["null", "number"] - }, - "included_pangle_audience_package_ids": { - "description": "The IDs of included Pangle audience packages", - "type": ["null", "array"], - "items": { - "type": "number" - } - }, - "excluded_pangle_audience_package_ids": { - "description": "The IDs of excluded Pangle audience packages", - "type": ["null", "array"], - "items": { - "type": "number" - } - }, - "is_new_structure": { - "description": "Flag indicating if the ad group follows a new structure", - "type": "boolean" - }, - "is_smart_performance_campaign": { - "description": "Flag indicating if the campaign is using smart performance", - "type": ["null", "boolean"] - }, - "catalog_id": { - "description": "The unique identifier of the catalog", - "type": ["null", "integer"] - }, - "product_set_id": { - "description": "The ID of the product set", - "type": ["null", "integer"] - }, - "catalog_authorized_bc_id": { - "description": "The authorized Business Center ID for the catalog", - "type": ["null", "integer"] - }, - "audience_rule": { - "description": "The rule set for targeting the audience", - "type": ["null", "object"] - }, - "included_custom_actions": { - "description": "Custom actions that are included", - "type": ["null", "array"], - "items": { - "type": "object" - } - }, - "excluded_custom_actions": { - "description": "Custom actions that are excluded", - "type": ["null", "array"], - "items": { - "type": "object" - } - }, - "shopping_ads_retargeting_type": { - "description": "The type of retargeting used for shopping ads", - "type": ["null", "string"] - }, - "split_test_adgroup_ids": { - "description": "The IDs of ad groups participating in split testing", - "type": ["null", "array"], - "items": { - "type": "number" - } - }, - "brand_safety_type": { - "description": "The type of brand safety measures", - "type": ["null", "string"] - }, - "brand_safety_partner": { - "description": "Information about the brand safety partners", - "type": ["null", "string"] - }, - "promotion_website_type": { - "description": "The type of website used for promotion", - "type": ["null", "string"] - }, - "ios_quota_type": { - "description": "The type of iOS quota", - "type": ["null", "string"] - }, - "roas_bid": { - "description": "The bid amount set for return on ad spend", - "type": ["null", "number"] - }, - "actions": { - "description": "Information about the actions taken on the ad group", - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "action_category_ids": { - "description": "The IDs of the action categories for the specific action", - "type": ["null", "array"], - "items": { - "type": "integer" - } - }, - "action_period": { - "description": "The period during which the action was taken", - "type": ["null", "number"] - }, - "action_scene": { - "description": "The scene in which the action took place", - "type": ["null", "string"] - }, - "video_user_actions": { - "description": "User actions specific to video content", - "type": ["null", "array"], - "items": { - "type": "string" - } - } - } - } - }, - "targeting_expansion": { - "description": "Settings for targeting expansion", - "type": ["null", "object"], - "properties": { - "expansion_enabled": { - "description": "Flag indicating if targeting expansion is enabled", - "type": "boolean" - }, - "expansion_types": { - "description": "Types of expansion enabled", - "type": ["null", "array"], - "items": { - "type": "string" - } - } - } - }, - "schedule_infos": { - "description": "Information about the scheduling arrangements", - "type": ["null", "array"], - "items": { - "type": "object" - } - }, - "share_disabled": { - "description": "Flag indicating if sharing is disabled", - "type": ["null", "boolean"] - }, - "auto_targeting_enabled": { - "description": "Flag indicating if auto-targeting is enabled", - "type": ["null", "boolean"] - }, - "ios14_quota_type": { - "description": "The type of iOS 14 quota", - "type": ["null", "string"] - }, - "campaign_name": { - "description": "The name of the campaign", - "type": ["null", "string"] - }, - "bid_display_mode": { - "description": "The display mode for bidding", - "type": ["null", "string"] - }, - "scheduled_budget": { - "description": "The budget allocated for scheduling", - "type": ["null", "number"] - }, - "adgroup_app_profile_page_state": { - "description": "The state of the app profile page related to the ad group", - "type": ["null", "string"] - }, - "keywords": { - "description": "Keywords associated with the ad group", - "type": ["null", "string"] - }, - "next_day_retention": { - "description": "Retention information for the next day", - "type": ["null", "number"] - }, - "category_id": { - "description": "The ID of the category for the ad group", - "type": ["null", "integer"] - }, - "search_result_enabled": { - "description": "Flag indicating if search results are enabled", - "type": ["null", "boolean"] - }, - "app_type": { - "description": "The type of the associated app", - "type": ["null", "string"] - }, - "feed_type": { - "description": "The type of feed used", - "type": ["null", "string"] - }, - "delivery_mode": { - "description": "The mode for delivery", - "type": ["null", "string"] - }, - "category_exclusion_ids": { - "description": "The IDs of the excluded categories", - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "contextual_tag_ids": { - "description": "The IDs of contextual tags for targeting", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "household_income": { - "description": "The targeted household income groups", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "isp_ids": { - "description": "The IDs of the targeted internet service providers", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "spending_power": { - "description": "Information about the spending power targeted", - "type": ["null", "string"] - }, - "zipcode_ids": { - "description": "The IDs of targeted ZIP codes", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ads.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ads.json deleted file mode 100644 index 2fa88b862bf7..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ads.json +++ /dev/null @@ -1,393 +0,0 @@ -{ - "type": "object", - "properties": { - "advertiser_id": { - "description": "The unique identifier of the advertiser", - "type": "integer" - }, - "campaign_id": { - "description": "The unique identifier of the campaign", - "type": "integer" - }, - "campaign_name": { - "description": "The name of the campaign", - "type": "string" - }, - "adgroup_id": { - "description": "The unique identifier of the ad group", - "type": "integer" - }, - "adgroup_name": { - "description": "The name of the ad group", - "type": "string" - }, - "ad_id": { - "description": "The unique identifier of the ad", - "type": "integer" - }, - "ad_name": { - "description": "The name of the ad", - "type": "string" - }, - "tracking_app_id": { - "description": "The unique identifier of the tracking app", - "type": ["null", "string"] - }, - "tracking_offline_event_set_ids": { - "description": "The unique identifiers of offline event sets for tracking", - "type": ["null", "array"], - "items": { - "description": "Unique identifier of an offline event set", - "type": ["null", "string"] - } - }, - "call_to_action": { - "description": "The call-to-action text for the ad", - "type": ["null", "string"] - }, - "call_to_action_id": { - "description": "The identifier of the call-to-action", - "type": ["null", "string"] - }, - "disclaimer_type": { - "description": "The type of disclaimer displayed", - "type": ["null", "string"] - }, - "disclaimer_text": { - "description": "The disclaimer text", - "type": ["null", "object"], - "properties": { - "text": { - "description": "The text of the disclaimer", - "type": ["null", "string"] - } - } - }, - "disclaimer_clickable_texts": { - "description": "Clickable disclaimer texts with URLs", - "type": ["null", "object"], - "properties": { - "text": { - "description": "The disclaimer text", - "type": ["null", "string"] - }, - "url": { - "description": "The URL associated with the disclaimer text", - "type": ["null", "string"] - } - } - }, - "card_id": { - "description": "The identifier of the card", - "type": ["null", "integer"] - }, - "secondary_status": { - "description": "The secondary status of the ad", - "type": "string" - }, - "operation_status": { - "description": "The operational status of the ad", - "type": ["null", "string"] - }, - "is_aco": { - "description": "Indicates if the ad is under Automated Creative Optimization", - "type": ["null", "boolean"] - }, - "image_ids": { - "description": "The unique identifiers of images used in the ad", - "type": ["null", "array"], - "items": { - "description": "Unique identifier of an image", - "type": "string" - } - }, - "image_mode": { - "description": "The mode of displaying images", - "type": ["null", "string"] - }, - "ad_format": { - "description": "The format of the ad (e.g., image, video, carousel)", - "type": ["null", "string"] - }, - "ad_text": { - "description": "The text content of the ad", - "type": ["null", "string"] - }, - "ad_texts": { - "description": "The text content of the ad in various languages", - "type": ["null", "array"], - "items": { - "description": "Text content in a specific language", - "type": "string" - } - }, - "video_id": { - "description": "The unique identifier of the video", - "type": ["null", "string"] - }, - "tiktok_item_id": { - "description": "The unique identifier of the TikTok item", - "type": ["null", "string"] - }, - "premium_badge_id": { - "description": "The unique identifier of the premium badge", - "type": ["null", "string"] - }, - "app_name": { - "description": "The name of the mobile app where the ad is displayed", - "type": ["null", "string"] - }, - "landing_page_url": { - "description": "The URL of the landing page for the ad", - "type": ["null", "string"] - }, - "landing_page_urls": { - "description": "The URLs of landing pages for the ad", - "type": ["null", "array"], - "items": { - "description": "URL of a landing page", - "type": "string" - } - }, - "display_name": { - "description": "The display name of the ad", - "type": ["null", "string"] - }, - "profile_image_url": { - "description": "The URL of the profile image associated with the ad", - "type": ["null", "string"] - }, - "impression_tracking_url": { - "description": "The URL for tracking ad impressions", - "type": ["null", "string"] - }, - "click_tracking_url": { - "description": "The URL for tracking ad clicks", - "type": ["null", "string"] - }, - "tracking_pixel_id": { - "description": "The unique identifier of the tracking pixel", - "type": ["null", "integer"] - }, - "deeplink": { - "description": "The deeplink URL for the ad", - "type": ["null", "string"] - }, - "deeplink_type": { - "description": "The type of deeplink used", - "type": ["null", "string"] - }, - "fallback_type": { - "description": "The type of fallback used", - "type": ["null", "string"] - }, - "playable_url": { - "description": "The URL for a playable ad", - "type": ["null", "string"] - }, - "vast_moat_enabled": { - "description": "Indicates if VAST MOAT is enabled", - "type": ["null", "boolean"] - }, - "page_id": { - "description": "The unique identifier of the page", - "type": ["null", "number"] - }, - "creative_authorized": { - "description": "Indicates if the creative is authorized", - "type": ["null", "boolean"] - }, - "is_new_structure": { - "description": "Indicates if the ad is part of a new structure", - "type": ["null", "boolean"] - }, - "create_time": { - "description": "The timestamp when the ad was created", - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "modify_time": { - "description": "The timestamp when the ad was last modified", - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "shopping_ads_fallback_type": { - "description": "The type of fallback for shopping ads", - "type": ["null", "string"] - }, - "shopping_deeplink_type": { - "description": "The type of deeplink for shopping", - "type": ["null", "string"] - }, - "shopping_ads_video_package_id": { - "description": "The unique identifier of the video package for shopping ads", - "type": ["null", "string"] - }, - "promotional_music_disabled": { - "description": "Indicates if promotional music is disabled", - "type": ["null", "boolean"] - }, - "item_duet_status": { - "description": "The status of item duet", - "type": ["null", "string"] - }, - "item_stitch_status": { - "description": "The status of item stitch", - "type": ["null", "string"] - }, - "avatar_icon_web_uri": { - "description": "The URL of the avatar icon for the ad", - "type": ["null", "string"] - }, - "brand_safety_postbid_partner": { - "description": "Details about post-bidding partner for brand safety", - "type": ["null", "string"] - }, - "brand_safety_vast_url": { - "description": "The VAST URL for brand safety tracking", - "type": ["null", "string"] - }, - "creative_type": { - "description": "The type of creative used in the ad", - "type": ["null", "string"] - }, - "identity_id": { - "description": "The identifier of the identity", - "type": ["null", "string"] - }, - "identity_type": { - "description": "The type of identity", - "type": ["null", "string"] - }, - "identity_authorized_bc_id": { - "description": "The authorized identity for branded content", - "type": ["null", "string"] - }, - "phone_region_code": { - "description": "The region code for the phone number", - "type": ["null", "string"] - }, - "phone_region_calling_code": { - "description": "The calling code region for the phone number", - "type": ["null", "string"] - }, - "optimization_event": { - "description": "The event used for optimization", - "type": ["null", "string"] - }, - "phone_number": { - "description": "The phone number associated with the ad", - "type": ["null", "string"] - }, - "carousel_image_index": { - "description": "The index of the image in a carousel ad", - "type": ["null", "integer"] - }, - "viewability_postbid_partner": { - "description": "Details about post-bidding partner for viewability tracking", - "type": ["null", "string"] - }, - "viewability_vast_url": { - "description": "The VAST URL for viewability tracking", - "type": ["null", "string"] - }, - "music_id": { - "description": "The unique identifier of the music used in the ad", - "type": ["null", "string"] - }, - "utm_params": { - "description": "UTM parameters for tracking", - "type": ["null", "array"], - "items": { - "description": "Key-value pair for a UTM parameter", - "type": ["null", "object"], - "properties": { - "key": { - "description": "The key of the UTM parameter", - "type": ["null", "string"] - }, - "value": { - "description": "The value of the UTM parameter", - "type": ["null", "string"] - } - } - } - }, - "shopping_ads_deeplink_type": { - "description": "The type of deeplink for shopping ads", - "type": ["null", "string"] - }, - "dark_post_status": { - "description": "The status of dark post", - "type": ["null", "string"] - }, - "branded_content_disabled": { - "description": "Indicates if branded content is disabled", - "type": ["null", "string"] - }, - "product_specific_type": { - "description": "The specific type of product", - "type": ["null", "string"] - }, - "catalog_id": { - "description": "The unique identifier of the catalog", - "type": ["null", "string"] - }, - "item_group_ids": { - "description": "The unique identifiers of item groups", - "type": ["null", "array"], - "items": { - "description": "Unique identifier of an item group", - "type": ["null", "string"] - } - }, - "product_set_id": { - "description": "The unique identifier of the product set", - "type": ["null", "string"] - }, - "sku_ids": { - "description": "The unique identifiers of SKUs associated with the ad", - "type": ["null", "array"], - "items": { - "description": "Unique identifier of a SKU", - "type": ["null", "string"] - } - }, - "dynamic_format": { - "description": "The dynamic format of the ad", - "type": ["null", "string"] - }, - "vertical_video_strategy": { - "description": "The strategy for displaying vertical videos", - "type": ["null", "string"] - }, - "dynamic_destination": { - "description": "The dynamic destination of the ad", - "type": ["null", "string"] - }, - "showcase_products": { - "description": "Products displayed in a showcase ad", - "type": ["null", "object"], - "properties": { - "item_group_id": { - "description": "The unique identifier of the item group", - "type": ["null", "string"] - }, - "store_id": { - "description": "The unique identifier of the store", - "type": ["null", "string"] - }, - "catalog_id": { - "description": "The unique identifier of the catalog", - "type": ["null", "string"] - } - } - }, - "tiktok_page_category": { - "description": "The category of the TikTok page", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/advertiser_ids.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/advertiser_ids.json deleted file mode 100644 index 58c094caba7d..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/advertiser_ids.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "object", - "properties": { - "advertiser_id": { - "description": "The unique identifier for each advertiser in the TikTok marketing platform.", - "type": "integer" - }, - "advertiser_name": { - "description": "The name of the advertiser registered in the TikTok marketing platform.", - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/advertisers.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/advertisers.json deleted file mode 100644 index 11031d5d61ca..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/advertisers.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "type": "object", - "properties": { - "advertiser_id": { - "description": "Unique identifier for the advertiser.", - "type": "integer" - }, - "name": { - "description": "The name of the advertiser or company.", - "type": "string" - }, - "address": { - "description": "The physical address of the advertiser.", - "type": ["null", "string"] - }, - "company": { - "description": "The name of the company associated with the advertiser.", - "type": ["null", "string"] - }, - "contacter": { - "description": "The contact person for the advertiser.", - "type": ["null", "string"] - }, - "country": { - "description": "The country where the advertiser is located.", - "type": ["null", "string"] - }, - "currency": { - "description": "The currency used for transactions in the account.", - "type": ["null", "string"] - }, - "description": { - "description": "A brief description or bio of the advertiser or company.", - "type": ["null", "string"] - }, - "email": { - "description": "The email address associated with the advertiser.", - "type": ["null", "string"] - }, - "industry": { - "description": "The industry or sector the advertiser operates in.", - "type": ["null", "string"] - }, - "language": { - "description": "The preferred language of communication for the advertiser.", - "type": ["null", "string"] - }, - "license_no": { - "description": "The license number of the advertiser.", - "type": ["null", "string"] - }, - "license_url": { - "description": "The URL link to the advertiser's license documentation.", - "type": ["null", "string"] - }, - "cellphone_number": { - "description": "The cellphone number of the advertiser.", - "type": ["null", "string"] - }, - "promotion_area": { - "description": "The specific area or region where the advertiser focuses promotion.", - "type": ["null", "string"] - }, - "rejection_reason": { - "description": "Reason for any advertisement rejection by the platform.", - "type": ["null", "string"] - }, - "role": { - "description": "The role or position of the advertiser within the company.", - "type": ["null", "string"] - }, - "status": { - "description": "The current status of the advertiser's account.", - "type": ["null", "string"] - }, - "timezone": { - "description": "The timezone setting for the advertiser's activities.", - "type": ["null", "string"] - }, - "balance": { - "description": "The current balance in the advertiser's account.", - "type": "number" - }, - "create_time": { - "description": "The timestamp when the advertiser account was created.", - "type": "integer" - }, - "telephone_number": { - "description": "The telephone number of the advertiser.", - "type": ["null", "string"] - }, - "display_timezone": { - "description": "The timezone for display purposes.", - "type": ["null", "string"] - }, - "promotion_center_province": { - "description": "The province or state at the center of the advertiser's promotion activities.", - "type": ["null", "string"] - }, - "advertiser_account_type": { - "description": "The type of advertiser's account (e.g., individual, business).", - "type": ["null", "string"] - }, - "license_city": { - "description": "The city where the advertiser's license is registered.", - "type": ["null", "string"] - }, - "brand": { - "description": "The brand name associated with the advertiser.", - "type": ["null", "string"] - }, - "license_province": { - "description": "The province or state where the advertiser's license is registered.", - "type": ["null", "string"] - }, - "promotion_center_city": { - "description": "The city at the center of the advertiser's promotion activities.", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audience_reports.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audience_reports.json deleted file mode 100644 index a504f4cfe25b..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audience_reports.json +++ /dev/null @@ -1,256 +0,0 @@ -{ - "type": "object", - "additionalProperties": true, - "properties": { - "advertiser_id": { - "description": "Unique identifier for the advertiser", - "type": ["null", "integer"] - }, - "adgroup_id": { - "description": "Unique identifier for the ad group", - "type": ["null", "integer"] - }, - "campaign_id": { - "description": "Unique identifier for the campaign", - "type": ["null", "integer"] - }, - "ad_id": { - "description": "Unique identifier for the ad", - "type": ["null", "integer"] - }, - "stat_time_day": { - "description": "Day timestamp for the statistics", - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "stat_time_hour": { - "description": "Hour timestamp for the statistics", - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "country_code": { - "description": "Country code of the target audience", - "type": ["null", "string"] - }, - "platform": { - "description": "Platform where the ad is displayed", - "type": ["null", "string"] - }, - "gender": { - "description": "Gender of the target audience", - "type": ["null", "string"] - }, - "age": { - "description": "Age group of the target audience", - "type": ["null", "string"] - }, - "province_id": { - "description": "Province identifier of the target audience", - "type": ["null", "string"] - }, - "metrics": { - "description": "Defines the metrics or quantitative measurements of the audience data such as number of views, engagement rate, share count, etc.", - "type": ["null", "object"], - "properties": { - "campaign_name": { - "description": "Name of the campaign", - "type": ["null", "string"] - }, - "campaign_id": { - "description": "Campaign identifier within metrics", - "type": ["null", "integer"] - }, - "adgroup_name": { - "description": "Name of the ad group", - "type": ["null", "string"] - }, - "placement_type": { - "description": "Type of ad placement", - "type": ["null", "string"] - }, - "adgroup_id": { - "description": "Unique identifier for the ad group within metrics", - "type": ["null", "integer"] - }, - "ad_name": { - "description": "Name of the ad", - "type": ["null", "string"] - }, - "ad_text": { - "description": "Text content of the ad", - "type": ["null", "string"] - }, - "tt_app_id": { - "description": "TikTok app identifier", - "type": ["null", "string"] - }, - "tt_app_name": { - "description": "TikTok app name", - "type": ["null", "string"] - }, - "mobile_app_id": { - "description": "Mobile app identifier", - "type": ["null", "string"] - }, - "promotion_type": { - "description": "Type of promotion", - "type": ["null", "string"] - }, - "dpa_target_audience_type": { - "description": "Dynamic product ads target audience type", - "type": ["null", "string"] - }, - "spend": { - "description": "Amount spent on the ad campaign", - "type": ["null", "string"] - }, - "cpc": { - "description": "Cost per click", - "type": ["null", "string"] - }, - "cpm": { - "description": "Cost per 1000 impressions", - "type": ["null", "string"] - }, - "impressions": { - "description": "Number of times the ad was displayed", - "type": ["null", "string"] - }, - "clicks": { - "description": "Number of clicks on the ad", - "type": ["null", "string"] - }, - "ctr": { - "description": "Click-through rate", - "type": ["null", "string"] - }, - "reach": { - "description": "Number of unique users who saw the ad", - "type": ["null", "string"] - }, - "cost_per_1000_reached": { - "description": "Cost per 1000 impressions reached", - "type": ["null", "string"] - }, - "conversion": { - "description": "Number of conversions from the ad", - "type": ["null", "string"] - }, - "cost_per_conversion": { - "description": "Cost per conversion", - "type": ["null", "string"] - }, - "conversion_rate": { - "description": "Rate of conversions from the ad", - "type": ["null", "string"] - }, - "real_time_conversion": { - "description": "Real-time conversions from the ad", - "type": ["null", "string"] - }, - "real_time_cost_per_conversion": { - "description": "Real-time cost per conversion", - "type": ["null", "string"] - }, - "real_time_conversion_rate": { - "description": "Real-time conversion rate", - "type": ["null", "string"] - }, - "result": { - "description": "Total results achieved", - "type": ["null", "string"] - }, - "cost_per_result": { - "description": "Cost per result achieved", - "type": ["null", "string"] - }, - "result_rate": { - "description": "Result rate", - "type": ["null", "string"] - }, - "real_time_result": { - "description": "Real-time results achieved", - "type": ["null", "string"] - }, - "real_time_cost_per_result": { - "description": "Real-time cost per result achieved", - "type": ["null", "string"] - }, - "real_time_result_rate": { - "description": "Real-time result rate", - "type": ["null", "string"] - }, - "province_id": { - "description": "Province identifier", - "type": ["null", "string"] - } - } - }, - "dimensions": { - "description": "Specifies the dimensions or attributes of the audience data being reported such as age, gender, location, etc.", - "type": ["null", "object"], - "properties": { - "stat_time_day": { - "description": "Day timestamp for the statistics", - "type": ["null", "string"], - "format": "date-time" - }, - "stat_time_hour": { - "description": "Hour timestamp for the statistics", - "type": ["null", "string"], - "format": "date-time" - }, - "country_code": { - "description": "Country code within dimensions", - "type": ["null", "string"] - }, - "campaign_id": { - "description": "Campaign identifier within dimensions", - "type": ["null", "integer"] - }, - "adgroup_id": { - "description": "Unique identifier for the ad group within dimensions", - "type": ["null", "integer"] - }, - "ad_id": { - "description": "Unique identifier for the ad within dimensions", - "type": ["null", "integer"] - }, - "advertiser_id": { - "description": "Unique identifier for the advertiser within dimensions", - "type": ["null", "integer"] - }, - "gender": { - "description": "Gender of the target audience within dimensions", - "type": ["null", "string"] - }, - "age": { - "description": "Age group within dimensions", - "type": ["null", "string"] - }, - "ac": { - "description": "AC description", - "type": ["null", "string"] - }, - "language": { - "description": "Language of the target audience", - "type": ["null", "string"] - }, - "platform": { - "description": "Platform type of the ad", - "type": ["null", "string"] - }, - "interest_category": { - "description": "Interest category of the target audience", - "type": ["null", "string"] - }, - "placement": { - "description": "Placement type of the ad", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json deleted file mode 100644 index 4f2bc2504ec8..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "type": "object", - "properties": { - "shared": { - "description": "Flag indicating if the audience is shared with others", - "type": ["null", "boolean"] - }, - "is_creator": { - "description": "Flag indicating if the audience creator is the user", - "type": ["null", "boolean"] - }, - "audience_id": { - "description": "Unique identifier for the audience", - "type": ["null", "string"] - }, - "cover_num": { - "description": "Number of audience members covered", - "type": ["null", "integer"] - }, - "create_time": { - "description": "Timestamp indicating when the audience was created", - "type": ["null", "string"], - "format": "date-time" - }, - "is_valid": { - "description": "Flag indicating if the audience data is valid", - "type": ["null", "boolean"] - }, - "is_expiring": { - "description": "Flag indicating if the audience data is expiring soon", - "type": ["null", "boolean"] - }, - "expired_time": { - "description": "Timestamp indicating when the audience data expires", - "type": ["null", "string"], - "format": "date-time" - }, - "name": { - "description": "Name of the audience", - "type": ["null", "string"] - }, - "audience_type": { - "description": "Type of audience (e.g., demographic, interest-based)", - "type": ["null", "string"] - }, - "calculate_type": { - "description": "Method used to calculate audience data", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json deleted file mode 100644 index 5702675a4cd4..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json +++ /dev/null @@ -1,396 +0,0 @@ -{ - "type": "object", - "additionalProperties": true, - "properties": { - "stat_time_day": { - "description": "The date for which the statistical data is recorded.", - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "stat_time_hour": { - "description": "The hour of the day for which the statistical data is recorded.", - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "campaign_id": { - "description": "The unique identifier for a marketing campaign.", - "type": ["null", "integer"] - }, - "adgroup_id": { - "description": "The unique identifier for an ad group.", - "type": ["null", "integer"] - }, - "ad_id": { - "description": "The unique identifier for an advertisement.", - "type": ["null", "integer"] - }, - "advertiser_id": { - "description": "The unique identifier for an advertiser.", - "type": ["null", "integer"] - }, - "metrics": { - "description": "A list of metrics for which data should be retrieved such as views, likes, comments, or shares.", - "type": ["null", "object"], - "properties": { - "campaign_name": { - "description": "The name of the marketing campaign.", - "type": ["null", "string"] - }, - "campaign_id": { - "description": "The unique identifier for a marketing campaign within the metrics level.", - "type": ["null", "integer"] - }, - "adgroup_name": { - "description": "The name of the ad group.", - "type": ["null", "string"] - }, - "placement_type": { - "description": "Type of advertisement placement.", - "type": ["null", "string"] - }, - "adgroup_id": { - "description": "The unique identifier for an ad group within the metrics level.", - "type": ["null", "integer"] - }, - "ad_name": { - "description": "The name of the advertisement.", - "type": ["null", "string"] - }, - "ad_text": { - "description": "The content or text of the advertisement.", - "type": ["null", "string"] - }, - "tt_app_id": { - "description": "The unique identifier for a TikTok application.", - "type": ["null", "integer"] - }, - "tt_app_name": { - "description": "The name of the TikTok application.", - "type": ["null", "string"] - }, - "mobile_app_id": { - "description": "The unique identifier for a mobile application.", - "type": ["null", "string"] - }, - "promotion_type": { - "description": "Type of promotion.", - "type": ["null", "string"] - }, - "dpa_target_audience_type": { - "description": "Dynamic product ad target audience type.", - "type": ["null", "string"] - }, - "spend": { - "description": "Amount of money spent.", - "type": ["null", "string"] - }, - "cash_spend": { - "description": "The amount of money spent in cash.", - "type": ["null", "string"] - }, - "voucher_spend": { - "description": "Amount spent on vouchers.", - "type": ["null", "string"] - }, - "cpc": { - "description": "Cost per click.", - "type": ["null", "string"] - }, - "cpm": { - "description": "Cost per thousand impressions.", - "type": ["null", "string"] - }, - "impressions": { - "description": "Number of times the advertisement is viewed.", - "type": ["null", "string"] - }, - "clicks": { - "description": "The number of clicks on the advertisement.", - "type": ["null", "string"] - }, - "ctr": { - "description": "Click-through rate.", - "type": ["null", "string"] - }, - "reach": { - "description": "Total number of unique users reached.", - "type": ["null", "string"] - }, - "cost_per_1000_reached": { - "description": "The cost per 1000 reached users.", - "type": ["null", "string"] - }, - "conversion": { - "description": "The number of conversions.", - "type": ["null", "string"] - }, - "cost_per_conversion": { - "description": "The cost per conversion.", - "type": ["null", "string"] - }, - "conversion_rate": { - "description": "The rate of conversion.", - "type": ["null", "string"] - }, - "real_time_conversion": { - "description": "Real-time conversions.", - "type": ["null", "string"] - }, - "real_time_cost_per_conversion": { - "description": "Cost per conversion in real-time.", - "type": ["null", "string"] - }, - "real_time_conversion_rate": { - "description": "Real-time conversion rate.", - "type": ["null", "string"] - }, - "result": { - "description": "Number of results.", - "type": ["null", "string"] - }, - "cost_per_result": { - "description": "The cost per result.", - "type": ["null", "string"] - }, - "result_rate": { - "description": "Rate of results.", - "type": ["null", "string"] - }, - "real_time_result": { - "description": "Real-time results.", - "type": ["null", "string"] - }, - "real_time_cost_per_result": { - "description": "Cost per result in real-time.", - "type": ["null", "string"] - }, - "real_time_result_rate": { - "description": "Real-time result rate.", - "type": ["null", "string"] - }, - "secondary_goal_result": { - "description": "Results for secondary goals.", - "type": ["null", "string"] - }, - "cost_per_secondary_goal_result": { - "description": "The cost per secondary goal result.", - "type": ["null", "string"] - }, - "secondary_goal_result_rate": { - "description": "Rate of results for secondary goals.", - "type": ["null", "string"] - }, - "frequency": { - "description": "Frequency of occurrence.", - "type": ["null", "string"] - }, - "total_purchase_value": { - "description": "Total value of purchases made.", - "type": ["null", "string"] - }, - "total_onsite_shopping_value": { - "description": "Total value of onsite shopping.", - "type": ["null", "string"] - }, - "onsite_shopping": { - "description": "Shopping happening on the site.", - "type": ["null", "string"] - }, - "vta_purchase": { - "description": "Purchase through vertical takeoff ad (VTA).", - "type": ["null", "string"] - }, - "cta_purchase": { - "description": "Purchase through call-to-action.", - "type": ["null", "string"] - }, - "cta_conversion": { - "description": "Conversion through call-to-action.", - "type": ["null", "string"] - }, - "vta_conversion": { - "description": "Conversion through vertical takeoff ad (VTA).", - "type": ["null", "string"] - }, - "total_pageview": { - "description": "Total number of page views.", - "type": ["null", "string"] - }, - "complete_payment": { - "description": "The number of completed payments.", - "type": ["null", "string"] - }, - "value_per_complete_payment": { - "description": "Value per completed payment.", - "type": ["null", "string"] - }, - "total_complete_payment_rate": { - "description": "Rate of total completed payments.", - "type": ["null", "string"] - }, - "video_play_actions": { - "description": "Actions related to video plays.", - "type": ["null", "number"] - }, - "video_watched_2s": { - "description": "Number of viewers watching at least 2 seconds of the video.", - "type": ["null", "number"] - }, - "video_watched_6s": { - "description": "Number of viewers watching at least 6 seconds of the video.", - "type": ["null", "number"] - }, - "average_video_play": { - "description": "The average number of video plays.", - "type": ["null", "number"] - }, - "average_video_play_per_user": { - "description": "The average number of video plays per user.", - "type": ["null", "number"] - }, - "video_views_p25": { - "description": "Percentage of viewers watching at least 25% of the video.", - "type": ["null", "number"] - }, - "video_views_p50": { - "description": "Percentage of viewers watching at least 50% of the video.", - "type": ["null", "number"] - }, - "video_views_p75": { - "description": "Percentage of viewers watching at least 75% of the video.", - "type": ["null", "number"] - }, - "video_views_p100": { - "description": "Percentage of viewers watching the entire video.", - "type": ["null", "number"] - }, - "profile_visits": { - "description": "Number of visits to the profile.", - "type": ["null", "number"] - }, - "likes": { - "description": "Number of likes received.", - "type": ["null", "number"] - }, - "comments": { - "description": "The number of comments received.", - "type": ["null", "number"] - }, - "shares": { - "description": "Number of shares.", - "type": ["null", "number"] - }, - "follows": { - "description": "Number of follows.", - "type": ["null", "number"] - }, - "clicks_on_music_disc": { - "description": "The number of clicks on the music disc.", - "type": ["null", "number"] - }, - "real_time_app_install": { - "description": "Real-time app installations.", - "type": ["null", "number"] - }, - "real_time_app_install_cost": { - "description": "Cost of real-time app installations.", - "type": ["null", "number"] - }, - "app_install": { - "description": "The number of app installations.", - "type": ["null", "number"] - }, - "profile_visits_rate": { - "description": "Rate of profile visits.", - "type": ["null", "number"] - }, - "purchase": { - "description": "Number of purchases made.", - "type": ["null", "number"] - }, - "purchase_rate": { - "description": "Rate of purchases.", - "type": ["null", "number"] - }, - "registration": { - "description": "Number of registrations.", - "type": ["null", "number"] - }, - "registration_rate": { - "description": "Rate of registrations.", - "type": ["null", "number"] - }, - "sales_lead": { - "description": "Number of sales leads.", - "type": ["null", "number"] - }, - "sales_lead_rate": { - "description": "Rate of sales leads.", - "type": ["null", "number"] - }, - "cost_per_app_install": { - "description": "The cost per app installation.", - "type": ["null", "number"] - }, - "cost_per_purchase": { - "description": "The cost per purchase.", - "type": ["null", "number"] - }, - "cost_per_registration": { - "description": "The cost per registration.", - "type": ["null", "number"] - }, - "cost_per_sales_lead": { - "description": "The cost per sales lead.", - "type": ["null", "number"] - }, - "cost_per_total_sales_lead": { - "description": "The cost per total sales lead.", - "type": ["null", "number"] - }, - "cost_per_total_app_event_add_to_cart": { - "description": "The cost per total app events adding to cart.", - "type": ["null", "number"] - }, - "total_app_event_add_to_cart": { - "description": "Total app events adding items to cart.", - "type": ["null", "number"] - } - } - }, - "dimensions": { - "description": "A list of dimensions for which data should be retrieved such as time, user demographics, or content type.", - "type": ["null", "object"], - "properties": { - "stat_time_day": { - "description": "The date for which the statistical data is recorded.", - "type": ["null", "string"], - "format": "date-time" - }, - "stat_time_hour": { - "description": "The hour of the day for which the statistical data is recorded.", - "type": ["null", "string"], - "format": "date-time" - }, - "campaign_id": { - "description": "The unique identifier for a marketing campaign within the dimension level.", - "type": ["null", "integer"] - }, - "adgroup_id": { - "description": "The unique identifier for an ad group within the dimension level.", - "type": ["null", "integer"] - }, - "ad_id": { - "description": "The unique identifier for an advertisement within the dimension level.", - "type": ["null", "integer"] - }, - "advertiser_id": { - "description": "The unique identifier for an advertiser within the dimension level.", - "type": ["null", "integer"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/campaigns.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/campaigns.json deleted file mode 100644 index 38db717bb1e7..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/campaigns.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "type": "object", - "properties": { - "campaign_id": { - "description": "The unique identifier of the campaign", - "type": "integer" - }, - "campaign_name": { - "description": "Name of the campaign for easy identification", - "type": "string" - }, - "campaign_type": { - "description": "Type of campaign (e.g., awareness, conversion)", - "type": "string" - }, - "advertiser_id": { - "description": "The unique identifier of the advertiser associated with the campaign", - "type": "integer" - }, - "budget": { - "description": "Total budget allocated for the campaign", - "type": "number" - }, - "budget_mode": { - "description": "Mode in which the budget is being managed (e.g., daily, lifetime)", - "type": "string" - }, - "secondary_status": { - "description": "Additional status information of the campaign", - "type": "string" - }, - "operation_status": { - "description": "Current operational status of the campaign (e.g., active, paused)", - "type": ["null", "string"] - }, - "objective": { - "description": "The objective or goal of the campaign", - "type": ["null", "string"] - }, - "objective_type": { - "description": "Type of objective selected for the campaign (e.g., brand awareness, app installs)", - "type": ["null", "string"] - }, - "budget_optimize_on": { - "description": "The metric or event that the budget optimization is based on", - "type": ["null", "boolean"] - }, - "bid_type": { - "description": "Type of bid strategy being used in the campaign", - "type": ["null", "string"] - }, - "deep_bid_type": { - "description": "Advanced bid type used for campaign optimization", - "type": ["null", "string"] - }, - "optimization_goal": { - "description": "Specific goal to be optimized for in the campaign", - "type": ["null", "string"] - }, - "split_test_variable": { - "description": "Variable being tested in a split test campaign", - "type": ["null", "string"] - }, - "is_new_structure": { - "description": "Flag indicating if the campaign utilizes a new campaign structure", - "type": "boolean" - }, - "create_time": { - "description": "Timestamp when the campaign was created", - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "modify_time": { - "description": "Timestamp when the campaign was last modified", - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "roas_bid": { - "description": "Return on ad spend goal set for the campaign", - "type": ["null", "number"] - }, - "is_smart_performance_campaign": { - "description": "Flag indicating if the campaign uses smart performance optimization", - "type": ["null", "boolean"] - }, - "is_search_campaign": { - "description": "Flag indicating if the campaign is a search campaign", - "type": ["null", "boolean"] - }, - "app_promotion_type": { - "description": "Type of app promotion being used in the campaign", - "type": ["null", "string"] - }, - "rf_campaign_type": { - "description": "Type of RF (reach and frequency) campaign being run", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json deleted file mode 100644 index 4a9a29c0610f..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "type": "object", - "properties": { - "image_id": { - "description": "The unique identifier for the image.", - "type": ["null", "string"] - }, - "format": { - "description": "The format type of the image file.", - "type": ["null", "string"] - }, - "image_url": { - "description": "The URL to access the image.", - "type": ["null", "string"] - }, - "height": { - "description": "The height dimension of the image.", - "type": ["null", "integer"] - }, - "width": { - "description": "The width dimension of the image.", - "type": ["null", "integer"] - }, - "signature": { - "description": "The signature of the image for security purposes.", - "type": ["null", "string"] - }, - "size": { - "description": "The size of the image file.", - "type": ["null", "integer"] - }, - "material_id": { - "description": "The ID associated with the material of the image.", - "type": ["null", "string"] - }, - "is_carousel_usable": { - "description": "Flag indicating if the image can be used in a carousel.", - "type": ["null", "boolean"] - }, - "file_name": { - "description": "The name of the image file.", - "type": ["null", "string"] - }, - "create_time": { - "description": "The timestamp when the creative asset image was created.", - "type": ["null", "string"], - "format": "date-time" - }, - "modify_time": { - "description": "The timestamp when the creative asset image was last modified.", - "type": ["null", "string"], - "format": "date-time" - }, - "displayable": { - "description": "Flag indicating if the image is displayable or not.", - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json deleted file mode 100644 index d6acebbdcd5e..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "type": "object", - "properties": { - "music_id": { - "description": "The unique identifier for the music asset.", - "type": ["null", "string"] - }, - "material_id": { - "description": "The unique ID assigned to the music asset.", - "type": ["null", "string"] - }, - "sources": { - "description": "The list of different sources or versions available for the music asset.", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "author": { - "description": "The author of the music asset.", - "type": ["null", "string"] - }, - "liked": { - "description": "The number of likes received by the music asset.", - "type": ["null", "boolean"] - }, - "cover_url": { - "description": "The URL to the cover image associated with the music asset.", - "type": ["null", "string"] - }, - "url": { - "description": "The URL to access or play the music asset.", - "type": ["null", "string"] - }, - "duration": { - "description": "The duration of the music asset in seconds.", - "type": ["null", "number"] - }, - "style": { - "description": "The style or genre of the music asset.", - "type": ["null", "string"] - }, - "signature": { - "description": "The digital signature associated with the music asset.", - "type": ["null", "string"] - }, - "name": { - "description": "The name or title of the music asset.", - "type": ["null", "string"] - }, - "file_name": { - "description": "The file name of the music asset.", - "type": ["null", "string"] - }, - "copyright": { - "description": "The copyright information related to the music asset.", - "type": ["null", "string"] - }, - "create_time": { - "description": "The timestamp indicating when the music asset was created.", - "type": ["null", "string"], - "format": "date-time" - }, - "modify_time": { - "description": "The timestamp indicating when the music asset was last modified.", - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json deleted file mode 100644 index 58544ab4295a..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "type": "object", - "properties": { - "creative_portfolio_id": { - "description": "The unique identifier for the creative portfolio.", - "type": ["null", "string"] - }, - "creative_portfolio_type": { - "description": "The type of the creative portfolio, such as image, video, or carousel.", - "type": ["null", "string"] - }, - "creative_portfolio_preview_url": { - "description": "The URL pointing to a preview image or video of the creative portfolio.", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json deleted file mode 100644 index d809fe3b3f90..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "type": "object", - "properties": { - "video_id": { - "description": "ID of the video.", - "type": ["null", "string"] - }, - "video_cover_url": { - "description": "URL for the cover image of the video.", - "type": ["null", "string"] - }, - "format": { - "description": "Format of the video file.", - "type": ["null", "string"] - }, - "preview_url": { - "description": "URL for previewing the video.", - "type": ["null", "string"] - }, - "preview_url_expire_time": { - "description": "Timestamp when the preview URL expires.", - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "duration": { - "description": "Duration of the video in seconds.", - "type": ["null", "number"] - }, - "height": { - "description": "Height of the video in pixels.", - "type": ["null", "integer"] - }, - "width": { - "description": "Width of the video in pixels.", - "type": ["null", "integer"] - }, - "bit_rate": { - "description": "The bitrate of the video.", - "type": ["null", "number"] - }, - "signature": { - "description": "Signature for authenticating the video request.", - "type": ["null", "string"] - }, - "size": { - "description": "Size of the video file in bytes.", - "type": ["null", "integer"] - }, - "material_id": { - "description": "ID of the video material.", - "type": ["null", "string"] - }, - "allowed_placements": { - "description": "List of placements where the video can be used.", - "type": ["null", "array"], - "items": { - "description": "Specific placement where the video is allowed.", - "type": ["null", "string"] - } - }, - "allow_download": { - "description": "Indicates if the video can be downloaded by users.", - "type": ["null", "boolean"] - }, - "file_name": { - "description": "Name of the video file.", - "type": ["null", "string"] - }, - "create_time": { - "description": "Timestamp when the video was created.", - "type": ["null", "string"], - "format": "date-time" - }, - "modify_time": { - "description": "Timestamp when the video was last modified.", - "type": ["null", "string"], - "format": "date-time" - }, - "displayable": { - "description": "Indicates if the video is displayable.", - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index 71e94d389015..8d0b1f1afcc2 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -2,236 +2,75 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import logging -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping -import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - -from .streams import ( - DEFAULT_END_DATE, - DEFAULT_START_DATE, - MINIMUM_START_DATE, - AdGroupAudienceReports, - AdGroupAudienceReportsByCountry, - AdGroupAudienceReportsByPlatform, - AdGroups, - AdGroupsReports, - Ads, - AdsAudienceReports, - AdsAudienceReportsByCountry, - AdsAudienceReportsByPlatform, - AdsAudienceReportsByProvince, - AdsReports, - AdvertiserIds, - Advertisers, - AdvertisersAudienceReports, - AdvertisersAudienceReportsByCountry, - AdvertisersAudienceReportsByPlatform, - AdvertisersReports, - Audiences, - BasicReports, - Campaigns, - CampaignsAudienceReports, - CampaignsAudienceReportsByCountry, - CampaignsAudienceReportsByPlatform, - CampaignsReports, - CreativeAssetsImages, - CreativeAssetsMusic, - CreativeAssetsPortfolios, - CreativeAssetsVideos, - Daily, - Hourly, - Lifetime, - ReportGranularity, -) logger = logging.getLogger("airbyte") DOCUMENTATION_URL = "https://docs.airbyte.com/integrations/sources/tiktok-marketing" +SANDBOX_STREAM_NAMES = [ + "ad_group_audience_reports_by_country_daily", + "ad_group_audience_reports_by_platform_daily", + "ad_group_audience_reports_daily", + "ad_groups", + "ad_groups_reports_daily", + "ad_groups_reports_hourly", + "ad_groups_reports_lifetime", + "ads", + "ads_audience_reports_by_country_daily", + "ads_audience_reports_by_platform_daily", + "ads_audience_reports_by_province_daily", + "ads_audience_reports_daily", + "ads_reports_daily", + "ads_reports_hourly", + "ads_reports_lifetime", + "advertisers", + "audiences", + "campaigns", + "campaigns_audience_reports_by_country_daily", + "campaigns_audience_reports_by_platform_daily", + "campaigns_audience_reports_daily", + "campaigns_reports_daily", + "campaigns_reports_hourly", + "campaigns_reports_lifetime", + "creative_assets_images", + "creative_assets_music", + "creative_assets_portfolios", + "creative_assets_videos", +] +REPORT_GRANULARITY = {"DAY": "daily", "HOUR": "hourly", "LIFETIME": "lifetime"} + + +class SourceTiktokMarketing(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) - -def get_report_stream(report: BasicReports, granularity: ReportGranularity) -> BasicReports: - """Fabric method to generate final class with name like: AdsReports + Hourly""" - report_class_name = f"{report.__name__}{granularity.__name__}" - return type(report_class_name, (granularity, report), {}) - - -class TiktokTokenAuthenticator(TokenAuthenticator): - """ - Docs: https://business-api.tiktok.com/marketing_api/docs?rid=sta6fe2yww&id=1701890922708994 - """ - - def __init__(self, token: str, **kwargs): - super().__init__(token, **kwargs) - self.token = token - - def get_auth_header(self) -> Mapping[str, Any]: - return {"Access-Token": self.token} - - -class SourceTiktokMarketing(AbstractSource): @staticmethod - def _prepare_stream_args(config: Mapping[str, Any]) -> Mapping[str, Any]: - """Converts an input configure to stream arguments""" - + def _is_sandbox(config: Mapping[str, Any]) -> bool: credentials = config.get("credentials") - if credentials: - # used for new config format is_sandbox = credentials["auth_type"] == "sandbox_access_token" - access_token = credentials["access_token"] - secret = credentials.get("secret") - app_id = int(credentials.get("app_id", 0)) - advertiser_id = credentials.get("advertiser_id") else: - # old config only has advertiser id in environment object - # if there is a secret it is a prod config - access_token = config["access_token"] secret = config.get("environment", {}).get("secret") is_sandbox = secret is None - app_id = int(config.get("environment", {}).get("app_id", 0)) - advertiser_id = config.get("environment", {}).get("advertiser_id") - - start_date = config.get("start_date") or DEFAULT_START_DATE - if pendulum.parse(start_date) < pendulum.parse(MINIMUM_START_DATE): - logger.warning(f"The start date is too far in the past. Setting it to {MINIMUM_START_DATE}.") - start_date = MINIMUM_START_DATE - stream_args = { - "authenticator": TiktokTokenAuthenticator(access_token), - "start_date": start_date, - "end_date": config.get("end_date") or DEFAULT_END_DATE, - "app_id": app_id, - "secret": secret, - "access_token": access_token, - "is_sandbox": is_sandbox, - "attribution_window": config.get("attribution_window"), - "include_deleted": config.get("include_deleted"), - } - if advertiser_id: - stream_args.update(**{"advertiser_id": advertiser_id}) - - return stream_args - - def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: - """ - Tests if the input configuration can be used to successfully connect to the integration - """ - try: - advertisers = Advertisers(**self._prepare_stream_args(config)) - for slice_ in advertisers.stream_slices(): - next(advertisers.read_records(SyncMode.full_refresh, stream_slice=slice_)) - except Exception as err: - return False, err - return True, None + return is_sandbox def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = self._prepare_stream_args(config) - - is_production = not (args["is_sandbox"]) - - report_granularity = config.get("report_granularity") - - # 1. Basic streams: - streams = [ - Advertisers(**args), - Ads(**args), - AdGroups(**args), - Audiences(**args), - Campaigns(**args), - CreativeAssetsImages(**args), - CreativeAssetsMusic(**args), - CreativeAssetsPortfolios(**args), - CreativeAssetsVideos(**args), - ] - - if is_production: - streams.append(AdvertiserIds(**args)) - - # Report streams in different connector version: - # for < 0.1.13 - expose report streams initialized with 'report_granularity' argument, like: - # AdsReports(report_granularity='DAILY') - # AdsReports(report_granularity='LIFETIME') - # for >= 0.1.13 - expose report streams in format: _, like: - # AdsReportsDaily(Daily, AdsReports) - # AdsReportsLifetime(Lifetime, AdsReports) - - if report_granularity: - # for version < 0.1.13 - compatibility with old config with 'report_granularity' option - - # 2. Basic report streams: - report_args = dict(report_granularity=report_granularity, **args) - streams.extend( - [ - AdsReports(**report_args), - AdGroupsReports(**report_args), - CampaignsReports(**report_args), - ] - ) - - # 3. Audience report streams: - if not report_granularity == ReportGranularity.LIFETIME: - # https://ads.tiktok.com/marketing_api/docs?id=1707957217727489 - # Audience report only supports lifetime metrics at the ADVERTISER level. - streams.extend( - [ - AdsAudienceReports(**report_args), - AdGroupAudienceReports(**report_args), - CampaignsAudienceReportsByCountry(**report_args), - ] - ) - - # 4. streams work only in prod env - if is_production: - streams.extend( - [ - AdvertisersReports(**report_args), - AdvertisersAudienceReports(**report_args), - ] - ) - - else: - # for version >= 0.1.13: - - # 2. Basic report streams: - reports = [AdsReports, AdGroupsReports, CampaignsReports] - audience_reports = [ - AdsAudienceReports, - AdsAudienceReportsByCountry, - AdsAudienceReportsByPlatform, - AdsAudienceReportsByProvince, - AdGroupAudienceReports, - AdGroupAudienceReportsByCountry, - AdGroupAudienceReportsByPlatform, - CampaignsAudienceReports, - CampaignsAudienceReportsByCountry, - CampaignsAudienceReportsByPlatform, - ] - if is_production: - # 2.1 streams work only in prod env - reports.append(AdvertisersReports) - audience_reports.extend( - [ - AdvertisersAudienceReports, - AdvertisersAudienceReportsByCountry, - AdvertisersAudienceReportsByPlatform, - ] - ) - - for Report in reports: - for Granularity in [Hourly, Daily, Lifetime]: - streams.append(get_report_stream(Report, Granularity)(**args)) - # add a for loop here for the other dimension to split by - - # 3. Audience report streams: - for Report in audience_reports: - # As per TikTok's documentation, audience reports only support daily (not hourly) time dimension for metrics - streams.append(get_report_stream(Report, Daily)(**args)) - - # Audience report supports lifetime metrics only at the ADVERTISER level (see 2.1). - if Report == AdvertisersAudienceReports: - streams.append(get_report_stream(Report, Lifetime)(**args)) + granularity = config.get("report_granularity") + streams = super().streams(config) + + if self._is_sandbox(config): + streams = [stream for stream in streams if stream.name in SANDBOX_STREAM_NAMES] + + if granularity: + granularity_streams = [] + for stream in streams: + if "report" not in stream.name or REPORT_GRANULARITY[granularity] in stream.name: + # old configs with provided report granularity don't have granularity in stream name + stream.name = stream.name.replace(f"_{REPORT_GRANULARITY[granularity]}", "") + granularity_streams.append(stream) + return granularity_streams return streams diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.json deleted file mode 100644 index 5751d01fdedb..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/tiktok-marketing", - "changelogUrl": "https://docs.airbyte.com/integrations/sources/tiktok-marketing", - "connectionSpecification": { - "title": "TikTok Marketing Source Spec", - "type": "object", - "properties": { - "credentials": { - "title": "Authentication Method", - "description": "Authentication method", - "default": {}, - "order": 0, - "type": "object", - "oneOf": [ - { - "title": "OAuth2.0", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" - }, - "app_id": { - "title": "App ID", - "description": "The Developer Application App ID.", - "airbyte_secret": true, - "type": "string" - }, - "secret": { - "title": "Secret", - "description": "The Developer Application Secret.", - "airbyte_secret": true, - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "Long-term Authorized Access Token.", - "airbyte_secret": true, - "type": "string" - }, - "advertiser_id": { - "title": "Advertiser ID", - "description": "The Advertiser ID to filter reports and streams. Let this empty to retrieve all.", - "type": "string" - } - }, - "required": ["app_id", "secret", "access_token"] - }, - { - "title": "Sandbox Access Token", - "type": "object", - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "sandbox_access_token", - "order": 0, - "type": "string" - }, - "advertiser_id": { - "title": "Advertiser ID", - "description": "The Advertiser ID which generated for the developer's Sandbox application.", - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The long-term authorized access token.", - "airbyte_secret": true, - "type": "string" - } - }, - "required": ["advertiser_id", "access_token"] - } - ] - }, - "start_date": { - "title": "Replication Start Date", - "description": "The Start Date in format: YYYY-MM-DD. Any data before this date will not be replicated. If this parameter is not set, all data will be replicated.", - "default": "2016-09-01", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 1, - "type": "string", - "format": "date" - }, - "end_date": { - "title": "End Date", - "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "order": 2, - "type": "string", - "format": "date" - }, - "attribution_window": { - "title": "Attribution Window", - "description": "The attribution window in days.", - "minimum": 0, - "maximum": 364, - "default": 3, - "order": 3, - "type": "integer" - }, - "include_deleted": { - "title": "Include Deleted Data in Reports", - "description": "Set to active if you want to include deleted data in reports.", - "default": false, - "order": 4, - "type": "boolean" - } - } - }, - "supportsIncremental": true, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "title": "CompleteOauthOutputSpecification", - "type": "object", - "properties": { - "access_token": { - "title": "Access Token", - "path_in_connector_config": ["credentials", "access_token"], - "type": "string" - } - }, - "required": ["access_token"] - }, - "complete_oauth_server_input_specification": { - "title": "CompleteOauthServerInputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "type": "string" - }, - "secret": { - "title": "Secret", - "type": "string" - } - }, - "required": ["app_id", "secret"] - }, - "complete_oauth_server_output_specification": { - "title": "CompleteOauthServerOutputSpecification", - "type": "object", - "properties": { - "app_id": { - "title": "App Id", - "path_in_connector_config": ["credentials", "app_id"], - "type": "string" - }, - "secret": { - "title": "Secret", - "path_in_connector_config": ["credentials", "secret"], - "type": "string" - } - }, - "required": ["app_id", "secret"] - } - } - }, - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py deleted file mode 100644 index e42a67f25602..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ /dev/null @@ -1,941 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import json -from abc import ABC, abstractmethod -from datetime import datetime -from decimal import Decimal -from enum import Enum -from functools import total_ordering -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, TypeVar, Union - -import pendulum -import pydantic -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.core import package_name_from_class -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer - -# TikTok Initial release date is September 2016 -DEFAULT_START_DATE = "2016-09-01" -MINIMUM_START_DATE = "2012-01-01" -DEFAULT_END_DATE = str(datetime.now().date()) -NOT_AUDIENCE_METRICS = [ - "reach", - "cost_per_1000_reached", - "frequency", - "secondary_goal_result", - "cost_per_secondary_goal_result", - "secondary_goal_result_rate", - "cash_spend", - "voucher_spend", - "video_play_actions", - "video_watched_2s", - "video_watched_6s", - "average_video_play", - "average_video_play_per_user", - "video_views_p25", - "video_views_p50", - "video_views_p75", - "video_views_p100", - "profile_visits", - "likes", - "comments", - "shares", - "follows", - "clicks_on_music_disc", - "real_time_app_install", - "real_time_app_install_cost", - "app_install", - "total_purchase_value", - "total_onsite_shopping_value", - "onsite_shopping", - "vta_purchase", - "cta_purchase", - "vta_conversion", - "cta_conversion", - "total_pageview", - "complete_payment", - "value_per_complete_payment", - "total_complete_payment_rate", - "profile_visits_rate", - "purchase", - "purchase_rate", - "registration", - "registration_rate", - "sales_lead", - "sales_lead_rate", - "cost_per_app_install", - "cost_per_purchase", - "cost_per_registration", - "total_purchase_value", - "cost_per_sales_lead", - "cost_per_total_sales_lead", - "cost_per_total_app_event_add_to_cart", - "total_app_event_add_to_cart", -] - -T = TypeVar("T") - - -# Hierarchy of classes -# TiktokStream -# ├─AdvertiserIds AdvertiserIds -# └─FullRefreshTiktokStream -# ├─Advertisers (1 advertisers) -# └─IncrementalTiktokStream -# ├─AdGroups (2 ad_groups) -# ├─Ads (3 ads) -# ├─Campaigns (4 campaigns) -# └─BasicReports -# ├─AdsReports (5 ads_reports) -# ├─AdvertisersReports (6 advertisers_reports) -# ├─CampaignsReports (7 campaigns_reports) -# ├─AdGroupsReports (8 ad_groups_reports) -# └─AudienceReport -# ├─AdGroupAudienceReports (9 ad_group_audience_reports) -# | ├─AdGroupAudienceReportsByCountry (10 ad_group_audience_reports_by_country) -# | └─AdGroupAudienceReportsByPlatform (11 ad_group_audience_reports_by_platform) -# ├─AdsAudienceReports (12 ads_audience_reports) -# | ├─AdsAudienceReportsByCountry (13 ads_audience_reports_by_country) -# | ├─AdsAudienceReportsByPlatform (14 ads_audience_reports_by_platform) -# | ├─AdsAudienceReportsByProvince (14 ads_audience_reports_by_platform) -# ├─AdvertisersAudienceReports (15 advertisers_audience_reports) -# | ├─AdvertisersAudienceReportsByCountry (16 advertisers_audience_reports_by_country) -# | └─AdvertisersAudienceReportsByPlatform (17 advertisers_audience_reports_by_platform) -# └─CampaignsAudienceReports (18 campaigns_audience_reports) -# ├─CampaignsAudienceReportsByCountry (19 campaigns_audience_reports_by_country) -# └─CampaignsAudienceReportsByPlatform (20 campaigns_audience_reports_by_platform) - - -@total_ordering -class JsonUpdatedState(pydantic.BaseModel): - current_stream_state: str - stream: T - - def __repr__(self): - """Overrides print view""" - return str(self.dict()) - - def dict(self, **kwargs): - """Overrides default logic. - A new updated stage has to be sent if all advertisers are used only - """ - if not self.stream.is_finished: - return self.current_stream_state - max_updated_at = self.stream.max_cursor_date or "" - return max(max_updated_at, self.current_stream_state) - - def __eq__(self, other): - if isinstance(other, JsonUpdatedState): - return self.current_stream_state == other.current_stream_state - return self.current_stream_state == other - - def __lt__(self, other): - if isinstance(other, JsonUpdatedState): - return self.current_stream_state < other.current_stream_state - return self.current_stream_state < other - - -class ReportLevel(str, Enum): - ADVERTISER = "ADVERTISER" - CAMPAIGN = "CAMPAIGN" - ADGROUP = "ADGROUP" - AD = "AD" - - -class ReportGranularity(str, Enum): - LIFETIME = "LIFETIME" - DAY = "DAY" - HOUR = "HOUR" - - @classmethod - def default(cls): - return cls.DAY - - -class Hourly: - report_granularity = ReportGranularity.HOUR - - -class Daily: - report_granularity = ReportGranularity.DAY - - -class Lifetime: - report_granularity = ReportGranularity.LIFETIME - - -class TiktokException(Exception): - """default exception for custom Tiktok logic""" - - -class TiktokStream(HttpStream, ABC): - # endpoints can have different list names - response_list_field = "list" - - # max value of page - page_size = 1000 - - def __init__(self, **kwargs): - super().__init__(authenticator=kwargs.get("authenticator")) - - self._advertiser_id = kwargs.get("advertiser_id") - self.is_sandbox = kwargs.get("is_sandbox") - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """All responses have the similar structure: - { - "message": "", - "code": , # 0 if error else error unique code - "request_id": "" - "data": { - "page_info": { - "total_number": , - "page": , - "page_size": , - "total_page": - }, - "list": [ - - ] - } - } - """ - data = response.json() - if data["code"]: - raise TiktokException(data) - data = data["data"] - if self.response_list_field in data: - data = data[self.response_list_field] - for record in data: - yield record - - @property - def url_base(self) -> str: - """ - Docs: https://business-api.tiktok.com/marketing_api/docs?id=1701890920013825 - """ - if self.is_sandbox: - return "https://sandbox-ads.tiktok.com/open_api/v1.3/" - return "https://business-api.tiktok.com/open_api/v1.3/" - - def next_page_token(self, *args, **kwargs) -> Optional[Mapping[str, Any]]: - # this data without listing - return None - - def should_retry(self, response: requests.Response) -> bool: - """ - Once the rate limit is met, the server returns "code": 40100 - Docs: https://business-api.tiktok.com/marketing_api/docs?id=1701890997610497 - Retry 50002 as well - it's a server error. - Retry when 504 error: response doesn't consist json, so we need to handle response status code to retry. - """ - try: - data = response.json() - except Exception: - if response.status_code == 504: - self.logger.error("Gateway Timeout: The proxy server did not receive a timely response from the upstream server.") - return super().should_retry(response) - self.logger.error(f"Incorrect JSON response: {response.text}") - raise - if data["code"] in (40100, 50002): - return True - return super().should_retry(response) - - def backoff_time(self, response: requests.Response) -> Optional[float]: - """ - The system uses a second call limit for each developer app. The set limit varies according to the app's call limit level. - """ - # Basic: 10/sec - # Advanced: 20/sec - # Premium: 30/sec - # All apps are set to basic call limit level by default. - # Returns maximum possible delay - return 0.6 - - -class AdvertiserIds(TiktokStream): - """Loading of all possible advertiser ids""" - - primary_key = "advertiser_id" - use_cache = True # it is used in all streams - - transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, authenticator, app_id: int, secret: str, **kwargs): - super().__init__(authenticator=authenticator, advertiser_id=0) - - # for Production env - self._secret = secret - self._app_id = app_id - - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - return {"secret": self._secret, "app_id": self._app_id} - - def path(self, *args, **kwargs) -> str: - return "oauth2/advertiser/get/" - - -class FullRefreshTiktokStream(TiktokStream, ABC): - primary_key = "id" - fields: List[str] = None - - transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) - - @transformer.registerCustomTransform - def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: - """Custom transformation""" - if original_value == "-": - return None - elif isinstance(original_value, float): - return Decimal(original_value) - return original_value - - def __init__(self, start_date: str, end_date: str, **kwargs): - super().__init__(**kwargs) - self.kwargs = kwargs - # convert a start date to TikTok format - # example: "2021-08-24" => "2021-08-24 00:00:00" - self._start_time = pendulum.parse(start_date or DEFAULT_START_DATE).strftime("%Y-%m-%d 00:00:00") - # convert end date to TikTok format - # example: "2021-08-24" => "2021-08-24 00:00:00" - self._end_time = pendulum.parse(end_date or DEFAULT_END_DATE).strftime("%Y-%m-%d 00:00:00") - self.max_cursor_date = None - self._advertiser_ids = [] - - @staticmethod - def convert_array_param(arr: List[Union[str, int]]) -> str: - return json.dumps(arr) - - def get_advertiser_ids(self) -> List[int]: - if self._advertiser_id: - # for sandbox: just return advertiser_id provided in spec - # for production: it will filter only the advertiser id provied in spec - ids = [self._advertiser_id] - else: - # for prod: return list of all available ids from AdvertiserIds stream if the field is empty - # in the connector configuration - advertiser_ids = AdvertiserIds(**self.kwargs).read_records(sync_mode=SyncMode.full_refresh) - ids = [advertiser["advertiser_id"] for advertiser in advertiser_ids] - - self._advertiser_ids = ids - return ids - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """Each stream slice is for separate advertiser id""" - self.get_advertiser_ids() - while self._advertiser_ids: - # self._advertiser_ids need to be exhausted so that JsonUpdatedState knows - # when all stream slices are processed (stream.is_finished) - advertiser_id = self._advertiser_ids.pop(0) - yield {"advertiser_id": advertiser_id} - - @property - def is_finished(self): - return len(self._advertiser_ids) == 0 - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """All responses have the following pagination data: - { - "data": { - "page_info": { - "total_number": < total_item_count >, - "page": < current_page_number >, - "page_size": < page_size >, - "total_page": < total_page_count > - }, - ... - } - } - """ - - page_info = response.json().get("data", {}).get("page_info", {}) - if not page_info: - return None - if page_info["page"] < page_info["total_page"]: - return {"page": page_info["page"] + 1} - return None - - def request_params( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = {"page_size": self.page_size} - if self.fields: - params["fields"] = self.convert_array_param(self.fields) - if stream_slice: - params.update(stream_slice) - if next_page_token: - params.update(next_page_token) - return params - - -class IncrementalTiktokStream(FullRefreshTiktokStream, ABC): - cursor_field = "modify_time" - - def select_cursor_field_value(self, data: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None) -> str: - if not data or not self.cursor_field: - return None - - cursor_field_path = self.cursor_field if isinstance(self.cursor_field, list) else [self.cursor_field] - - # backward capability to support old state objects - if "dimensions" in data: - cursor_field_path = self.deprecated_cursor_field - - result = data - for key in cursor_field_path: - result = result.get(key) - return result - - def unnest_cursor_and_pk(self, record: Mapping[str, Any]): - """ - unnest nested cursor_field and primary_key from nested `dimensions` object to root-level for *_reports streams - """ - - def to_list(s): - if not isinstance(s, list): - s = [s] - return s - - dimensions = record.get("dimensions", {}) - fields = to_list(self.cursor_field) + to_list(self.primary_key) - for field in fields: - if field in dimensions: - record[field] = dimensions.get(field) - return record - - def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs - ) -> Iterable[Mapping]: - """Additional data filtering""" - state_cursor_value = self.select_cursor_field_value(stream_state) or self._start_time - for record in super().parse_response(response=response, stream_state=stream_state, **kwargs): - record = self.unnest_cursor_and_pk(record) - updated_cursor_value = self.select_cursor_field_value(record, stream_slice) - if updated_cursor_value is None: - yield record - elif updated_cursor_value < state_cursor_value: - continue - else: - if not self.max_cursor_date or self.max_cursor_date < updated_cursor_value: - self.max_cursor_date = updated_cursor_value - yield record - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - if not self.cursor_field: - # BasicReports streams are incremental. However, report streams configured to use LIFETIME granularity only work as - # full refresh and don't have a cursor field. There is no state value to extract from the record - return {} - - # needs to save a last state if all advertisers are used before only - current_stream_state_value = (self.select_cursor_field_value(current_stream_state)) or "" - - # a object JsonUpdatedState is related with a current stream and should return a new updated state if needed - if not isinstance(current_stream_state_value, JsonUpdatedState): - current_stream_state_value = JsonUpdatedState(stream=self, current_stream_state=current_stream_state_value) - - # reports streams have cursor fields which be allocated into a nested object - cursor_field_path = self.cursor_field if isinstance(self.cursor_field, list) else [self.cursor_field] - # generate a dict with nested items - # ["key1", "key1"] => {"key1": {"key2": }} - tree_dict = current_stream_state_value - for key in reversed(cursor_field_path): - tree_dict = {key: tree_dict} - return tree_dict - - -class Advertisers(FullRefreshTiktokStream): - """Docs: https://ads.tiktok.com/marketing_api/docs?id=1739593083610113""" - - primary_key = "advertiser_id" - - def request_params( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - stream_slice = stream_slice or {} - return {key: self.convert_array_param(value) for key, value in stream_slice.items()} - - def path(self, *args, **kwargs) -> str: - return "advertiser/info/" - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - ids = self.get_advertiser_ids() - start, end, step = 0, len(ids), 100 - for i in range(start, end, step): - yield {"advertiser_ids": ids[i : min(end, i + step)]} - - -class Audiences(FullRefreshTiktokStream): - """Docs: https://business-api.tiktok.com/portal/docs?id=1739940506015746""" - - page_size = 100 - primary_key = "audience_id" - - def path(self, *args, **kwargs) -> str: - return "dmp/custom_audience/list/" - - -class CreativeAssetsMusic(FullRefreshTiktokStream): - """Docs: https://business-api.tiktok.com/portal/docs?id=1740053909509122""" - - primary_key = "music_id" - response_list_field = "musics" - - def path(self, *args, **kwargs) -> str: - return "file/music/get/" - - -class CreativeAssetsPortfolios(FullRefreshTiktokStream): - """Docs: https://business-api.tiktok.com/portal/docs?id=1766324010279938""" - - page_size = 100 - primary_key = "creative_portfolio_id" - response_list_field = "creative_portfolios" - - def path(self, *args, **kwargs) -> str: - return "creative/portfolio/list/" - - -class Campaigns(IncrementalTiktokStream): - """Docs: https://ads.tiktok.com/marketing_api/docs?id=1739315828649986""" - - primary_key = "campaign_id" - - def path(self, *args, **kwargs) -> str: - return "campaign/get/" - - -class AdGroups(IncrementalTiktokStream): - """Docs: https://ads.tiktok.com/marketing_api/docs?id=1739314558673922""" - - primary_key = "adgroup_id" - - def path(self, *args, **kwargs) -> str: - return "adgroup/get/" - - -class Ads(IncrementalTiktokStream): - """Docs: https://ads.tiktok.com/marketing_api/docs?id=1735735588640770""" - - primary_key = "ad_id" - - def path(self, *args, **kwargs) -> str: - return "ad/get/" - - -class CreativeAssetsImages(IncrementalTiktokStream): - """Docs: https://business-api.tiktok.com/portal/docs?id=1740052016789506""" - - page_size = 100 - primary_key = "image_id" - - def path(self, *args, **kwargs) -> str: - return "file/image/ad/search/" - - -class CreativeAssetsVideos(IncrementalTiktokStream): - """Docs: https://business-api.tiktok.com/portal/docs?id=1740050472224769""" - - page_size = 100 - primary_key = "video_id" - - def path(self, *args, **kwargs) -> str: - return "file/video/ad/search/" - - -class BasicReports(IncrementalTiktokStream, ABC): - """Docs: https://ads.tiktok.com/marketing_api/docs?id=1738864915188737""" - - schema_name = "basic_reports" - report_granularity = None - - spec_id_dimensions = { - ReportLevel.ADVERTISER: "advertiser_id", - ReportLevel.CAMPAIGN: "campaign_id", - ReportLevel.ADGROUP: "adgroup_id", - ReportLevel.AD: "ad_id", - } - - spec_time_dimensions = { - ReportGranularity.DAY: "stat_time_day", - ReportGranularity.HOUR: "stat_time_hour", - } - - @property - def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: - return self._get_reporting_dimensions() - - def __init__(self, **kwargs): - report_granularity = kwargs.pop("report_granularity", None) - self.attribution_window = kwargs.get("attribution_window") or 0 - self.include_deleted = kwargs.get("include_deleted", False) - super().__init__(**kwargs) - - # Important: - # for >= 0.1.13 - granularity is set via inheritance - # for < 0.1.13 - granularity is set via init param - if report_granularity: - self.report_granularity = report_granularity - - @property - def filters(self) -> List[MutableMapping[str, Any]]: - if self.include_deleted: - return [ - {"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"}, - {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"}, - {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"}, - ] - return [] - - @property - @abstractmethod - def report_level(self) -> ReportLevel: - """ - Returns a necessary level value - """ - - @property - def deprecated_cursor_field(self): - if self.report_granularity == ReportGranularity.DAY: - return ["dimensions", "stat_time_day"] - if self.report_granularity == ReportGranularity.HOUR: - return ["dimensions", "stat_time_hour"] - if self.report_granularity == ReportGranularity.LIFETIME: - return ["dimensions", "stat_time_day"] - - @property - def cursor_field(self): - return self.spec_time_dimensions.get(self.report_granularity, []) - - @staticmethod - def _get_time_interval( - start_date: Union[datetime, str], - ending_date: Union[datetime, str], - granularity: ReportGranularity, - attr_window: int = 0, - ) -> Iterable[Tuple[datetime, datetime]]: - """Due to time range restrictions based on the level of granularity of reports, we have to chunk API calls in order - to get the desired time range. - Docs: https://ads.tiktok.com/marketing_api/docs?id=1714590313280513 - :param start_date - Timestamp from which we should start the report - :param granularity - Level of granularity of the report; one of [HOUR, DAY, LIFETIME] - :param atttr_window - The attribution window in days - :return Iterator for pair of start_date and end_date that can be used as request parameters - """ - if isinstance(start_date, str): - start_date = pendulum.parse(start_date).subtract(days=attr_window) - elif isinstance(start_date, datetime): - start_date = start_date.subtract(days=attr_window) - - end_date = pendulum.parse(ending_date) if ending_date else pendulum.now() - - # TikTok API only allows certain amount of days of data based on the reporting granularity - if granularity == ReportGranularity.DAY: - max_interval = 30 - elif granularity == ReportGranularity.HOUR: - max_interval = 1 - elif granularity == ReportGranularity.LIFETIME: - max_interval = 364 - else: - raise ValueError(f"Unsupported reporting granularity: {granularity}, must be one of DAY, HOUR, LIFETIME") - - # for incremental sync with abnormal state produce at least one state message - # by producing at least one stream slice from today - if end_date < start_date: - start_date = end_date - - total_date_diff = end_date - start_date - - iterations = total_date_diff.days // max_interval - - for i in range(iterations + 1): - chunk_start = start_date + pendulum.duration(days=(i * max_interval)) - chunk_end = min(chunk_start + pendulum.duration(days=max_interval, seconds=-1), end_date) - yield chunk_start, chunk_end - - def _get_reporting_dimensions(self): - result = [self.spec_id_dimensions[self.report_level]] - if self.report_granularity in self.spec_time_dimensions: - result.append(self.spec_time_dimensions[self.report_granularity]) - return result - - def _get_metrics(self): - # common metrics for all reporting levels - result = [ - "spend", - "cpc", - "cpm", - "impressions", - "clicks", - "ctr", - "reach", - "cost_per_1000_reached", - "frequency", - "video_play_actions", - "video_watched_2s", - "video_watched_6s", - "average_video_play", - "average_video_play_per_user", - "video_views_p25", - "video_views_p50", - "video_views_p75", - "video_views_p100", - "profile_visits", - "likes", - "comments", - "shares", - "follows", - "clicks_on_music_disc", - "real_time_app_install", - "real_time_app_install_cost", - "app_install", - ] - - if self.report_level == ReportLevel.ADVERTISER and self.report_granularity == ReportGranularity.DAY: - # https://ads.tiktok.com/marketing_api/docs?id=1707957200780290 - result.extend(["cash_spend", "voucher_spend"]) - - if self.report_level in (ReportLevel.CAMPAIGN, ReportLevel.ADGROUP, ReportLevel.AD): - result.extend(["campaign_name"]) - - if self.report_level in (ReportLevel.ADGROUP, ReportLevel.AD): - result.extend( - [ - "campaign_id", - "adgroup_name", - "placement_type", - "tt_app_id", - "tt_app_name", - "mobile_app_id", - "promotion_type", - "dpa_target_audience_type", - ] - ) - - result.extend( - [ - "conversion", - "cost_per_conversion", - "conversion_rate", - "real_time_conversion", - "real_time_cost_per_conversion", - "real_time_conversion_rate", - "result", - "cost_per_result", - "result_rate", - "real_time_result", - "real_time_cost_per_result", - "real_time_result_rate", - "secondary_goal_result", - "cost_per_secondary_goal_result", - "secondary_goal_result_rate", - ] - ) - - if self.report_level == ReportLevel.AD: - result.extend( - [ - "adgroup_id", - "ad_name", - "ad_text", - "total_purchase_value", - "total_onsite_shopping_value", - "onsite_shopping", - "vta_purchase", - "vta_conversion", - "cta_purchase", - "cta_conversion", - "total_pageview", - "complete_payment", - "value_per_complete_payment", - "total_complete_payment_rate", - ] - ) - - return result - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - stream_start = self.select_cursor_field_value(stream_state) or self._start_time - stream_end = self._end_time - - for slice_adv_id in super().stream_slices(**kwargs): - for start_date, end_date in self._get_time_interval(stream_start, stream_end, self.report_granularity, self.attribution_window): - slice = { - "advertiser_id": slice_adv_id["advertiser_id"], - "start_date": start_date.strftime("%Y-%m-%d"), - "end_date": end_date.strftime("%Y-%m-%d"), - } - self.logger.debug( - f'name: {self.name}, advertiser_id: {slice["advertiser_id"]}, slice: {slice["start_date"]} - {slice["end_date"]}' - ) - yield slice - - def path(self, *args, **kwargs) -> str: - return "report/integrated/get/" - - def request_params( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) - - params["advertiser_id"] = stream_slice["advertiser_id"] - params["service_type"] = "AUCTION" - params["report_type"] = "BASIC" - params["data_level"] = f"AUCTION_{self.report_level}" - params["dimensions"] = json.dumps(self._get_reporting_dimensions()) - params["metrics"] = json.dumps(self._get_metrics()) - if self.report_granularity == ReportGranularity.LIFETIME: - params["lifetime"] = "true" - else: - params["start_date"] = stream_slice["start_date"] - params["end_date"] = stream_slice["end_date"] - - if self.filters: - params["filters"] = json.dumps(self.filters) - return params - - def get_json_schema(self) -> Mapping[str, Any]: - """All reports have same schema""" - return ResourceSchemaLoader(package_name_from_class(AdvertiserIds)).get_schema(self.schema_name) - - def select_cursor_field_value(self, data: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None) -> str: - if stream_slice: - return stream_slice["end_date"] - return super().select_cursor_field_value(data) - - -class AdsReports(BasicReports): - """Custom reports for ads""" - - ref_pk = "ad_id" - report_level = ReportLevel.AD - - -class AdvertisersReports(BasicReports): - """Custom reports for advertiser""" - - ref_pk = "advertiser_id" - report_level = ReportLevel.ADVERTISER - - -class CampaignsReports(BasicReports): - """Custom reports for campaigns""" - - ref_pk = "campaign_id" - report_level = ReportLevel.CAMPAIGN - - -class AdGroupsReports(BasicReports): - """Custom reports for adgroups""" - - ref_pk = "adgroup_id" - report_level = ReportLevel.ADGROUP - - -class AudienceReport(BasicReports, ABC): - """Docs: https://ads.tiktok.com/marketing_api/docs?id=1738864928947201""" - - audience_dimensions: List = ["gender", "age"] - schema_name = "audience_reports" - - def _get_metrics(self): - result = super()._get_metrics() - result = [e for e in result if e not in NOT_AUDIENCE_METRICS] - return result - - def _get_reporting_dimensions(self): - result = super()._get_reporting_dimensions() - result += self.audience_dimensions - return result - - def request_params( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) - params["report_type"] = "AUDIENCE" - return params - - -class CampaignsAudienceReports(AudienceReport): - ref_pk = "campaign_id" - report_level = ReportLevel.CAMPAIGN - - -class AdGroupAudienceReports(AudienceReport): - ref_pk = "adgroup_id" - report_level = ReportLevel.ADGROUP - - -class AdsAudienceReports(AudienceReport): - ref_pk = "ad_id" - report_level = ReportLevel.AD - - -class AdvertisersAudienceReports(AudienceReport): - ref_pk = "advertiser_id" - report_level = ReportLevel.ADVERTISER - - -class CampaignsAudienceReportsByCountry(CampaignsAudienceReports): - """Custom reports for campaigns by country""" - - audience_dimensions = ["country_code"] - - -class AdGroupAudienceReportsByCountry(AdGroupAudienceReports): - """Custom reports for adgroups by country""" - - audience_dimensions = ["country_code"] - - -class AdsAudienceReportsByCountry(AdsAudienceReports): - """Custom reports for ads by country""" - - audience_dimensions = ["country_code"] - - -class AdvertisersAudienceReportsByCountry(AdvertisersAudienceReports): - """Custom reports for advertisers by country""" - - audience_dimensions = ["country_code"] - - -class CampaignsAudienceReportsByPlatform(CampaignsAudienceReports): - """Custom reports for campaigns by platform""" - - audience_dimensions = ["platform"] - - -class AdGroupAudienceReportsByPlatform(AdGroupAudienceReports): - """Custom reports for adgroups by platform""" - - audience_dimensions = ["platform"] - - -class AdsAudienceReportsByPlatform(AdsAudienceReports): - """Custom reports for ads by platform""" - - audience_dimensions = ["platform"] - - -class AdvertisersAudienceReportsByPlatform(AdvertisersAudienceReports): - """Custom reports for advertisers by platform""" - - audience_dimensions = ["platform"] - - -class AdsAudienceReportsByProvince(AdsAudienceReports): - """Custom reports for ads by province""" - - audience_dimensions = ["province_id"] diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py index 969410ee24e1..4cd06a02eaf2 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py @@ -3,9 +3,6 @@ import os import pytest -from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH - -os.environ[ENV_REQUEST_CACHE_PATH] = ENV_REQUEST_CACHE_PATH @pytest.fixture(autouse=True) @@ -13,6 +10,13 @@ def patch_sleep(mocker): mocker.patch("time.sleep") -@pytest.fixture(autouse=True) -def disable_cache(mocker): - mocker.patch("source_tiktok_marketing.streams.AdvertiserIds.use_cache", new_callable=mocker.PropertyMock, return_value=False) +@pytest.fixture(name="config") +def config_fixture(): + config = { + "account_id": 123, + "access_token": "TOKEN", + "start_date": "2019-10-10T00:00:00", + "end_date": "2020-10-10T00:00:00", + } + return config + diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py new file mode 100644 index 000000000000..9a0abf1bc253 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py @@ -0,0 +1,18 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template + +ADVERTISERS_FILE = "advertisers" + + +def mock_advertisers_slices(http_mocker: HttpMocker, config: dict): + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", + query_params={"secret": config["credentials"]["secret"], "app_id": config["credentials"]["app_id"]}, + ), + HttpResponse(body=json.dumps(find_template(ADVERTISERS_FILE, __file__)), status_code=200), + ) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py new file mode 100644 index 000000000000..5c32c0b9ea7d --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +from typing import Any, Dict + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = { + "credentials": { + "auth_type": "oauth2.0", + "access_token": "access token", + "app_id": "11111111111111111111", + "secret": "secret" + }, + "start_date": "2024-01-01", + "include_deleted": False + } + + def with_include_deleted(self) -> "ConfigBuilder": + self._config["include_deleted"] = True + return self + + def with_end_date(self, date: str) -> "ConfigBuilder": + self._config["end_date"] = date + return self + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py new file mode 100644 index 000000000000..492c3cae854e --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import json +from unittest import TestCase + +from advetiser_slices import mock_advertisers_slices +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template +from airbyte_protocol.models import SyncMode +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + + +class TestCreativeAssetsMusic(TestCase): + stream_name = "creative_assets_music" + advertiser_id = "872746382648" + + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() + + def config(self): + return ConfigBuilder().build() + + @HttpMocker() + def test_basic_read(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/file/music/get/?page_size=100&advertiser_id=872746382648", + ), + HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), + ) + + output = read(SourceTiktokMarketing(), self.config(), self.catalog()) + assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py new file mode 100644 index 000000000000..90ee18eb1a11 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import json +from unittest import TestCase + +from advetiser_slices import mock_advertisers_slices +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template +from airbyte_protocol.models import SyncMode +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + + +class TestCreativeAssetsPortfolios(TestCase): + stream_name = "creative_assets_portfolios" + advertiser_id = "872746382648" + + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() + + def config(self): + return ConfigBuilder().build() + + @HttpMocker() + def test_basic_read(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/creative/portfolio/list/?page_size=100&advertiser_id=872746382648", + ), + HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), + ) + + output = read(SourceTiktokMarketing(), self.config(), self.catalog()) + assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py new file mode 100644 index 000000000000..ed9f19bde77c --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py @@ -0,0 +1,579 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import json +from unittest import TestCase + +from advetiser_slices import mock_advertisers_slices +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import SyncMode +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + +EMPTY_LIST_RESPONSE = {"code": 0, "message": "ok", "data": {"list": []}} + + +class TestAdsReportHourly(TestCase): + stream_name = "ads_reports_hourly" + advertiser_id = "872746382648" + cursor = "2024-01-01 10:00:00" + cursor_field = "stat_time_hour" + metrics = [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "secondary_goal_result", + "cost_per_secondary_goal_result", + "secondary_goal_result_rate", + "adgroup_id", + "ad_name", + "ad_text", + "total_purchase_value", + "total_onsite_shopping_value", + "onsite_shopping", + "vta_purchase", + "vta_conversion", + "cta_purchase", + "cta_conversion", + "total_pageview", + "complete_payment", + "value_per_complete_payment", + "total_complete_payment_rate", + "spend", + "cpc", + "cpm", + "impressions", + "clicks", + "ctr", + "reach", + "cost_per_1000_reached", + "frequency", + "video_play_actions", + "video_watched_2s", + "video_watched_6s", + "average_video_play", + "average_video_play_per_user", + "video_views_p25", + "video_views_p50", + "video_views_p75", + "video_views_p100", + "profile_visits", + "likes", + "comments", + "shares", + "follows", + "clicks_on_music_disc", + "real_time_app_install", + "real_time_app_install_cost", + "app_install", + ] + + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() + + def config(self): + return ConfigBuilder().with_end_date("2024-01-02").build() + + def state(self): + return ( + StateBuilder() + .with_stream_state( + stream_name=self.stream_name, + state={ + "states": [ + {"partition": {"advertiser_id": self.advertiser_id, "parent_slice": {}}, "cursor": {self.cursor_field: self.cursor}} + ] + }, + ) + .build() + ) + + def mock_response(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_AD", + "dimensions": '["ad_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), + ) + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_AD", + "dimensions": '["ad_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["end_date"], + "end_date": self.config()["end_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), + ) + + @HttpMocker() + def test_basic_read(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + self.mock_response(http_mocker) + + output = read(SourceTiktokMarketing(), self.config(), self.catalog()) + assert len(output.records) == 2 + assert output.records[0].record.data.get("ad_id") is not None + assert output.records[0].record.data.get("stat_time_hour") is not None + + @HttpMocker() + def test_read_with_state(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + self.mock_response(http_mocker) + + output = read( + source=SourceTiktokMarketing(), config=self.config(), catalog=self.catalog(sync_mode=SyncMode.incremental), state=self.state() + ) + + assert len(output.records) == 1 + assert output.state_messages[0].state.stream.stream_state.dict()["states"] == [ + {"cursor": {"stat_time_hour": self.cursor}, "partition": {"advertiser_id": self.advertiser_id, "parent_slice": {}}} + ] + + +class TestAdGroupsReportsHourly(TestCase): + stream_name = "ad_groups_reports_hourly" + advertiser_id = "872746382648" + cursor = "2024-01-01 10:00:00" + cursor_field = "stat_time_hour" + metrics = [ + "campaign_name", + "campaign_id", + "adgroup_name", + "placement_type", + "tt_app_id", + "tt_app_name", + "mobile_app_id", + "promotion_type", + "dpa_target_audience_type", + "conversion", + "cost_per_conversion", + "conversion_rate", + "real_time_conversion", + "real_time_cost_per_conversion", + "real_time_conversion_rate", + "result", + "cost_per_result", + "result_rate", + "real_time_result", + "real_time_cost_per_result", + "real_time_result_rate", + "secondary_goal_result", + "cost_per_secondary_goal_result", + "secondary_goal_result_rate", + "spend", + "cpc", + "cpm", + "impressions", + "clicks", + "ctr", + "reach", + "cost_per_1000_reached", + "frequency", + "video_play_actions", + "video_watched_2s", + "video_watched_6s", + "average_video_play", + "average_video_play_per_user", + "video_views_p25", + "video_views_p50", + "video_views_p75", + "video_views_p100", + "profile_visits", + "likes", + "comments", + "shares", + "follows", + "clicks_on_music_disc", + "real_time_app_install", + "real_time_app_install_cost", + "app_install", + ] + + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() + + def config(self): + return ConfigBuilder().with_end_date("2024-01-02").build() + + def state(self): + return ( + StateBuilder() + .with_stream_state( + stream_name=self.stream_name, + state={ + "states": [ + {"partition": {"advertiser_id": self.advertiser_id, "parent_slice": {}}, "cursor": {self.cursor_field: self.cursor}} + ] + }, + ) + .build() + ) + + @HttpMocker() + def test_basic_read(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), + ) + + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["end_date"], + "end_date": self.config()["end_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), + ) + + output = read(SourceTiktokMarketing(), self.config(), self.catalog()) + assert len(output.records) == 2 + assert output.records[0].record.data.get("adgroup_id") is not None + assert output.records[0].record.data.get("stat_time_hour") is not None + + @HttpMocker() + def test_read_with_state(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), + ) + + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["end_date"], + "end_date": self.config()["end_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), + ) + + output = read( + source=SourceTiktokMarketing(), config=self.config(), catalog=self.catalog(sync_mode=SyncMode.incremental), state=self.state() + ) + + assert len(output.records) == 1 + assert output.state_messages[0].state.stream.stream_state.dict()["states"] == [ + {"cursor": {"stat_time_hour": self.cursor}, "partition": {"advertiser_id": self.advertiser_id, "parent_slice": {}}} + ] + + +class TestAdvertisersReportsHourly(TestCase): + stream_name = "advertisers_reports_hourly" + advertiser_id = "872746382648" + cursor = "2024-01-01 10:00:00" + cursor_field = "stat_time_hour" + metrics = [ + "spend", + "cpc", + "cpm", + "impressions", + "clicks", + "ctr", + "reach", + "cost_per_1000_reached", + "frequency", + "video_play_actions", + "video_watched_2s", + "video_watched_6s", + "average_video_play", + "average_video_play_per_user", + "video_views_p25", + "video_views_p50", + "video_views_p75", + "video_views_p100", + "profile_visits", + "likes", + "comments", + "shares", + "follows", + "clicks_on_music_disc", + "real_time_app_install", + "real_time_app_install_cost", + "app_install", + ] + + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() + + def config(self): + return ConfigBuilder().with_end_date("2024-01-02").build() + + def state(self): + return ( + StateBuilder() + .with_stream_state( + stream_name=self.stream_name, + state={ + "states": [ + {"partition": {"advertiser_id": self.advertiser_id, "parent_slice": {}}, "cursor": {self.cursor_field: self.cursor}} + ] + }, + ) + .build() + ) + + def mock_response(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADVERTISER", + "dimensions": '["advertiser_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), + ) + + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADVERTISER", + "dimensions": '["advertiser_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["end_date"], + "end_date": self.config()["end_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), + ) + + @HttpMocker() + def test_basic_read(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + self.mock_response(http_mocker) + + output = read(SourceTiktokMarketing(), self.config(), self.catalog()) + assert len(output.records) == 2 + assert output.records[0].record.data.get("advertiser_id") is not None + assert output.records[0].record.data.get("stat_time_hour") is not None + + @HttpMocker() + def test_read_with_state(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + self.mock_response(http_mocker) + + output = read( + source=SourceTiktokMarketing(), config=self.config(), catalog=self.catalog(sync_mode=SyncMode.incremental), state=self.state() + ) + + assert len(output.records) == 1 + assert output.state_messages[0].state.stream.stream_state.dict()["states"] == [ + {"cursor": {"stat_time_hour": self.cursor}, "partition": {"advertiser_id": self.advertiser_id, "parent_slice": {}}} + ] + + +class TestCampaignsReportsHourly(TestCase): + stream_name = "campaigns_reports_hourly" + advertiser_id = "872746382648" + cursor = "2024-01-01 10:00:00" + cursor_field = "stat_time_hour" + metrics = [ + "campaign_name", + "spend", + "cpc", + "cpm", + "impressions", + "clicks", + "ctr", + "reach", + "cost_per_1000_reached", + "frequency", + "video_play_actions", + "video_watched_2s", + "video_watched_6s", + "average_video_play", + "average_video_play_per_user", + "video_views_p25", + "video_views_p50", + "video_views_p75", + "video_views_p100", + "profile_visits", + "likes", + "comments", + "shares", + "follows", + "clicks_on_music_disc", + "real_time_app_install", + "real_time_app_install_cost", + "app_install", + ] + + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() + + def config(self): + return ConfigBuilder().with_end_date("2024-01-02").build() + + def state(self): + return ( + StateBuilder() + .with_stream_state( + stream_name=self.stream_name, + state={ + "states": [ + {"partition": {"advertiser_id": self.advertiser_id, "parent_slice": {}}, "cursor": {self.cursor_field: self.cursor}} + ] + }, + ) + .build() + ) + + def mock_response(self, http_mocker: HttpMocker): + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_CAMPAIGN", + "dimensions": '["campaign_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), + ) + + http_mocker.get( + HttpRequest( + url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + query_params={ + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_CAMPAIGN", + "dimensions": '["campaign_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["end_date"], + "end_date": self.config()["end_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + }, + ), + HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), + ) + + @HttpMocker() + def test_basic_read(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + self.mock_response(http_mocker) + + output = read(SourceTiktokMarketing(), self.config(), self.catalog()) + assert len(output.records) == 2 + assert output.records[0].record.data.get("campaign_id") is not None + assert output.records[0].record.data.get("stat_time_hour") is not None + + @HttpMocker() + def test_read_with_state(self, http_mocker: HttpMocker): + mock_advertisers_slices(http_mocker, self.config()) + self.mock_response(http_mocker) + + output = read( + source=SourceTiktokMarketing(), config=self.config(), catalog=self.catalog(sync_mode=SyncMode.incremental), state=self.state() + ) + + assert len(output.records) == 1 + assert output.state_messages[0].state.stream.stream_state.dict()["states"] == [ + {"cursor": {"stat_time_hour": self.cursor}, "partition": {"advertiser_id": self.advertiser_id, "parent_slice": {}}} + ] diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/ad_groups_reports_hourly.json b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/ad_groups_reports_hourly.json new file mode 100644 index 000000000000..af49d8e34aa2 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/ad_groups_reports_hourly.json @@ -0,0 +1,128 @@ +{ + "code": 0, + "message": "ok", + "data": { + "list": [ + { + "dimensions": { + "stat_time_hour": "2024-01-01 09:00:00", + "adgroup_id": "11111111" + }, + "metrics": { + "shares": 0, + "real_time_result": "69", + "result": "69", + "clicks": "69", + "cpc": "0.29", + "real_time_result_rate": "1.18", + "conversion": "0", + "average_video_play_per_user": 1.64, + "cost_per_conversion": "0.00", + "cost_per_secondary_goal_result": null, + "spend": "20.00", + "likes": 36, + "profile_visits": 0, + "clicks_on_music_disc": 0, + "video_play_actions": 5173, + "secondary_goal_result": null, + "tt_app_name": "0", + "ctr": "1.18", + "promotion_type": "Website", + "video_views_p50": 214, + "cpm": "3.43", + "real_time_app_install_cost": 0, + "video_views_p75": 140, + "mobile_app_id": "0", + "cost_per_1000_reached": "4.16", + "video_watched_6s": 180, + "average_video_play": 1.52, + "adgroup_name": "Ad Group20211020010107", + "video_watched_2s": 686, + "real_time_conversion_rate": "0.00", + "video_views_p100": 92, + "placement_type": "Automatic Placement", + "conversion_rate": "0.00", + "cost_per_result": "0.290", + "real_time_cost_per_result": "0.290", + "tt_app_id": 0, + "secondary_goal_result_rate": null, + "dpa_target_audience_type": null, + "result_rate": "1.18", + "real_time_app_install": 0, + "comments": 0, + "campaign_name": "Website Traffic20211020010104", + "app_install": 0, + "real_time_cost_per_conversion": "0.00", + "impressions": "5830", + "reach": "4806", + "real_time_conversion": "0", + "follows": 0, + "video_views_p25": 513, + "frequency": "1.21", + "campaign_id": 1714125042508817 + }, + "advertiser_id": 872746382648 + }, + { + "dimensions": { + "stat_time_hour": "2024-01-01 10:00:00", + "adgroup_id": "11111111" + }, + "metrics": { + "shares": 0, + "real_time_result": "69", + "result": "69", + "clicks": "69", + "cpc": "0.29", + "real_time_result_rate": "1.18", + "conversion": "0", + "average_video_play_per_user": 1.64, + "cost_per_conversion": "0.00", + "cost_per_secondary_goal_result": null, + "spend": "20.00", + "likes": 36, + "profile_visits": 0, + "clicks_on_music_disc": 0, + "video_play_actions": 5173, + "secondary_goal_result": null, + "tt_app_name": "0", + "ctr": "1.18", + "promotion_type": "Website", + "video_views_p50": 214, + "cpm": "3.43", + "real_time_app_install_cost": 0, + "video_views_p75": 140, + "mobile_app_id": "0", + "cost_per_1000_reached": "4.16", + "video_watched_6s": 180, + "average_video_play": 1.52, + "adgroup_name": "Ad Group20211020010107", + "video_watched_2s": 686, + "real_time_conversion_rate": "0.00", + "video_views_p100": 92, + "placement_type": "Automatic Placement", + "conversion_rate": "0.00", + "cost_per_result": "0.290", + "real_time_cost_per_result": "0.290", + "tt_app_id": 0, + "secondary_goal_result_rate": null, + "dpa_target_audience_type": null, + "result_rate": "1.18", + "real_time_app_install": 0, + "comments": 0, + "campaign_name": "Website Traffic20211020010104", + "app_install": 0, + "real_time_cost_per_conversion": "0.00", + "impressions": "5830", + "reach": "4806", + "real_time_conversion": "0", + "follows": 0, + "video_views_p25": 513, + "frequency": "1.21", + "campaign_id": 1714125042508817 + }, + "advertiser_id": 872746382648 + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/ads_reports_hourly.json b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/ads_reports_hourly.json new file mode 100644 index 000000000000..b288c6c5d819 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/ads_reports_hourly.json @@ -0,0 +1,156 @@ +{ + "code": 0, + "message": "ok", + "data": { + "list": [ + { + "dimensions": { + "stat_time_hour": "2024-01-01 09:00:00", + "ad_id": "11111111" + }, + "metrics": { + "cpm": "3.43", + "shares": 0, + "real_time_cost_per_result": "0.290", + "video_views_p75": 140, + "follows": 0, + "comments": 0, + "mobile_app_id": "0", + "tt_app_id": 0, + "video_watched_6s": 180, + "cost_per_result": "0.290", + "average_video_play_per_user": 1.64, + "cta_purchase": "0", + "real_time_conversion": "0", + "real_time_cost_per_conversion": "0.00", + "promotion_type": "Website", + "video_views_p50": 214, + "cost_per_secondary_goal_result": null, + "ctr": "1.18", + "real_time_result_rate": "1.18", + "real_time_app_install_cost": 0, + "impressions": "5830", + "conversion": "0", + "cta_conversion": "0", + "placement_type": "Automatic Placement", + "profile_visits": 0, + "result": "69", + "cost_per_1000_reached": "4.16", + "video_views_p25": 513, + "campaign_id": 1714125042508817, + "vta_purchase": "0", + "tt_app_name": "0", + "onsite_shopping": "0", + "total_pageview": "0", + "cpc": "0.29", + "complete_payment": "0", + "dpa_target_audience_type": null, + "total_onsite_shopping_value": "0.00", + "vta_conversion": "0", + "spend": "20.00", + "real_time_result": "69", + "secondary_goal_result_rate": null, + "conversion_rate": "0.00", + "secondary_goal_result": null, + "adgroup_name": "Ad Group20211020010107", + "total_purchase_value": "0.00", + "result_rate": "1.18", + "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", + "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", + "likes": 36, + "video_watched_2s": 686, + "real_time_app_install": 0, + "reach": "4806", + "total_complete_payment_rate": "0.00", + "clicks": "69", + "cost_per_conversion": "0.00", + "app_install": 0, + "real_time_conversion_rate": "0.00", + "video_play_actions": 5173, + "value_per_complete_payment": "0.00", + "frequency": "1.21", + "average_video_play": 1.52, + "video_views_p100": 92, + "clicks_on_music_disc": 0, + "adgroup_id": 1714125049901106, + "campaign_name": "Website Traffic20211020010104" + }, + "advertiser_id": 872746382648 + }, + { + "dimensions": { + "stat_time_hour": "2024-01-01 10:00:00", + "ad_id": "11111111" + }, + "metrics": { + "cpm": "3.43", + "shares": 0, + "real_time_cost_per_result": "0.290", + "video_views_p75": 140, + "follows": 0, + "comments": 0, + "mobile_app_id": "0", + "tt_app_id": 0, + "video_watched_6s": 180, + "cost_per_result": "0.290", + "average_video_play_per_user": 1.64, + "cta_purchase": "0", + "real_time_conversion": "0", + "real_time_cost_per_conversion": "0.00", + "promotion_type": "Website", + "video_views_p50": 214, + "cost_per_secondary_goal_result": null, + "ctr": "1.18", + "real_time_result_rate": "1.18", + "real_time_app_install_cost": 0, + "impressions": "5830", + "conversion": "0", + "cta_conversion": "0", + "placement_type": "Automatic Placement", + "profile_visits": 0, + "result": "69", + "cost_per_1000_reached": "4.16", + "video_views_p25": 513, + "campaign_id": 1714125042508817, + "vta_purchase": "0", + "tt_app_name": "0", + "onsite_shopping": "0", + "total_pageview": "0", + "cpc": "0.29", + "complete_payment": "0", + "dpa_target_audience_type": null, + "total_onsite_shopping_value": "0.00", + "vta_conversion": "0", + "spend": "20.00", + "real_time_result": "69", + "secondary_goal_result_rate": null, + "conversion_rate": "0.00", + "secondary_goal_result": null, + "adgroup_name": "Ad Group20211020010107", + "total_purchase_value": "0.00", + "result_rate": "1.18", + "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", + "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", + "likes": 36, + "video_watched_2s": 686, + "real_time_app_install": 0, + "reach": "4806", + "total_complete_payment_rate": "0.00", + "clicks": "69", + "cost_per_conversion": "0.00", + "app_install": 0, + "real_time_conversion_rate": "0.00", + "video_play_actions": 5173, + "value_per_complete_payment": "0.00", + "frequency": "1.21", + "average_video_play": 1.52, + "video_views_p100": 92, + "clicks_on_music_disc": 0, + "adgroup_id": 1714125049901106, + "campaign_name": "Website Traffic20211020010104" + }, + "advertiser_id": 872746382648 + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/advertisers.json b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/advertisers.json new file mode 100644 index 000000000000..d4b8b453a0f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/advertisers.json @@ -0,0 +1,13 @@ +{ + "code": 0, + "message": "ok", + "data": { + "list": [ + { + "advertiser_id": "872746382648", + "name": "test name", + "address": "test address" + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/advertisers_reports_hourly.json b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/advertisers_reports_hourly.json new file mode 100644 index 000000000000..a80ea3740b06 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/advertisers_reports_hourly.json @@ -0,0 +1,84 @@ +{ + "code": 0, + "message": "ok", + "data": { + "list": [ + { + "dimensions": { + "stat_time_hour": "2024-01-01 09:00:00", + "adgroup_id": "advertiser_id" + }, + "metrics": { + "cpm": "4.18", + "impressions": "4787", + "video_watched_6s": 120, + "profile_visits": 0, + "average_video_play_per_user": 1.54, + "shares": 0, + "app_install": 0, + "clicks_on_music_disc": 0, + "likes": 18, + "cpc": "0.42", + "real_time_app_install": 0, + "average_video_play": 1.42, + "cash_spend": "20.00", + "reach": "3938", + "real_time_app_install_cost": 0, + "video_play_actions": 4253, + "video_views_p75": 100, + "video_views_p100": 70, + "video_watched_2s": 471, + "comments": 0, + "cost_per_1000_reached": "5.08", + "follows": 0, + "spend": "20.00", + "ctr": "1.00", + "video_views_p25": 328, + "frequency": "1.22", + "video_views_p50": 144, + "voucher_spend": "0.00", + "clicks": "48" + }, + "advertiser_id": 872746382648 + }, + { + "dimensions": { + "stat_time_hour": "2024-01-01 10:00:00", + "advertiser_id": "11111111" + }, + "metrics": { + "cpm": "4.18", + "impressions": "4787", + "video_watched_6s": 120, + "profile_visits": 0, + "average_video_play_per_user": 1.54, + "shares": 0, + "app_install": 0, + "clicks_on_music_disc": 0, + "likes": 18, + "cpc": "0.42", + "real_time_app_install": 0, + "average_video_play": 1.42, + "cash_spend": "20.00", + "reach": "3938", + "real_time_app_install_cost": 0, + "video_play_actions": 4253, + "video_views_p75": 100, + "video_views_p100": 70, + "video_watched_2s": 471, + "comments": 0, + "cost_per_1000_reached": "5.08", + "follows": 0, + "spend": "20.00", + "ctr": "1.00", + "video_views_p25": 328, + "frequency": "1.22", + "video_views_p50": 144, + "voucher_spend": "0.00", + "clicks": "48" + }, + "advertiser_id": 872746382648 + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/campaigns_reports_hourly.json b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/campaigns_reports_hourly.json new file mode 100644 index 000000000000..2a5ac5965daa --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/campaigns_reports_hourly.json @@ -0,0 +1,82 @@ +{ + "code": 0, + "message": "ok", + "data": { + "list": [ + { + "dimensions": { + "stat_time_hour": "2024-01-01 09:00:00", + "campaign_id": "advertiser_id" + }, + "metrics": { + "video_watched_2s": 493, + "profile_visits": 0, + "ctr": "1.09", + "likes": 18, + "video_play_actions": 4179, + "shares": 0, + "cpc": "0.39", + "cpm": "4.26", + "video_views_p75": 108, + "video_views_p100": 76, + "real_time_app_install_cost": 0, + "video_views_p25": 355, + "spend": "20.00", + "video_watched_6s": 132, + "reach": "3822", + "impressions": "4696", + "real_time_app_install": 0, + "campaign_name": "Website Traffic20211020010104", + "app_install": 0, + "video_views_p50": 164, + "comments": 0, + "follows": 0, + "cost_per_1000_reached": "5.23", + "average_video_play": 1.48, + "average_video_play_per_user": 1.61, + "clicks": "51", + "clicks_on_music_disc": 0, + "frequency": "1.23" + }, + "advertiser_id": 872746382648 + }, + { + "dimensions": { + "stat_time_hour": "2024-01-01 10:00:00", + "campaign_id": "11111111" + }, + "metrics": { + "video_watched_2s": 493, + "profile_visits": 0, + "ctr": "1.09", + "likes": 18, + "video_play_actions": 4179, + "shares": 0, + "cpc": "0.39", + "cpm": "4.26", + "video_views_p75": 108, + "video_views_p100": 76, + "real_time_app_install_cost": 0, + "video_views_p25": 355, + "spend": "20.00", + "video_watched_6s": 132, + "reach": "3822", + "impressions": "4696", + "real_time_app_install": 0, + "campaign_name": "Website Traffic20211020010104", + "app_install": 0, + "video_views_p50": 164, + "comments": 0, + "follows": 0, + "cost_per_1000_reached": "5.23", + "average_video_play": 1.48, + "average_video_play_per_user": 1.61, + "clicks": "51", + "clicks_on_music_disc": 0, + "frequency": "1.23" + }, + "advertiser_id": 872746382648 + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/creative_assets_music.json b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/creative_assets_music.json new file mode 100644 index 000000000000..5a41eb37513a --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/creative_assets_music.json @@ -0,0 +1,42 @@ +{ + "code": 0, + "message": "ok", + "data": { + "musics": [ + { + "music_id": "111111111111111111", + "material_id": "111111111111111111", + "sources": [{ "type": "main page" }], + "author": "test author", + "liked": true, + "cover_url": "https://cover.com", + "url": "https://url.com", + "duration": 4.34, + "style": "rock", + "signature": "signature", + "name": "test name", + "file_name": "test file_name", + "copyright": "copyright", + "create_time": "2024-01-01 12:12:23", + "modify_time": "2024-01-03 12:12:23" + }, + { + "music_id": "22222222222", + "material_id": "22222222222222222", + "sources": [{ "type": "main page" }], + "author": "test author", + "liked": true, + "cover_url": "https://cover.com", + "url": "https://url.com", + "duration": 4.34, + "style": "rock", + "signature": "signature", + "name": "test name", + "file_name": "test file_name", + "copyright": "copyright", + "create_time": "2024-02-01 12:12:23", + "modify_time": "2024-02-03 12:12:23" + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/creative_assets_portfolios.json b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/creative_assets_portfolios.json new file mode 100644 index 000000000000..192a20fdb12d --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/resource/http/response/creative_assets_portfolios.json @@ -0,0 +1,18 @@ +{ + "code": 0, + "message": "ok", + "data": { + "creative_portfolios": [ + { + "creative_portfolio_id": "111111111111111111", + "creative_portfolio_type": "portfolio", + "creative_portfolio_preview_url": "https://url1.com" + }, + { + "creative_portfolio_id": "222222222222222222", + "creative_portfolio_type": "portfolio", + "creative_portfolio_preview_url": "https://url2.com" + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py deleted file mode 100644 index b77b35b3a528..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py +++ /dev/null @@ -1,296 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from decimal import Decimal -from unittest.mock import MagicMock, PropertyMock, patch - -import pendulum -import pytest -import requests -from source_tiktok_marketing.source import get_report_stream -from source_tiktok_marketing.streams import ( - AdGroupsReports, - Ads, - AdsAudienceReports, - AdsReports, - Advertisers, - AdvertisersAudienceReports, - AdvertisersReports, - BasicReports, - CampaignsReports, - Daily, - FullRefreshTiktokStream, - Hourly, - Lifetime, - ReportGranularity, -) - -START_DATE = "2020-01-01" -END_DATE = "2020-03-01" -CONFIG = { - "access_token": "access_token", - "secret": "secret", - "authenticator": None, - "start_date": START_DATE, - "end_date": END_DATE, - "app_id": 1234, - "advertiser_id": 0, - "include_deleted": True, -} -CONFIG_SANDBOX = { - "access_token": "access_token", - "secret": "secret", - "authenticator": None, - "start_date": START_DATE, - "end_date": END_DATE, - "app_id": 1234, - "advertiser_id": 2000, -} -ADV_IDS = [{"advertiser_id": 1}, {"advertiser_id": 2}] - - -@pytest.fixture(scope="module") -def pendulum_now_mock(): - with patch.object(pendulum, "now", return_value=pendulum.parse(END_DATE)): - yield - - -@pytest.fixture(name="advertiser_ids") -def advertiser_ids_fixture(): - with patch("source_tiktok_marketing.streams.AdvertiserIds") as advertiser_ids_stream: - advertiser_ids_stream().read_records = MagicMock(return_value=ADV_IDS) - yield - - -@pytest.mark.parametrize( - "granularity,intervals_len", - [ - (ReportGranularity.LIFETIME, 1), - (ReportGranularity.DAY, 3), - (ReportGranularity.HOUR, 61), - ], -) -def test_get_time_interval(pendulum_now_mock, granularity, intervals_len): - intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity) - assert len(list(intervals)) == intervals_len - - -@patch.object(pendulum, "now", return_value=pendulum.parse("2018-12-25")) -def test_get_time_interval_past(pendulum_now_mock_past): - intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY) - assert len(list(intervals)) == 1 - - -@patch("source_tiktok_marketing.streams.AdvertiserIds.read_records", MagicMock(return_value=[{"advertiser_id": i} for i in range(354)])) -def test_stream_slices_advertisers(): - slices = Advertisers(**CONFIG).stream_slices() - assert len(list(slices)) == 4 # math.ceil(354 / 100) - - -@pytest.mark.parametrize( - "config_name, slices_expected", - [ - (CONFIG, ADV_IDS), - (CONFIG_SANDBOX, [{"advertiser_id": 2000}]), - ], -) -def test_stream_slices_basic_sandbox(advertiser_ids, config_name, slices_expected): - slices = Ads(**config_name).stream_slices() - assert list(slices) == slices_expected - - -@pytest.mark.parametrize( - "granularity, slices_expected", - [ - ( - Lifetime, - [ - {"advertiser_id": 1, "end_date": END_DATE, "start_date": START_DATE}, - {"advertiser_id": 2, "end_date": END_DATE, "start_date": START_DATE}, - ], - ), - ( - Daily, - [ - {"advertiser_id": 1, "end_date": "2020-01-30", "start_date": "2020-01-01"}, - {"advertiser_id": 1, "end_date": "2020-02-29", "start_date": "2020-01-31"}, - {"advertiser_id": 1, "end_date": "2020-03-01", "start_date": "2020-03-01"}, - {"advertiser_id": 2, "end_date": "2020-01-30", "start_date": "2020-01-01"}, - {"advertiser_id": 2, "end_date": "2020-02-29", "start_date": "2020-01-31"}, - {"advertiser_id": 2, "end_date": "2020-03-01", "start_date": "2020-03-01"}, - ], - ), - ], -) -def test_stream_slices_report(advertiser_ids, granularity, slices_expected, pendulum_now_mock): - slices = get_report_stream(AdsReports, granularity)(**CONFIG).stream_slices() - assert list(slices) == slices_expected - - -@pytest.mark.parametrize( - "stream, metrics_number", - [ - (AdsReports, 65), - (AdGroupsReports, 51), - (AdvertisersReports, 29), - (CampaignsReports, 28), - (AdvertisersAudienceReports, 6), - (AdsAudienceReports, 30), - ], -) -def test_basic_reports_get_metrics_day(stream, metrics_number): - metrics = get_report_stream(stream, Daily)(**CONFIG)._get_metrics() - assert len(metrics) == metrics_number - - -@pytest.mark.parametrize( - "stream, metrics_number", - [ - (AdsReports, 65), - (AdGroupsReports, 51), - (AdvertisersReports, 27), - (CampaignsReports, 28), - (AdvertisersAudienceReports, 6), - ], -) -def test_basic_reports_get_metrics_lifetime(stream, metrics_number): - metrics = get_report_stream(stream, Lifetime)(**CONFIG)._get_metrics() - assert len(metrics) == metrics_number - - -@pytest.mark.parametrize( - "stream, dimensions_expected", - [ - (AdsReports, ["ad_id"]), - (AdGroupsReports, ["adgroup_id"]), - (AdvertisersReports, ["advertiser_id"]), - (CampaignsReports, ["campaign_id"]), - (AdvertisersAudienceReports, ["advertiser_id", "gender", "age"]), - ], -) -def test_basic_reports_get_reporting_dimensions_lifetime(stream, dimensions_expected): - dimensions = get_report_stream(stream, Lifetime)(**CONFIG)._get_reporting_dimensions() - assert dimensions == dimensions_expected - - -@pytest.mark.parametrize( - "stream, dimensions_expected", - [ - (AdsReports, ["ad_id", "stat_time_day"]), - (AdGroupsReports, ["adgroup_id", "stat_time_day"]), - (AdvertisersReports, ["advertiser_id", "stat_time_day"]), - (CampaignsReports, ["campaign_id", "stat_time_day"]), - (AdvertisersAudienceReports, ["advertiser_id", "stat_time_day", "gender", "age"]), - ], -) -def test_basic_reports_get_reporting_dimensions_day(stream, dimensions_expected): - dimensions = get_report_stream(stream, Daily)(**CONFIG)._get_reporting_dimensions() - assert dimensions == dimensions_expected - - -@pytest.mark.parametrize( - "granularity, cursor_field_expected", - [ - (Daily, "stat_time_day"), - (Hourly, "stat_time_hour"), - (Lifetime, []), - ], -) -def test_basic_reports_cursor_field(granularity, cursor_field_expected): - ads_reports = get_report_stream(AdsReports, granularity)(**CONFIG) - cursor_field = ads_reports.cursor_field - assert cursor_field == cursor_field_expected - - -@pytest.mark.parametrize( - "granularity, cursor_field_expected", - [ - (Daily, ["dimensions", "stat_time_day"]), - (Hourly, ["dimensions", "stat_time_hour"]), - (Lifetime, ["dimensions", "stat_time_day"]), - ], -) -def test_basic_reports_deprecated_cursor_field(granularity, cursor_field_expected): - ads_reports = get_report_stream(AdsReports, granularity)(**CONFIG) - deprecated_cursor_field = ads_reports.deprecated_cursor_field - assert deprecated_cursor_field == cursor_field_expected - - -def test_request_params(): - stream_slice = {"advertiser_id": 1, "start_date": "2020", "end_date": "2021"} - params = get_report_stream(AdvertisersAudienceReports, Daily)(**CONFIG).request_params(stream_slice=stream_slice) - assert params == { - "advertiser_id": 1, - "data_level": "AUCTION_ADVERTISER", - "dimensions": '["advertiser_id", "stat_time_day", "gender", "age"]', - "end_date": "2021", - "metrics": '["spend", "cpc", "cpm", "impressions", "clicks", "ctr"]', - "filters": '[{"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"}, {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"}, {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"}]', - "page_size": 1000, - "report_type": "AUDIENCE", - "service_type": "AUCTION", - "start_date": "2020", - } - - -def test_get_updated_state(): - with patch.object(Ads, "is_finished", new_callable=PropertyMock) as is_finished: - - ads = Ads(**CONFIG_SANDBOX) - - # initial state. - state = {} - - # state should be empty while stream is reading records - ads.max_cursor_date = "2020-01-08 00:00:00" - is_finished.return_value = False - state1 = ads.get_updated_state(current_stream_state=state, latest_record={}) - assert state1 == {"modify_time": ""} - - # state should be updated only when all records have been read (is_finished = True) - is_finished.return_value = True - state2 = ads.get_updated_state(current_stream_state=state, latest_record={}) - # state2_modify_time is JsonUpdatedState object - state2_modify_time = state2["modify_time"] - assert state2_modify_time.dict() == "2020-01-08 00:00:00" - - -def test_get_updated_state_no_cursor_field(): - """ - Some full_refresh streams (which don't define a cursor) inherit the get_updated_state() method from an incremental - stream. This test verifies that the stream does not attempt to extract the cursor value from the latest record - """ - ads_reports = AdsReports(**CONFIG_SANDBOX) - state1 = ads_reports.get_updated_state(current_stream_state={}, latest_record={}) - assert state1 == {} - - -@pytest.mark.parametrize( - "value, expected", - [ - (["str1", "str2", "str3"], '["str1", "str2", "str3"]'), - ([1, 2, 3], "[1, 2, 3]"), - ], -) -def test_convert_array_param(value, expected): - stream = Advertisers("2021-01-01", "2021-01-02") - test = stream.convert_array_param(value) - assert test == expected - - -def test_no_next_page_token(requests_mock): - stream = Advertisers("2021-01-01", "2021-01-02") - url = stream.url_base + stream.path() - requests_mock.get(url, json={"data": {"page_info": {}}}) - test_response = requests.get(url) - assert stream.next_page_token(test_response) is None - - -@pytest.mark.parametrize( - ("original_value", "expected_value"), - (("-", None), (26.10, Decimal(26.10)), ("some_str", "some_str")), -) -def test_transform_function(original_value, expected_value): - field_schema = {} - assert FullRefreshTiktokStream.transform_function(original_value, field_schema) == expected_value diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py new file mode 100644 index 000000000000..1825fa49d955 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py @@ -0,0 +1,216 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from airbyte_cdk.sources.declarative.types import StreamSlice +from source_tiktok_marketing import SourceTiktokMarketing +from source_tiktok_marketing.components.advertiser_ids_partition_router import ( + MultipleAdvertiserIdsPerPartition, + SingleAdvertiserIdPerPartition, +) +from source_tiktok_marketing.components.hourly_datetime_based_cursor import HourlyDatetimeBasedCursor +from source_tiktok_marketing.components.semi_incremental_record_filter import PerPartitionRecordFilter +from source_tiktok_marketing.components.transformations import TransformEmptyMetrics + + +@pytest.mark.parametrize( + "config, expected", + [ + ({"credentials": {"advertiser_id": "11111111111"}}, "11111111111"), + ({"environment": {"advertiser_id": "2222222222"}}, "2222222222"), + ({"credentials": {"access_token": "access_token"}}, None) + ], +) +def test_get_partition_value_from_config(config, expected): + router = MultipleAdvertiserIdsPerPartition( + parent_stream_configs=[MagicMock()], + config=config, + parameters={ + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_id" + } + ) + actual = router.get_partition_value_from_config() + assert actual == expected + + +@pytest.mark.parametrize( + "config, expected, json_data", + [ + ({"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, + [{"advertiser_ids": '["11111111111"]', "parent_slice": {}}], None), + ({"environment": {"advertiser_id": "2222222222"}}, [{"advertiser_ids": '["2222222222"]', "parent_slice": {}}], None), + ( + {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, + [{"advertiser_ids": '["11111111", "22222222"]', "parent_slice": {}}], + {"code": 0, "message": "ok", "data": + {"list": [{"advertiser_id": "11111111", "advertiser_name": "name"}, + {"advertiser_id": "22222222", "advertiser_name": "name"}]} + } + ) + ], +) +def test_stream_slices_multiple(config, expected, requests_mock, json_data): + advertiser_ids_stream = [s for s in SourceTiktokMarketing().streams(config=config) if s.name == "advertiser_ids"] + advertiser_ids_stream = advertiser_ids_stream[0] if advertiser_ids_stream else MagicMock() + + router = MultipleAdvertiserIdsPerPartition( + parent_stream_configs=[ParentStreamConfig( + partition_field="advertiser_ids", + config=config, + parent_key="advertiser_id", + stream=advertiser_ids_stream, + parameters={} + )], + config=config, + parameters={ + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_ids" + } + ) + if json_data: + requests_mock.get( + "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", + json=json_data + ) + actual = list(router.stream_slices()) + assert actual == expected + + +@pytest.mark.parametrize( + "config, expected, json_data", + [ + ({"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, [{"advertiser_id": "11111111111", "parent_slice": {}}], + None), + ({"environment": {"advertiser_id": "2222222222"}}, [{"advertiser_id": "2222222222", "parent_slice": {}}], None), + ( + {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, + [{"advertiser_id": "11111111", "parent_slice": {}}, + {"advertiser_id": "22222222", "parent_slice": {}}], + {"code": 0, "message": "ok", + "data": {"list": [ + {"advertiser_id": "11111111", "advertiser_name": "name"}, + {"advertiser_id": "22222222", "advertiser_name": "name"}]} + } + ) + ], +) +def test_stream_slices_single(config, expected, requests_mock, json_data): + advertiser_ids_stream = [s for s in SourceTiktokMarketing().streams(config=config) if s.name == "advertiser_ids"] + advertiser_ids_stream = advertiser_ids_stream[0] if advertiser_ids_stream else MagicMock() + + router = SingleAdvertiserIdPerPartition( + parent_stream_configs=[ParentStreamConfig( + partition_field="advertiser_id", + config=config, + parent_key="advertiser_id", + stream=advertiser_ids_stream, + parameters={} + )], + config=config, + parameters={ + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_id" + } + ) + if json_data: + requests_mock.get( + "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", + json=json_data + ) + actual = list(router.stream_slices()) + assert actual == expected + + +@pytest.mark.parametrize( + "records, state, slice, expected", + [ + ( + [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], + {}, + {}, + [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}] + ), + ( + [{"advertiser_id": 1, "start_time": "2022-01-01"}, {"advertiser_id": 1, "start_time": "2024-01-02"}], + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, + {"advertiser_id": 1}, + [{"advertiser_id": 1, "start_time": "2024-01-02"}] + ), + ( + [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, + {"advertiser_id": 2}, + [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], + ), + ], +) +def test_record_filter(records, state, slice, expected): + config = {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}} + record_filter = PerPartitionRecordFilter( + config=config, + parameters={"partition_field": "advertiser_id"}, + condition="{{ record['start_time'] >= stream_state.get('start_time', config.get('start_date', '')) }}" + ) + filtered_records = list(record_filter.filter_records( + records=records, + stream_state=state, + stream_slice=StreamSlice(partition=slice, cursor_slice={}) + )) + assert filtered_records == expected + + +def test_hourly_datetime_based_cursor(): + config = {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}, "start_date": "2022-01-01", "end_date": "2022-01-02"} + + cursor = HourlyDatetimeBasedCursor( + start_datetime=MinMaxDatetime(datetime="{{ config.get('start_date', '2016-09-01') }}", datetime_format="%Y-%m-%d", parameters={}), + end_datetime=MinMaxDatetime(datetime="{{ config.get('end_date', today_utc()) }}", datetime_format="%Y-%m-%d", parameters={}), + step="P1D", + cursor_granularity="PT1H", + config=config, + cursor_field="stat_time_hour", + datetime_format="%Y-%m-%d", + cursor_datetime_formats=["%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"], + parameters={} + ) + cursor._cursor = "2022-01-01 00:00:00" + partition_daterange = list(cursor.stream_slices()) + assert partition_daterange == [ + {"start_time": "2022-01-01", "end_time": "2022-01-01"}, + {"start_time": "2022-01-02", "end_time": "2022-01-02"} + ] + + cursor._cursor = "2022-01-01 10:00:00" + partition_daterange = list(cursor.stream_slices()) + assert partition_daterange == [ + {"start_time": "2022-01-01", "end_time": "2022-01-01"}, + {"start_time": "2022-01-02", "end_time": "2022-01-02"} + ] + + +@pytest.mark.parametrize( + "record, expected", + [ + ( + {"metrics": {"metric_1": "not empty", "metric_2": "-"}}, + {"metrics": {"metric_1": "not empty", "metric_2": None}} + ), + ( + {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}, + {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}} + ), + ( + {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, + {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}} + ), + ( + {}, + {} + ), + ], +) +def test_transform_empty_metrics(record, expected): + transformer = TransformEmptyMetrics() + actual_record = transformer.transform(record) + assert actual_record == expected diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py new file mode 100644 index 000000000000..c573988deef7 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.models import ConnectorSpecification +from source_tiktok_marketing import SourceTiktokMarketing + + +@pytest.mark.parametrize( + "config, stream_len", + [ + ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01"}, 36), + ({"access_token": "token", "start_date": "2021-01-01", "environment": {"advertiser_id": "1111"}}, 28), + ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01", "report_granularity": "LIFETIME"}, 15), + ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01", "report_granularity": "DAY"}, 27), + ], +) +def test_source_streams(config, stream_len): + streams = SourceTiktokMarketing().streams(config=config) + assert len(streams) == stream_len + + +def test_source_spec(): + spec = SourceTiktokMarketing().spec(logger=None) + assert isinstance(spec, ConnectorSpecification) + + +@pytest.fixture(name="config") +def config_fixture(): + config = { + "account_id": 123, + "access_token": "TOKEN", + "start_date": "2019-10-10T00:00:00", + "end_date": "2020-10-10T00:00:00", + } + return config + + +def test_source_check_connection_ok(config, requests_mock): + requests_mock.get( + "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", + json={"code": 0, "message": "ok", "data": {"list": [{"advertiser_id": "917429327", "advertiser_name": "name"}, ]}} + ) + requests_mock.get( + "https://business-api.tiktok.com/open_api/v1.3/advertiser/info/?page_size=100&advertiser_ids=%5B%22917429327%22%5D", + json={"code": 0, "message": "ok", "data": {"list": [{"advertiser_id": "917429327", "advertiser_name": "name"}, ]}} + ) + logger_mock = MagicMock() + assert SourceTiktokMarketing().check_connection(logger_mock, config) == (True, None) + + +def test_source_check_connection_failed(config, requests_mock): + requests_mock.get( + "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", + json={"code": 40105, "message": "Access token is incorrect or has been revoked."} + ) + logger_mock = MagicMock() + assert SourceTiktokMarketing().check_connection(logger_mock, config) == ( + False, "Unable to connect to stream advertisers - Access token is incorrect or has been revoked." + ) + diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py deleted file mode 100644 index 6377e8eae84f..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ /dev/null @@ -1,198 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -import random -from typing import Any, Dict, Iterable, List, Mapping, Tuple -from unittest.mock import patch - -import pendulum -import pytest -import requests_mock -import timeout_decorator -from airbyte_cdk.models import ConnectorSpecification -from airbyte_cdk.sources.streams.http.exceptions import UserDefinedBackoffException -from source_tiktok_marketing import SourceTiktokMarketing -from source_tiktok_marketing.streams import Ads, Advertisers, JsonUpdatedState - -SANDBOX_CONFIG_FILE = "secrets/sandbox_config.json" -PROD_CONFIG_FILE = "secrets/prod_config.json" - - -@pytest.fixture(scope="module") -def prepared_sandbox_args(): - """Generates streams settings from a file for sandbox""" - with open(SANDBOX_CONFIG_FILE, "r") as f: - return SourceTiktokMarketing._prepare_stream_args(json.loads(f.read())) - - -@pytest.fixture(scope="module") -def prepared_prod_args(): - """Generates streams settings from a file for production""" - with open(PROD_CONFIG_FILE, "r") as f: - return SourceTiktokMarketing._prepare_stream_args(json.loads(f.read())) - - -@timeout_decorator.timeout(20) -@pytest.mark.parametrize("error_code", (40100, 50002)) -def test_backoff(prepared_sandbox_args, error_code): - """TiktokMarketing sends the header 'Retry-After' about needed delay. - All streams have to handle it""" - stream = Advertisers(**prepared_sandbox_args) - with requests_mock.Mocker() as m: - url = stream.url_base + stream.path() - m.get(url, text=json.dumps({"code": error_code})) - with pytest.raises(UserDefinedBackoffException): - list(stream.read_records(sync_mode=None)) - - -def generate_pages(items: List[Mapping[str, Any]], page_size: int, last_empty: bool = False) -> Iterable[Tuple[int, Dict]]: - pages = [] - for i in range(0, len(items), page_size): - pages.append(items[i : i + page_size]) - if last_empty: - pages.append([]) - total_number = len(items) - for page_number, page_items in enumerate(pages, start=1): - yield ( - page_number, - { - "message": "OK", - "code": 0, - "request_id": "unique_request_id", - "data": { - "page_info": {"total_number": total_number, "page": page_number, "page_size": page_size, "total_page": len(page_items)}, - "list": page_items, - }, - }, - ) - - -def random_integer(max_value: int = 1634125471, min_value: int = 1) -> int: - return random.randint(min_value, max_value) - - -def unixtime2str(unix_time: int) -> str: - "Converts unix time to string" - return pendulum.from_timestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S") - - -def test_random_items(prepared_prod_args): - stream = Ads(**prepared_prod_args) - advertiser_count = 100 - test_advertiser_ids = set([str(random_integer()) for _ in range(advertiser_count)]) - advertiser_count = len(test_advertiser_ids) - page_size = 100 - with requests_mock.Mocker() as m: - # mock for advertisers' list - advertisers = [{"advertiser_id": i, "advertiser_name": str(i)} for i in test_advertiser_ids] - for _, page_response in generate_pages(items=advertisers, page_size=advertiser_count): - m.register_uri("GET", "/open_api/v1.3/oauth2/advertiser/get/", json=page_response) - stream = Ads(**prepared_prod_args) - stream.page_size = page_size - stream.get_advertiser_ids() - assert not set(test_advertiser_ids).symmetric_difference(stream._advertiser_ids), "stream found not all advertiser IDs" - - current_state = None - max_updated_value = None - for stream_slice in stream.stream_slices(): - advertiser_id = stream_slice["advertiser_id"] - test_ad_ids = [random_integer() for _ in range(random_integer(max_value=999))] - ad_items = [] - for ad_id in test_ad_ids: - create_time = random_integer(min_value=1507901660) - ad_items.append( - { - "create_time": unixtime2str(create_time), - "modify_time": unixtime2str(create_time + 60), - "advertiser_id": advertiser_id, - "ad_id": ad_id, - } - ) - if not max_updated_value or max_updated_value < ad_items[-1][stream.cursor_field]: - max_updated_value = ad_items[-1][stream.cursor_field] - - # mock for ads - for page, page_response in generate_pages(items=ad_items, page_size=page_size, last_empty=True): - uri = f"/open_api/v1.3/ad/get/?page_size={page_size}&advertiser_id={advertiser_id}" - if page != 1: - uri += f"&page={page}" - m.register_uri("GET", uri, complete_qs=True, json=page_response) - - for record in stream.read_records(sync_mode=None, stream_slice=stream_slice): - current_state = stream.get_updated_state(current_state, record) - assert isinstance(current_state[stream.cursor_field], JsonUpdatedState), "state should be an JsonUpdatedState object" - if advertisers[-1]["advertiser_id"] != advertiser_id: - assert ( - current_state[stream.cursor_field].dict() == "" - ), "max updated cursor value should be returned for last slice only" - assert len(stream._advertiser_ids) == 0, "all advertisers should be popped" - assert current_state[stream.cursor_field].dict() == max_updated_value - - -@pytest.mark.parametrize( - "config, stream_len", - [ - (PROD_CONFIG_FILE, 36), - (SANDBOX_CONFIG_FILE, 28), - ], -) -def test_source_streams(config, stream_len): - with open(config) as f: - config = json.load(f) - streams = SourceTiktokMarketing().streams(config=config) - assert len(streams) == stream_len - - -def test_source_spec(): - spec = SourceTiktokMarketing().spec(logger=None) - assert isinstance(spec, ConnectorSpecification) - - -@pytest.fixture(name="config") -def config_fixture(): - config = { - "account_id": 123, - "access_token": "TOKEN", - "start_date": "2019-10-10T00:00:00", - "end_date": "2020-10-10T00:00:00", - } - return config - - -@pytest.fixture(name="logger_mock") -def logger_mock_fixture(): - return patch("source_tiktok_marketing.source.logger") - - -def test_source_check_connection_ok(config, logger_mock): - with patch.object(Advertisers, "stream_slices"): - with patch.object(Advertisers, "read_records", return_value=iter([1])): - assert SourceTiktokMarketing().check_connection(logger_mock, config=config) == (True, None) - - -def test_source_check_connection_failed(config, logger_mock): - with patch.object(Advertisers, "read_records", return_value=0): - assert SourceTiktokMarketing().check_connection(logger_mock, config=config)[0] is False - - -@pytest.mark.parametrize( - "config_file", - ["integration_tests/invalid_config_oauth.json", "integration_tests/invalid_config_access_token.json", "secrets/config.json"], -) -def test_source_prepare_stream_args(config_file): - with open(config_file) as f: - config = json.load(f) - args = SourceTiktokMarketing._prepare_stream_args(config) - assert "authenticator" in args - - -def test_minimum_start_date(config, caplog): - config["start_date"] = "2000-01-01" - source = SourceTiktokMarketing() - streams = source.streams(config) - - for stream in streams: - assert stream._start_time == "2012-01-01 00:00:00" - assert "The start date is too far in the past. Setting it to 2012-01-01" in caplog.text diff --git a/docs/connector-development/config-based/advanced-topics.md b/docs/connector-development/config-based/advanced-topics.md index 86a9b18cf928..8626826eef7f 100644 --- a/docs/connector-development/config-based/advanced-topics.md +++ b/docs/connector-development/config-based/advanced-topics.md @@ -22,7 +22,7 @@ and instantiate the object from the resulting mapping If the component definition is a mapping with neither a "class_name" nor a "type" field, the factory will do a best-effort attempt at inferring the component type by looking up the parent object's constructor type hints. -If the type hint is an interface present in [DEFAULT_IMPLEMENTATIONS_REGISTRY](https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py, +If the type hint is an interface present in [DEFAULT_IMPLEMENTATIONS_REGISTRY](https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py), then the factory will create an object of its default implementation. If the component definition is a list, then the factory will iterate over the elements of the list, diff --git a/docs/connector-development/config-based/tutorial/1-create-source.md b/docs/connector-development/config-based/tutorial/1-create-source.md index 905aa3a87912..460e4f053189 100644 --- a/docs/connector-development/config-based/tutorial/1-create-source.md +++ b/docs/connector-development/config-based/tutorial/1-create-source.md @@ -14,7 +14,7 @@ cd airbyte-integrations/connector-templates/generator ./generate.sh ``` -This will bring up an interactive helper application. Use the arrow keys to pick a template from the list. Select the `Configuration Based Source` template and then input the name of your connector. The application will create a new directory in `airbyte/airbyte-integrations/connectors/` with the name of your new connector. +This will bring up an interactive helper application. Use the arrow keys to pick a template from the list. Select the `Low-code Source` template and then input the name of your connector. The application will create a new directory in `airbyte/airbyte-integrations/connectors/` with the name of your new connector. The generator will create a new module for your connector with the name `source-`. ``` diff --git a/docs/connector-development/config-based/tutorial/4-reading-data.md b/docs/connector-development/config-based/tutorial/4-reading-data.md index a7deaedbf3f3..ed3fd4aa5f3c 100644 --- a/docs/connector-development/config-based/tutorial/4-reading-data.md +++ b/docs/connector-development/config-based/tutorial/4-reading-data.md @@ -27,7 +27,7 @@ Let's define the stream schema in `source-exchange-rates-tutorial/source_exchang You can download the JSON file describing the output schema with all currencies [here](./exchange_rates_schema.json) for convenience and place it in `schemas/`. ```bash -curl https://raw.githubusercontent.com/airbytehq/airbyte/master/docs/connector-development/tutorials/cdk-tutorial-python-http/exchange_rates_schema.json > source_exchange_rates_tutorial/schemas/rates.json +curl https://raw.githubusercontent.com/airbytehq/airbyte/master/docs/connector-development/config-based/tutorial/exchange_rates_schema.json > source_exchange_rates_tutorial/schemas/rates.json ``` We can also delete the boilerplate schema files diff --git a/docs/contributing-to-airbyte/README.md b/docs/contributing-to-airbyte/README.md index 5452aa458cc0..8df2c876fef4 100644 --- a/docs/contributing-to-airbyte/README.md +++ b/docs/contributing-to-airbyte/README.md @@ -13,9 +13,8 @@ Before getting started, please review Airbyte's Code of Conduct. Everyone intera ## Code Contributions Most of the issues that are open for contributions are tagged with [`good-first-issue`](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Agood-first-issue) or [`help-welcome`](https://github.com/airbytehq/airbyte/labels/help-welcome). -A great place to start looking will be our GitHub projects for [**Community Connector Issues**](https://github.com/orgs/airbytehq/projects/50) -Contributions for new connectors should come with documentation. Refer to the [guidelines on updateing documentation](writing-docs.md) which include a template to use while documentiong a new connector. +Please include documentation when contributing a new connector. Refer to the [guidelines on updating documentation](writing-docs.md) which includes a template to use while documenting a new connector. Proposed updates to a connector should include updates to the connector's documentation. diff --git a/docs/enterprise-setup/upgrading-from-community.md b/docs/enterprise-setup/upgrading-from-community.md index c13a10a90b11..52559328c295 100644 --- a/docs/enterprise-setup/upgrading-from-community.md +++ b/docs/enterprise-setup/upgrading-from-community.md @@ -16,10 +16,10 @@ These instructions are for you if: ### Step 1: Update Airbyte Open Source -You must first update to the latest Open Source community release. We assume you are running the following steps from the root of the `airbytehq/airbyte-platform` cloned repo. +You must first update to the latest Open Source Community release. We assume you are running the following steps from the root of the `airbytehq/airbyte-platform` cloned repo. 1. Determine your current helm release name by running `helm list`. This will now be referred to as `[RELEASE_NAME]` for the rest of this guide. -2. Upgrade to the latest Open Source community release. The output will now be refered to as `[RELEASE_VERSION]` for the rest of this guide: +2. Upgrade to the latest Open Source Community release. The output will now be refered to as `[RELEASE_VERSION]` for the rest of this guide: ```sh helm upgrade [RELEASE_NAME] airbyte/airbyte diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 12f4db6a1bca..adeaa930026d 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -14,7 +14,7 @@ Learn more about contributing to Airbyte [here](/contributing-to-airbyte/). ## Connector Support Levels -Airbyte uses a tiered system for connectors to help you understand what to expect from a connector. In short, there are three tiers: Certified Connectors, Community Connectors, and Custom Connectors. Review the documentation on [connector support levels](./connector-support-levels.md) for details on each tier. +Airbyte uses a tiered system for connectors to help you understand what to expect from a connector. In short, there are three tiers: Integrations, Marketplace Connectors, and Custom Connectors. Review the documentation on [connector support levels](./connector-support-levels.md) for details on each tier. _[View the connector registries in full](https://connectors.airbyte.com/files/generated_reports/connector_registry_report.html)_ diff --git a/docs/integrations/connector-support-levels.md b/docs/integrations/connector-support-levels.md index a46ae6a9f23c..1c9394c8e268 100644 --- a/docs/integrations/connector-support-levels.md +++ b/docs/integrations/connector-support-levels.md @@ -6,10 +6,10 @@ products: all The following table describes the support levels of Airbyte connectors. -| | Certified | Community | Custom | +| | Integration | Marketplace | Custom | | ------------------------------------ | ----------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | **Availability** | Available to all users | Available to all users | Available to all users | -| **Who builds them?** | Either the community or the Airbyte team. | Typically they are built by the community. The Airbyte team may upgrade them to Certified at any time. | Anyone can build custom connectors. We recommend using our [Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview) or [Low-code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview). | +| **Who builds them?** | Either the community or the Airbyte team. | Typically they are built by the community. The Airbyte team may upgrade them to become an Integration at any time. | Anyone can build custom connectors. We recommend using our [Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview) or [Low-code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview). | | **Who maintains them?** | The Airbyte team | Users | Users | | **Production Readiness** | Guaranteed by Airbyte | Not guaranteed | Not guaranteed | | **Support: Cloud** | Supported\* | No Support | Supported\*\* | @@ -17,7 +17,7 @@ The following table describes the support levels of Airbyte connectors. | **Support: Self-Managed Enterprise** | Supported\* | No Support | Supported\*\* | | **Support: Community (OSS)** | Slack Support only | No Support | Slack Support only | -\*For Certified connectors, Official Support SLAs are only available to customers with Premium +\*For Integrations, Official Support SLAs are only available to customers with Premium Support included in their contract. Otherwise, please use our support portal and we will address your issues as soon as possible. @@ -25,35 +25,36 @@ your issues as soon as possible. Support included in their contract. This support is provided with best efforts, and maintenance/upgrades are owned by the customer. -## Certified +## Integrations -A **Certified** connector is actively maintained and supported by the Airbyte team and maintains a +An **Integration** type connector is actively maintained and supported by the Airbyte team and maintains a high quality bar. It is production ready. -### What you should know about Certified connectors: +### What you should know about Integrations: -- Certified connectors are available to all users. -- These connectors have been tested and vetted in order to be certified and are production ready. -- Certified connectors should go through minimal breaking change but in the event an upgrade is +- Integrations are official Airbyte connectors that are available to all users. +- These connectors have been tested and vetted. They are production ready. +- Integrations should go through minimal breaking change but in the event an upgrade is needed users will be given an adequate upgrade window. -## Community +## Marketplace -A **Community** connector is maintained by the Airbyte community until it becomes Certified. Airbyte +A **Marketplace** connector is maintained by the community members until it becomes an official Integration. Airbyte has over 800 code contributors and 15,000 people in the Slack community to help. The Airbyte team is -continually certifying Community connectors as usage grows. As these connectors are not maintained -by Airbyte, we do not offer support SLAs around them, and we encourage caution when using them in +continually reviewing Marketplace connectors as usage grows to determine when a Marketplace connector should become an Airbyte Integration. Marketplace connectors are not maintained +by Airbyte and we do not offer support SLAs around them. We encourage caution when using them in production. -### What you should know about Community connectors: +### What you should know about Marketplace connectors: -- Community connectors are available to all users. -- Community connectors may be upgraded to Certified at any time, and we will notify users of these +- Marketplace connectors are available to all users. +- Marketplace connectors may be upgraded to an official Integration at any time, and we will notify users of these upgrades via our Slack Community and in our Connector Catalog. -- Community connectors might not be feature-complete (features planned for release are under +- Marketplace connectors might not be feature-complete (features planned for release are under development or not prioritized) and may include backward-incompatible/breaking API changes with no or short notice. -- Community connectors have no Support SLAs. +- Marketplace connectors have no Support SLAs. +- You're very welcome to contribute new features and streams to an existing Marketplace integration. Airbyte Contributor Experience team is happy to review PRs when we have capacity. ## Archived diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 637c1017b8e3..3404930e527e 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -244,6 +244,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.2.0 | 2024-07-02 | [40201](https://github.com/airbytehq/airbyte/pull/40201) | Add `_airbyte_generation_id` column, and add `sync_id` to `_airbyte_meta` column | | 3.1.1 | 2024-06-26 | [39008](https://github.com/airbytehq/airbyte/pull/39008) | Internal code changes | | 3.1.0 | 2024-06-26 | [39141](https://github.com/airbytehq/airbyte/pull/39141) | Remove nonfunctional "encrypted staging" option | | 3.0.0 | 2024-06-04 | [38886](https://github.com/airbytehq/airbyte/pull/38886) | Remove standard inserts mode | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 32bededc31fa..8d4609a498fd 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -268,6 +268,7 @@ desired namespace. | Version | Date | Pull Request | Subject | |:----------------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.11.0 | 2024-06-25 | [\#39473](https://github.com/airbytehq/airbyte/pull/39473) | Support for [refreshes](../../operator-guides/refreshes.md) and resumable full refresh. WARNING: You must upgrade to platform 0.63.0 before upgrading to this connector version. | | 3.10.1 | 2024-06-11 | [\#39399](https://github.com/airbytehq/airbyte/pull/39399) | Bug fix for _airbyte_meta not migrated in OVERWRITE mode | | 3.10.0 | 2024-06-10 | [\#39107](https://github.com/airbytehq/airbyte/pull/39107) | _airbyte_meta and _airbyte_generation_id in Raw tables and final tables | | 3.9.1 | 2024-06-05 | [\#39135](https://github.com/airbytehq/airbyte/pull/39135) | Improved error handling for Staging files | diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index a76142f42177..bb0f076c03b0 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -58,7 +58,7 @@ To pass the check for Seller and Vendor accounts, you must have access to the [O 4. Enter a name for the Amazon Seller Partner connector. 5. Click `Authenticate your account`. 6. Log in and Authorize to your Amazon Seller Partner account. -7. For `Start Date`, enter the date in `YYYY-MM-DD` format. The data added on and after this date will be replicated. This field is optional - if not provided, the date 2 years ago from today will be used. +7. For `Start Date`, enter the date in `YYYY-MM-DD` format. The data added on and after this date will be replicated. This field is optional - if not provided or older than 2 years ago from today, the date 2 years ago from today will be used. 8. For `End Date`, enter the date in `YYYY-MM-DD` format. Any data after this date will not be replicated. This field is optional - if not provided, today's date will be used. 9. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). 10. Click `Set up source`. @@ -178,11 +178,12 @@ Information about rate limits you may find [here](https://developer-docs.amazon. | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 4.3.5 | 2024-06-27 | [40215](https://github.com/airbytehq/airbyte/pull/40215) | Replaced deprecated AirbyteLogger with logging.Logger | -| 4.3.4 | 2024-06-25 | [40384](https://github.com/airbytehq/airbyte/pull/40384) | Update dependencies | -| 4.3.3 | 2024-06-22 | [40008](https://github.com/airbytehq/airbyte/pull/40008) | Update dependencies | -| 4.3.2 | 2024-06-13 | [39441](https://github.com/airbytehq/airbyte/pull/39441) | Update state handling for incremental streams | -| 4.3.1 | 2024-06-04 | [38969](https://github.com/airbytehq/airbyte/pull/38969) | [autopull] Upgrade base image to v1.2.1 | +| 4.3.6 | 2024-07-01 | [40590](https://github.com/airbytehq/airbyte/pull/40590) | Add log message when data only accessible to seller accounts, add report id in log message for fatal report status, add check for start date. | +| 4.3.5 | 2024-06-27 | [40215](https://github.com/airbytehq/airbyte/pull/40215) | Replaced deprecated AirbyteLogger with logging.Logger | +| 4.3.4 | 2024-06-25 | [40384](https://github.com/airbytehq/airbyte/pull/40384) | Update dependencies | +| 4.3.3 | 2024-06-22 | [40008](https://github.com/airbytehq/airbyte/pull/40008) | Update dependencies | +| 4.3.2 | 2024-06-13 | [39441](https://github.com/airbytehq/airbyte/pull/39441) | Update state handling for incremental streams | +| 4.3.1 | 2024-06-04 | [38969](https://github.com/airbytehq/airbyte/pull/38969) | [autopull] Upgrade base image to v1.2.1 | | 4.3.0 | 2024-05-24 | [#00000](https://github.com/airbytehq/airbyte/pull/00000) | Extend the report_options spec config with a `stream_name` attribute | | 4.2.4 | 2024-05-15 | [#38210](https://github.com/airbytehq/airbyte/pull/38210) | Fix `GET_VENDOR_TRAFFIC_REPORT` stream with report option `reportPeriod=DAY` | | 4.2.3 | 2024-05-09 | [#38078](https://github.com/airbytehq/airbyte/pull/38078) | Hide OSS-only streams in report options config for cloud users | diff --git a/docs/integrations/sources/clazar.md b/docs/integrations/sources/clazar.md new file mode 100644 index 000000000000..c1718ce47612 --- /dev/null +++ b/docs/integrations/sources/clazar.md @@ -0,0 +1,119 @@ +# Clazar + + + +This page contains the setup guide and reference information for the [Clazar](https://clazar.io/) source connector. + + + +## Prerequisites + +- Clazar Account +- Clazar [API Credentials](https://app.clazar.io/settings/api-access) + +## Setup Guide + +## Step 1: Set up the Clazar connector in Airbyte + + + +**For Airbyte Cloud:** + +1. Generate client credentials by visiting the [API access page](https://app.clazar.io/settings/api-access). +2. Click on `Create API Credentials` in `Integration Type` select **Airbyte** from dropdown and copy the `Client ID` & `Client secret`. +3. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +4. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +5. On the Set up the source page, select **Clazar** from the Source type dropdown and enter the name for the Clazar connector +6. Enter your `Client ID` & `Client secret`. +7. Click **Set up source** + + + + + +**For Airbyte OSS:** + +1. Generate client credentials by visiting the [API access page](https://app.clazar.io/settings/api-access). +2. Click on `Create API Credentials` in `Integration Type` select **Airbyte** from dropdown and copy the `Client ID` & `Client secret`. +3. Navigate to the Airbyte Open Source dashboard. +4. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +5. On the Set up the source page, select **Clazar** from the Source type dropdown and enter the name for the Clazar connector. +6. Enter your `Client ID` & `Client secret`. +7. Click **Set up source** + + + + + +## Supported sync modes + +The Clazar source connector supports the following [sync modes](https://docs.airbyte.com/using-airbyte/core-concepts/sync-modes/): + +| Feature | Supported? | +|:-----------------------------|:-----------| +| Full Refresh Overwrite | Yes | +| Full Refresh Append | Yes | +| Incremental Append | No | +| Incremental Append + Deduped | No | + +## Supported Streams + + +- [Listings](https://developers.clazar.io/reference/get-listings) +- [Private Offers](https://developers.clazar.io/reference/get-private-offers) +- [Buyers](https://developers.clazar.io/reference/get-buyers) +- [Contracts](https://developers.clazar.io/reference/get-contracts-1) +- Opportunities +- Analytics + - AWS + - Disbursement + - Revenue + - Opportunities + - Azure + - Revenue + - Customers + - Orders + - Metered usage + - Opportunities + - GCP + - Disbursement + - Disbursement summary + - Charges & usage + - Daily insights + - Daily incremental insights + - Monthly insights + - Monthly incremental insights + +## Limitations & Troubleshooting + +
+ +Expand to see details about Clazar connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +Clazar Analytics APIs has the rate limits of (30/minute) and for other APIs it's (120/minute), but the connector should not run into API limitations under normal usage. + +Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +### Troubleshooting + +- Check out common troubleshooting issues for the Clazar source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
+ +## Changelog + +
+ Expand to review + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------| +| 0.1.0 | 2024-06-27 | [40562](https://github.com/airbytehq/airbyte/pull/40562) | New Source: Clazar | + +
+ + diff --git a/docs/integrations/sources/google-analytics-v4-service-account-only.md b/docs/integrations/sources/google-analytics-v4-service-account-only.md index a4bbee4cf74d..3d164dfe6d25 100644 --- a/docs/integrations/sources/google-analytics-v4-service-account-only.md +++ b/docs/integrations/sources/google-analytics-v4-service-account-only.md @@ -8,6 +8,12 @@ This connector supports Universal Analytics properties through the [Reporting AP +:::danger + +Google Analytics Universal Analytics Source Connector will be deprecated due to the deprecation of the Google Analytics Universal Analytics API by Google. This deprecation is scheduled by Google on July 1, 2024 (see Google's Documentation for more details). Transition to the Google Analytics 4 (GA4) Source Connector by July 1, 2024, to continue accessing your analytics data. + +::: + :::caution **The Google Analytics (Universal Analytics) connector will be deprecated soon.** @@ -267,7 +273,8 @@ The Google Analytics connector should not run into the "requests per 100 seconds Expand to review | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------- | +|:--------|:-----------| :------------------------------------------------------- |:-----------------------------------------| +| 0.1.0 | 2024-07-01 | [40244](https://github.com/airbytehq/airbyte/pull/40244) | Deprecate the connector | | 0.0.2 | 2024-04-19 | [37432](https://github.com/airbytehq/airbyte/pull/36267) | Fix empty response error for test stream | | 0.0.1 | 2024-01-29 | [34323](https://github.com/airbytehq/airbyte/pull/34323) | Initial Release | diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index 7e001d02d02e..08a60d0e2747 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -8,6 +8,12 @@ This connector supports Universal Analytics properties through the [Reporting AP +:::danger + +Google Analytics Universal Analytics Source Connector will be deprecated due to the deprecation of the Google Analytics Universal Analytics API by Google. This deprecation is scheduled by Google on July 1, 2024 (see Google's Documentation for more details). Transition to the Google Analytics 4 (GA4) Source Connector by July 1, 2024, to continue accessing your analytics data. + +::: + :::caution **The Google Analytics (Universal Analytics) connector will be deprecated soon.** @@ -269,9 +275,10 @@ The Google Analytics connector should not run into the "requests per 100 seconds Expand to review | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------- | -| 0.3.3 | 2024-06-21 | [39940](https://github.com/airbytehq/airbyte/pull/39940) | Update dependencies | -| 0.3.2 | 2024-06-04 | [38934](https://github.com/airbytehq/airbyte/pull/38934) | [autopull] Upgrade base image to v1.2.1 | +|:--------| :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------- | +| 0.4.0 | 2024-07-01 | [40244](https://github.com/airbytehq/airbyte/pull/40244) | Deprecate the connector | +| 0.3.3 | 2024-06-21 | [39940](https://github.com/airbytehq/airbyte/pull/39940) | Update dependencies | +| 0.3.2 | 2024-06-04 | [38934](https://github.com/airbytehq/airbyte/pull/38934) | [autopull] Upgrade base image to v1.2.1 | | 0.3.1 | 2024-04-19 | [37432](https://github.com/airbytehq/airbyte/pull/36267) | Fix empty response error for test stream | | 0.3.0 | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0` | | 0.2.5 | 2024-02-09 | [35101](https://github.com/airbytehq/airbyte/pull/35101) | Manage dependencies with Poetry. | diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 56552865e1c8..9f1a4e4cd7c5 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -115,10 +115,11 @@ Instagram limits the number of requests that can be made at a time. See Facebook Expand to review | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------ | -| 3.0.14 | 2024-06-26 | [39303](https://github.com/airbytehq/airbyte/pull/40524) | Fix Api stream when the results contain not business accounts | +| :------ |:-----------|:---------------------------------------------------------| :------------------------------------------------------------------------------------------------------------------------ | +| 3.0.15 | 2024-07-02 | [40569](https://github.com/airbytehq/airbyte/pull/40569) | Migrate MediaInsights and StoryInsights to low-code | +| 3.0.14 | 2024-06-26 | [40524](https://github.com/airbytehq/airbyte/pull/40524) | Fix Api stream when the results contain not business accounts | | 3.0.13 | 2024-06-25 | [40456](https://github.com/airbytehq/airbyte/pull/40456) | Update dependencies | -| 3.0.12 | 2024-06-24 | [39303](https://github.com/airbytehq/airbyte/pull/39504) | Migrate Media, Users, UserLifeTimeInsights and Stories to low-code | +| 3.0.12 | 2024-06-24 | [39504](https://github.com/airbytehq/airbyte/pull/39504) | Migrate Media, Users, UserLifeTimeInsights and Stories to low-code | | 3.0.11 | 2024-06-22 | [40127](https://github.com/airbytehq/airbyte/pull/40127) | Update dependencies | | 3.0.10 | 2024-06-06 | [39303](https://github.com/airbytehq/airbyte/pull/39303) | [autopull] Upgrade base image to v1.2.2 | | 3.0.9 | 2024-05-21 | [38554](https://github.com/airbytehq/airbyte/pull/38554) | Upgrade to API v19.0 | @@ -152,10 +153,10 @@ Instagram limits the number of requests that can be made at a time. See Facebook | 1.0.0 | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state | | 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | | 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | -| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | -| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | -| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | +| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | +| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | +| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index c365ca32b1b3..9f0375d4a845 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -199,6 +199,7 @@ For more information regarding configuration parameters, please see [MongoDb Doc | Version | Date | Pull Request | Subject | |:--------|:-----------| :------------------------------------------------------- |:----------------------------------------------------------------------------------------------------------| +| 1.4.2 | 2024-07-01 | [40516](https://github.com/airbytehq/airbyte/pull/40516) | Remove dbz hearbeat. | | 1.4.1 | 2024-06-11 | [39530](https://github.com/airbytehq/airbyte/pull/39530) | Adopt new CDK. | | 1.4.0 | 2024-06-11 | [38238](https://github.com/airbytehq/airbyte/pull/38238) | Update mongodbv2 to use dbz 2.6.2 | | 1.3.15 | 2024-05-30 | [38781](https://github.com/airbytehq/airbyte/pull/38781) | Sync sending trace status messages indicating progress. | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 9617df924aac..79d57b3cecdc 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -422,6 +422,8 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 4.0.34 | 2024-07-01 | [40516](https://github.com/airbytehq/airbyte/pull/40516) | Remove dbz hearbeat. | +| 4.0.33 | 2024-06-30 | [40638](https://github.com/airbytehq/airbyte/pull/40638) | Fix a bug that could slow down an initial load of a large table using a different clustered index from the primary key. | | 4.0.32 | 2024-06-26 | [40558](https://github.com/airbytehq/airbyte/pull/40558) | Handle DatetimeOffset correctly. | | 4.0.31 | 2024-06-14 | [39419](https://github.com/airbytehq/airbyte/pull/39419) | Handle DatetimeOffset correctly. | | 4.0.30 | 2024-06-14 | [39349](https://github.com/airbytehq/airbyte/pull/39349) | Full refresh stream sending internal count metadata. | @@ -567,4 +569,4 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | 0.1.5 | 2020-11-30 | [1038](https://github.com/airbytehq/airbyte/pull/1038) | Change JDBC sources to discover more than standard schemas | | 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | - \ No newline at end of file + diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 3a851d7b059a..bb08f32ba977 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -233,6 +233,7 @@ Any database or table encoding combination of charset and collation is supported | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.4.12 | 2024-07-01 | [40516](https://github.com/airbytehq/airbyte/pull/40516) | Remove dbz hearbeat. | | 3.4.11 | 2024-06-26 | [40561](https://github.com/airbytehq/airbyte/pull/40561) | Support PlanetScale MySQL's per-query row limit. | | 3.4.10 | 2024-06-14 | [39349](https://github.com/airbytehq/airbyte/pull/39349) | Full refresh stream sending internal count metadata. | | 3.4.9 | 2024-06-11 | [39405](https://github.com/airbytehq/airbyte/pull/39405) | Adopt latest CDK. | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index fbce88c714d3..05d3ec261c4b 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -310,7 +310,8 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp Expand to review | Version | Date | Pull Request | Subject | -| ------- | ---------- | -------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|---------| ---------- | -------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 3.4.21 | 2024-07-01 | [40516](https://github.com/airbytehq/airbyte/pull/40516) | Remove dbz hearbeat. | | 3.4.20 | 2024-06-23 | [40559](https://github.com/airbytehq/airbyte/pull/40559) | Remove strict check for stream states of unknown types | | 3.4.19 | 2024-06-23 | [40223](https://github.com/airbytehq/airbyte/pull/40223) | Revert the changes introduced in version 3.4.15. | | 3.4.18 | 2024-06-14 | [39349](https://github.com/airbytehq/airbyte/pull/39349) | Full refresh stream sending internal count metadata. | diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index 7aa291dd442f..c6a92d1ede19 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -212,6 +212,7 @@ For all `Shopify GraphQL BULK` api requests these limitations are applied: https | Version | Date | Pull Request | Subject | | :------ |:-----------| :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2.4.7 | 2024-06-27 | [40593](https://github.com/airbytehq/airbyte/pull/40593) | Use latest `CDK` version possible | | 2.4.6 | 2024-06-26 | [40526](https://github.com/airbytehq/airbyte/pull/40526) | Made `BULK Job termination threshold` limit adjustable from `input configuration`, increased the default value to `1 hour`. | | 2.4.5 | 2024-06-25 | [40484](https://github.com/airbytehq/airbyte/pull/40484) | Update dependencies | | 2.4.4 | 2024-06-19 | [39594](https://github.com/airbytehq/airbyte/pull/39594) | Extended the `Discount Codes`, `Fulfillment Orders`, `Inventory Items`, `Inventory Levels`, `Products`, `Product Variants` and `Transactions` stream schemas | diff --git a/docs/integrations/sources/snapchat-marketing-migrations.md b/docs/integrations/sources/snapchat-marketing-migrations.md new file mode 100644 index 000000000000..42bdef63aff5 --- /dev/null +++ b/docs/integrations/sources/snapchat-marketing-migrations.md @@ -0,0 +1,27 @@ +# Snapchat Marketing Migration Guide + +## Upgrading to 1.0.0 + +We're continuously striving to enhance the quality and reliability of our connectors at Airbyte. +As part of our commitment to delivering exceptional service, we are transitioning Snapchat Marketing source from +the Python Connector Development Kit (CDK) to our innovative low-code framework. This is part of a strategic move +to streamline many processes across connectors, bolstering maintainability and freeing us to focus more of our efforts +on improving the performance and features of our evolving platform and growing catalog. However, due to differences +between the Python and low-code CDKs, this migration constitutes a breaking change. + +To gracefully handle these changes for your existing connections, we highly recommend resetting your data before resuming your data syncs with the new version. + +### Migration Steps + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Schema** tab. +3. Uncheck all streams except the affected ones. +4. Select **Save changes** at the bottom of the page. +5. Select the **Settings** tab. +6. Press the **Clear your data** button. +7. Return to the **Schema** tab. +8. Check all your streams. +9. Select **Sync now** to sync your data + +For more information on resetting your data in Airbyte, see [this page](/operator-guides/clear). diff --git a/docs/integrations/sources/snapchat-marketing.md b/docs/integrations/sources/snapchat-marketing.md index f4c25bb35e75..0618170b7297 100644 --- a/docs/integrations/sources/snapchat-marketing.md +++ b/docs/integrations/sources/snapchat-marketing.md @@ -89,14 +89,14 @@ This page guides you through the process of setting up the Snapchat Marketing so ## Supported streams and sync modes | Stream | Incremental | Key | -| :---------------------- | :---------- | ----------------------------------- | +|:------------------------|:------------|-------------------------------------| | Adaccounts | Yes | "id" | | Ads | Yes | "id" | | Adsquads | Yes | "id" | | Campaigns | Yes | "id" | | Creatives | Yes | "id" | | Media | Yes | "id" | -| Organizations | No | "id" | +| Organizations | Yes | "id" | | Segments | Yes | "id" | | AdaccountsStatsHourly | Yes | ["id", "granularity", "start_time"] | | AdaccountsStatsDaily | Yes | ["id", "granularity", "start_time"] | @@ -123,8 +123,9 @@ Snapchat Marketing API has limitations to 1000 items per page. Expand to review | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------- | -| 0.6.2 | 2024-15-22 | [38574](https://github.com/airbytehq/airbyte/pull/38574) | Update authenticator package | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| +| 1.0.0 | 2024-06-20 | [39507](https://github.com/airbytehq/airbyte/pull/39507) | Migrate to low-code CDK and add incremental functionality to `organizations` | +| 0.6.2 | 2024-05-22 | [38574](https://github.com/airbytehq/airbyte/pull/38574) | Update authenticator package | | 0.6.1 | 2024-04-24 | [36662](https://github.com/airbytehq/airbyte/pull/36662) | Schema descriptions | | 0.6.0 | 2024-04-10 | [30586](https://github.com/airbytehq/airbyte/pull/30586) | Add `attribution_windows`,`action_report_time` as optional configurable params | | 0.5.0 | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0` | diff --git a/docs/integrations/sources/tiktok-marketing-migrations.md b/docs/integrations/sources/tiktok-marketing-migrations.md new file mode 100644 index 000000000000..930fadf2c016 --- /dev/null +++ b/docs/integrations/sources/tiktok-marketing-migrations.md @@ -0,0 +1,74 @@ +# TikTok Marketing Migration Guide + +## Upgrading to 4.0.0 + +We're continuously striving to enhance the quality and reliability of our connectors at Airbyte. As part of our commitment to delivering exceptional service, we are transitioning source TikTok Marketing from the Python Connector Development Kit (CDK) to our innovative low-code framework. This is part of a strategic move to streamline many processes across connectors, bolstering maintainability and freeing us to focus more of our efforts on improving the performance and features of our evolving platform and growing catalog. However, due to differences between the Python and low-code CDKs, this migration constitutes a breaking change. + +We’ve evolved and standardized how state is managed for incremental streams that are nested within a parent stream. This change impacts how individual states are tracked and stored for each partition, using a more structured approach to ensure the most granular and flexible state management. + +This change will affect the following streams: +- `ad_group_audience_reports_by_country_daily` +- `ad_group_audience_reports_by_platform_daily` +- `ad_group_audience_reports_daily` +- `ad_groups` +- `ad_groups_reports_daily` +- `ad_groups_reports_hourly` +- `ads` +- `ads_audience_reports_by_country_daily` +- `ads_audience_reports_by_platform_daily` +- `ads_audience_reports_by_province_daily` +- `ads_audience_reports_daily` +- `ads_reports_daily` +- `ads_reports_hourly` +- `advertisers_audience_reports_by_country_daily` +- `advertisers_audience_reports_by_platform_daily` +- `advertisers_audience_reports_daily` +- `advertisers_reports_daily` +- `advertisers_reports_hourly` +- `campaigns` +- `campaigns_audience_reports_by_country_daily` +- `campaigns_audience_reports_by_platform_daily` +- `campaigns_audience_reports_daily` +- `campaigns_reports_daily` +- `campaigns_reports_hourly` +- `creative_assets_images` +- `creative_assets_videos` + +See `Clearing data` to update your connection. + +Schema changes for `advertiser_ids` stream. +Type of advertiser_id field was changed from integer to string to use actual data types as it's declared in API docs. Users will need to refresh stream schema. + +See `Refresh schemas` to update your connection. + +## Migration Steps + +### Refresh schemas + +Refreshing `advertiser_ids` schema is required in order to continue syncing `advertiser_ids` stream data. To refresh schema follow the steps below: + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Schema** tab. + 1. Select **Refresh source schema**. + 2. Select **OK** and **Save changes**. +3. Select **Connections** in the main nav bar. + 1. Select the connection(s) affected by the update. +4. Select the **Status** tab. + 1. In the **Enabled streams** list, click the three dots on the right side of the `advertiser_ids` stream and select **Clear Data**. + +Important: If you were using `advertiser_ids` without provided advertiser_id in the source configuration you should firstly refresh source schema for `advertiser_ids` stream and then clear data for affected streams from the list above. + +### Clearing data + +Clearing your data is required in order to continue syncing affected stream from list above successfully. To clear your data for the streams, follow the steps below: + +1. Select **Connections** in the main nav bar. + 1. Select the connection(s) affected by the update. +2. Select the **Status** tab. + 1. In the **Enabled streams** list, click the three dots on the right side of the impacted stream and select **Clear Data**. +3. Do the same steps from 1-2.1 for all streams in your connection that were affected by this update. + +After the clear succeeds, trigger a sync by clicking **Sync Now**. For more information on clearing your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + + diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index 712ff2431bea..258acf9666e0 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -66,7 +66,7 @@ To access the Sandbox environment: ## Supported streams and sync modes | Stream | Environment | Key | Incremental | -| :---------------------------------------- | :----------- | :----------------------------------------- | :---------- | +| :---------------------------------------- | :----------- | :----------------------------------------- |:------------| | Advertisers | Prod,Sandbox | advertiser_id | No | | AdGroups | Prod,Sandbox | adgroup_id | Yes | | Ads | Prod,Sandbox | ad_id | Yes | @@ -85,10 +85,10 @@ To access the Sandbox environment: | CampaignsReportsDaily | Prod,Sandbox | campaign_id, stat_time_day | Yes | | CampaignsReportsLifetime | Prod,Sandbox | campaign_id | No | | CreativeAssetsImages | Prod,Sandbox | image_id | Yes | -| CreativeAssetsMusic | Prod,Sandbox | music_id | Yes | +| CreativeAssetsMusic | Prod,Sandbox | music_id | No | | CreativeAssetsPortfolios | Prod,Sandbox | creative_portfolio_id | No | | CreativeAssetsVideos | Prod,Sandbox | video_id | Yes | -| AdvertiserIds | Prod | advertiser_id | Yes | +| AdvertiserIds | Prod | advertiser_id | No | | AdvertisersAudienceReportsDaily | Prod | advertiser_id, stat_time_day, gender, age | Yes | | AdvertisersAudienceReportsByCountryDaily | Prod | advertiser_id, stat_time_day, country_code | Yes | | AdvertisersAudienceReportsByPlatformDaily | Prod | advertiser_id, stat_time_day, platform | Yes | @@ -125,58 +125,59 @@ The connector is restricted by [requests limitation](https://business-api.tiktok Expand to review | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------- | -| 3.9.10 | 2024-06-25 | [40373](https://github.com/airbytehq/airbyte/pull/40373) | Update dependencies | -| 3.9.9 | 2024-06-22 | [40133](https://github.com/airbytehq/airbyte/pull/40133) | Update dependencies | -| 3.9.8 | 2024-06-06 | [39253](https://github.com/airbytehq/airbyte/pull/39253) | [autopull] Upgrade base image to v1.2.2 | -| 3.9.7 | 2024-05-15 | [38250](https://github.com/airbytehq/airbyte/pull/38250) | Replace AirbyteLogger with logging.Logger and upgrade to latest base image | -| 3.9.6 | 2024-04-19 | [36665](https://github.com/airbytehq/airbyte/pull/36665) | Updating to 0.80.0 CDK | -| 3.9.5 | 2024-04-12 | [36665](https://github.com/airbytehq/airbyte/pull/36665) | Schema descriptions | -| 3.9.4 | 2024-03-20 | [36302](https://github.com/airbytehq/airbyte/pull/36302) | Don't extract state from the latest record if stream doesn't have a cursor_field | -| 3.9.3 | 2024-02-12 | [35161](https://github.com/airbytehq/airbyte/pull/35161) | Manage dependencies with Poetry. | -| 3.9.2 | 2023-11-02 | [32091](https://github.com/airbytehq/airbyte/pull/32091) | Fix incremental syncs; update docs; fix field type of `preview_url_expire_time` to `date-time`. | -| 3.9.1 | 2023-10-25 | [31812](https://github.com/airbytehq/airbyte/pull/31812) | Update `support level` in `metadata`, removed duplicated `tracking_pixel_id` field from `Ads` stream schema | -| 3.9.0 | 2023-10-23 | [31623](https://github.com/airbytehq/airbyte/pull/31623) | Add AdsAudienceReportsByProvince stream and expand base report metrics | -| 3.8.0 | 2023-10-19 | [31610](https://github.com/airbytehq/airbyte/pull/31610) | Add Creative Assets and Audiences streams | -| 3.7.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 3.7.0 | 2023-10-19 | [31493](https://github.com/airbytehq/airbyte/pull/31493) | Add fields to Ads stream | -| 3.6.0 | 2023-10-18 | [31537](https://github.com/airbytehq/airbyte/pull/31537) | Use default availability strategy | -| 3.5.0 | 2023-10-16 | [31445](https://github.com/airbytehq/airbyte/pull/31445) | Apply minimum date restrictions | -| 3.4.1 | 2023-08-04 | [29083](https://github.com/airbytehq/airbyte/pull/29083) | Added new `is_smart_performance_campaign` property to `ad groups` stream schema | -| 3.4.0 | 2023-07-13 | [27910](https://github.com/airbytehq/airbyte/pull/27910) | Added `include_deleted` config param - include deleted `ad_groups`, `ad`, `campaigns` to reports | -| 3.3.1 | 2023-07-06 | [25423](https://github.com/airbytehq/airbyte/pull/25423) | Add new fields to ad reports streams | -| 3.3.0 | 2023-07-05 | [27988](https://github.com/airbytehq/airbyte/pull/27988) | Add `category_exclusion_ids` field to `ad_groups` schema. | -| 3.2.1 | 2023-05-26 | [26569](https://github.com/airbytehq/airbyte/pull/26569) | Fixed syncs with `advertiser_id` provided in input configuration | -| 3.2.0 | 2023-05-25 | [26565](https://github.com/airbytehq/airbyte/pull/26565) | Change default value for `attribution window` to 3 days; add min/max validation | -| 3.1.0 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Updated the `Ads` stream schema | -| 3.0.1 | 2023-04-07 | [24712](https://github.com/airbytehq/airbyte/pull/24712) | Added `attribution window` for \*-reports streams | -| 3.0.0 | 2023-03-29 | [24630](https://github.com/airbytehq/airbyte/pull/24630) | Migrate to v1.3 API | -| 2.0.6 | 2023-03-30 | [22134](https://github.com/airbytehq/airbyte/pull/22134) | Add `country_code` and `platform` audience reports. | -| 2.0.5 | 2023-03-29 | [22863](https://github.com/airbytehq/airbyte/pull/22863) | Specified date formatting in specification | -| 2.0.4 | 2023-02-23 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Add Advertiser ID to filter reports and streams | -| 2.0.3 | 2023-02-15 | [23091](https://github.com/airbytehq/airbyte/pull/23091) | Add more clear log message for 504 error | -| 2.0.2 | 2023-02-02 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Chunk Advertiser IDs | -| 2.0.1 | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 2.0.0 | 2022-12-20 | [20415](https://github.com/airbytehq/airbyte/pull/20415) | Update schema types for `AudienceReports` and `BasicReports` streams. | -| 1.0.1 | 2022-12-16 | [20598](https://github.com/airbytehq/airbyte/pull/20598) | Remove Audience Reports with Hourly granularity due to deprecated dimension. | -| 1.0.0 | 2022-12-05 | [19758](https://github.com/airbytehq/airbyte/pull/19758) | Convert `mobile_app_id` from integer to string in AudienceReport streams. | -| 0.1.17 | 2022-10-04 | [17557](https://github.com/airbytehq/airbyte/pull/17557) | Retry error 50002 | -| 0.1.16 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream state | -| 0.1.15 | 2022-08-30 | [16137](https://github.com/airbytehq/airbyte/pull/16137) | Fixed bug with normalization caused by unsupported nested cursor field | -| 0.1.14 | 2022-06-29 | [13890](https://github.com/airbytehq/airbyte/pull/13890) | Removed granularity config option | -| 0.1.13 | 2022-06-28 | [13650](https://github.com/airbytehq/airbyte/pull/13650) | Added video metrics to report streams | -| 0.1.12 | 2022-05-24 | [13127](https://github.com/airbytehq/airbyte/pull/13127) | Fixed integration test | -| 0.1.11 | 2022-04-27 | [12838](https://github.com/airbytehq/airbyte/pull/12838) | Added end date configuration for tiktok | -| 0.1.10 | 2022-05-07 | [12545](https://github.com/airbytehq/airbyte/pull/12545) | Removed odd production authenication method | -| 0.1.9 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.8 | 2022-04-28 | [12435](https://github.com/airbytehq/airbyte/pull/12435) | Updated spec descriptions | -| 0.1.7 | 2022-04-27 | [12380](https://github.com/airbytehq/airbyte/pull/12380) | Fixed spec descriptions and documentation | -| 0.1.6 | 2022-04-19 | [11378](https://github.com/airbytehq/airbyte/pull/11378) | Updated logic for stream initializations, fixed errors in schemas, updated SAT and unit tests | -| 0.1.5 | 2022-02-17 | [10398](https://github.com/airbytehq/airbyte/pull/10398) | Add Audience reports | -| 0.1.4 | 2021-12-30 | [7636](https://github.com/airbytehq/airbyte/pull/7636) | Add OAuth support | -| 0.1.3 | 2021-12-10 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | -| 0.1.2 | 2021-12-02 | [8292](https://github.com/airbytehq/airbyte/pull/8292) | Support reports | -| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-09-18 | [5887](https://github.com/airbytehq/airbyte/pull/5887) | Release TikTok Marketing CDK Connector | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------| +| 4.0.0 | 2024-07-01 | [38316](https://github.com/airbytehq/airbyte/pull/38316) | Migration to low-code CDK; Support include deleted statuses for Ads, Ad Groups and Campaign streams. | +| 3.9.10 | 2024-06-25 | [40373](https://github.com/airbytehq/airbyte/pull/40373) | Update dependencies | +| 3.9.9 | 2024-06-22 | [40133](https://github.com/airbytehq/airbyte/pull/40133) | Update dependencies | +| 3.9.8 | 2024-06-06 | [39253](https://github.com/airbytehq/airbyte/pull/39253) | [autopull] Upgrade base image to v1.2.2 | +| 3.9.7 | 2024-05-15 | [38250](https://github.com/airbytehq/airbyte/pull/38250) | Replace AirbyteLogger with logging.Logger and upgrade to latest base image | +| 3.9.6 | 2024-04-19 | [36665](https://github.com/airbytehq/airbyte/pull/36665) | Updating to 0.80.0 CDK | +| 3.9.5 | 2024-04-12 | [36665](https://github.com/airbytehq/airbyte/pull/36665) | Schema descriptions | +| 3.9.4 | 2024-03-20 | [36302](https://github.com/airbytehq/airbyte/pull/36302) | Don't extract state from the latest record if stream doesn't have a cursor_field | +| 3.9.3 | 2024-02-12 | [35161](https://github.com/airbytehq/airbyte/pull/35161) | Manage dependencies with Poetry. | +| 3.9.2 | 2023-11-02 | [32091](https://github.com/airbytehq/airbyte/pull/32091) | Fix incremental syncs; update docs; fix field type of `preview_url_expire_time` to `date-time`. | +| 3.9.1 | 2023-10-25 | [31812](https://github.com/airbytehq/airbyte/pull/31812) | Update `support level` in `metadata`, removed duplicated `tracking_pixel_id` field from `Ads` stream schema | +| 3.9.0 | 2023-10-23 | [31623](https://github.com/airbytehq/airbyte/pull/31623) | Add AdsAudienceReportsByProvince stream and expand base report metrics | +| 3.8.0 | 2023-10-19 | [31610](https://github.com/airbytehq/airbyte/pull/31610) | Add Creative Assets and Audiences streams | +| 3.7.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 3.7.0 | 2023-10-19 | [31493](https://github.com/airbytehq/airbyte/pull/31493) | Add fields to Ads stream | +| 3.6.0 | 2023-10-18 | [31537](https://github.com/airbytehq/airbyte/pull/31537) | Use default availability strategy | +| 3.5.0 | 2023-10-16 | [31445](https://github.com/airbytehq/airbyte/pull/31445) | Apply minimum date restrictions | +| 3.4.1 | 2023-08-04 | [29083](https://github.com/airbytehq/airbyte/pull/29083) | Added new `is_smart_performance_campaign` property to `ad groups` stream schema | +| 3.4.0 | 2023-07-13 | [27910](https://github.com/airbytehq/airbyte/pull/27910) | Added `include_deleted` config param - include deleted `ad_groups`, `ad`, `campaigns` to reports | +| 3.3.1 | 2023-07-06 | [25423](https://github.com/airbytehq/airbyte/pull/25423) | Add new fields to ad reports streams | +| 3.3.0 | 2023-07-05 | [27988](https://github.com/airbytehq/airbyte/pull/27988) | Add `category_exclusion_ids` field to `ad_groups` schema. | +| 3.2.1 | 2023-05-26 | [26569](https://github.com/airbytehq/airbyte/pull/26569) | Fixed syncs with `advertiser_id` provided in input configuration | +| 3.2.0 | 2023-05-25 | [26565](https://github.com/airbytehq/airbyte/pull/26565) | Change default value for `attribution window` to 3 days; add min/max validation | +| 3.1.0 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Updated the `Ads` stream schema | +| 3.0.1 | 2023-04-07 | [24712](https://github.com/airbytehq/airbyte/pull/24712) | Added `attribution window` for \*-reports streams | +| 3.0.0 | 2023-03-29 | [24630](https://github.com/airbytehq/airbyte/pull/24630) | Migrate to v1.3 API | +| 2.0.6 | 2023-03-30 | [22134](https://github.com/airbytehq/airbyte/pull/22134) | Add `country_code` and `platform` audience reports. | +| 2.0.5 | 2023-03-29 | [22863](https://github.com/airbytehq/airbyte/pull/22863) | Specified date formatting in specification | +| 2.0.4 | 2023-02-23 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Add Advertiser ID to filter reports and streams | +| 2.0.3 | 2023-02-15 | [23091](https://github.com/airbytehq/airbyte/pull/23091) | Add more clear log message for 504 error | +| 2.0.2 | 2023-02-02 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Chunk Advertiser IDs | +| 2.0.1 | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 2.0.0 | 2022-12-20 | [20415](https://github.com/airbytehq/airbyte/pull/20415) | Update schema types for `AudienceReports` and `BasicReports` streams. | +| 1.0.1 | 2022-12-16 | [20598](https://github.com/airbytehq/airbyte/pull/20598) | Remove Audience Reports with Hourly granularity due to deprecated dimension. | +| 1.0.0 | 2022-12-05 | [19758](https://github.com/airbytehq/airbyte/pull/19758) | Convert `mobile_app_id` from integer to string in AudienceReport streams. | +| 0.1.17 | 2022-10-04 | [17557](https://github.com/airbytehq/airbyte/pull/17557) | Retry error 50002 | +| 0.1.16 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream state | +| 0.1.15 | 2022-08-30 | [16137](https://github.com/airbytehq/airbyte/pull/16137) | Fixed bug with normalization caused by unsupported nested cursor field | +| 0.1.14 | 2022-06-29 | [13890](https://github.com/airbytehq/airbyte/pull/13890) | Removed granularity config option | +| 0.1.13 | 2022-06-28 | [13650](https://github.com/airbytehq/airbyte/pull/13650) | Added video metrics to report streams | +| 0.1.12 | 2022-05-24 | [13127](https://github.com/airbytehq/airbyte/pull/13127) | Fixed integration test | +| 0.1.11 | 2022-04-27 | [12838](https://github.com/airbytehq/airbyte/pull/12838) | Added end date configuration for tiktok | +| 0.1.10 | 2022-05-07 | [12545](https://github.com/airbytehq/airbyte/pull/12545) | Removed odd production authenication method | +| 0.1.9 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.8 | 2022-04-28 | [12435](https://github.com/airbytehq/airbyte/pull/12435) | Updated spec descriptions | +| 0.1.7 | 2022-04-27 | [12380](https://github.com/airbytehq/airbyte/pull/12380) | Fixed spec descriptions and documentation | +| 0.1.6 | 2022-04-19 | [11378](https://github.com/airbytehq/airbyte/pull/11378) | Updated logic for stream initializations, fixed errors in schemas, updated SAT and unit tests | +| 0.1.5 | 2022-02-17 | [10398](https://github.com/airbytehq/airbyte/pull/10398) | Add Audience reports | +| 0.1.4 | 2021-12-30 | [7636](https://github.com/airbytehq/airbyte/pull/7636) | Add OAuth support | +| 0.1.3 | 2021-12-10 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | +| 0.1.2 | 2021-12-02 | [8292](https://github.com/airbytehq/airbyte/pull/8292) | Support reports | +| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-09-18 | [5887](https://github.com/airbytehq/airbyte/pull/5887) | Release TikTok Marketing CDK Connector | diff --git a/docs/readme.md b/docs/readme.md index fed07952f701..c446b1a1af18 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -12,9 +12,9 @@ There are three major components to know in Airbyte: 1. **The connector catalog** - **350+ pre-built connectors**: Airbyte’s connector catalog comes “out-of-the-box” with over 350 pre-built connectors. These connectors can be used to start replicating data from a source to a destination in just a few minutes. - - **No-Code Connector Builder**: You can easily extend Airbyte’s functionality to support your custom use cases through tools like the [No-Code Connector Builder](/connector-development/connector-builder-ui/overview). + - **No-Code Connector Builder**: You can easily extend Airbyte’s functionality to support your custom use cases through tools like the [No-Code Connector Builder](./connector-development/connector-builder-ui/overview). 2. **The platform:** Airbyte’s platform provides all the horizontal services required to configure and scale data movement operations, available as [cloud-managed](https://airbyte.com/product/airbyte-cloud) or [self-managed](https://airbyte.com/product/airbyte-enterprise). -3. **The user interface:** Airbyte features a UI, [**PyAirbyte**](/using-airbyte/pyairbyte/getting-started) (Python library), [**API**](/api-documentation), and [**Terraform Provider**](/terraform-documentation) to integrate with your preferred tooling and approach to infrastructure management. +3. **The user interface:** Airbyte features a UI, [**PyAirbyte**](./using-airbyte/pyairbyte/getting-started) (Python library), [**API**](./api-documentation), and [**Terraform Provider**](./terraform-documentation) to integrate with your preferred tooling and approach to infrastructure management. Airbyte is suitable for a wide range of data integration use cases, including AI data infrastructure and EL(T) workloads. Airbyte is also [embeddable](https://airbyte.com/product/powered-by-airbyte) within your own application or platform to power your product. @@ -22,18 +22,18 @@ Airbyte is suitable for a wide range of data integration use cases, including AI ## For Airbyte Cloud users -Browse the [connector catalog](/integrations/) to find the connector you want. In case the connector is not yet supported on Airbyte Cloud, consider using [Airbyte Open Source](#for-airbyte-open-source-users). +Browse the [connector catalog](./integrations/) to find the connector you want. In case the connector is not yet supported on Airbyte Cloud, consider using [Airbyte Open Source](#for-airbyte-open-source-users). -Next, check out the [step-by-step tutorial](/using-airbyte/getting-started) to sign up for Airbyte Cloud, understand Airbyte [concepts](/using-airbyte/core-concepts), and run your first sync. +Next, check out the [step-by-step tutorial](./using-airbyte/getting-started) to sign up for Airbyte Cloud, understand Airbyte [concepts](./using-airbyte/core-concepts), and run your first sync. ## For Airbyte Open Source users -Browse the [connector catalog](/integrations/) to find the connector you want. If the connector is not yet supported on Airbyte Open Source, [build your own connector](/connector-development/). +Browse the [connector catalog](./integrations/) to find the connector you want. If the connector is not yet supported on Airbyte Open Source, [build your own connector](./connector-development/). -Next, check out the [Airbyte Open Source QuickStart](/deploying-airbyte/quickstart). Then learn how to [deploy](/deploying-airbyte/quickstart) and [manage](/operator-guides/upgrading-airbyte) Airbyte Open Source in your cloud infrastructure. +Next, check out the [Airbyte Open Source QuickStart](./deploying-airbyte/quickstart). Then learn how to [deploy](./deploying-airbyte/quickstart) and [manage](./operator-guides/upgrading-airbyte) Airbyte Open Source in your cloud infrastructure. ## For Airbyte contributors -To contribute to Airbyte code, connectors, and documentation, refer to our [Contributing Guide](/contributing-to-airbyte/). +To contribute to Airbyte code, connectors, and documentation, refer to our [Contributing Guide](./contributing-to-airbyte/). [![GitHub stars](https://img.shields.io/github/stars/airbytehq/airbyte?style=social&label=Star&maxAge=2592000)](https://GitHub.com/airbytehq/airbyte/stargazers/) [![License](https://img.shields.io/static/v1?label=license&message=MIT&color=brightgreen)](https://github.com/airbytehq/airbyte/tree/a9b1c6c0420550ad5069aca66c295223e0d05e27/LICENSE/README.md) [![License](https://img.shields.io/static/v1?label=license&message=ELv2&color=brightgreen)](https://github.com/airbytehq/airbyte/tree/a9b1c6c0420550ad5069aca66c295223e0d05e27/LICENSE/README.md) diff --git a/docs/release_notes/assets/connection-stream-status-graph.png b/docs/release_notes/assets/connection-stream-status-graph.png new file mode 100644 index 000000000000..2101a4f6d7bc Binary files /dev/null and b/docs/release_notes/assets/connection-stream-status-graph.png differ diff --git a/docs/release_notes/assets/sync-progress.png b/docs/release_notes/assets/sync-progress.png new file mode 100644 index 000000000000..0bc7aac532b9 Binary files /dev/null and b/docs/release_notes/assets/sync-progress.png differ diff --git a/docs/release_notes/june_2024.md b/docs/release_notes/june_2024.md new file mode 100644 index 000000000000..ce267f8911aa --- /dev/null +++ b/docs/release_notes/june_2024.md @@ -0,0 +1,42 @@ +# June 2024 + +## airbyte v0.62.3 to v0.63.4 + +This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms. + +## ✨ Highlights + +Airbyte added [Refresh](operator-guides/refreshes) support for our BigQuery destination, which brings an improved experience to resyncing all of your data again. This enables data to never be deleted from final tables during a historical resyncing of data. More certified destinations will support Refresh in the coming weeks. + +Users can now monitor the incremental progress of syncs on the Connection Status page. The sync progress feature displays record counts and sync duration for each stream during a sync, as well as which streams are syncing and when a connection sync is actively running. The Connection Status Page also now shows counts from the last time each enabled stream synced. Lastly, each stream individually displays stream status, which helps users troubleshoot by indicating exactly which streams are erroring. + +![Sync Progress](./assets/sync-progress.png) + +## Platform Releases + +- (Cloud, Cloud Teams, and Self-Managed Enterprise only) Connections now display both per stream status and tracking of records moved for recent syncs to aid with troubleshooting. + +![Connection Graph](./assets/connection-stream-status-graph.png) + +- (Cloud Teams and Self-Managed Enterprise only) RBAC (Role Based Access Control) now available! New roles have been released to support more defined access for larger teams. Editors (for Workspaces) and Readers (both Organization and Workspace) are now available. Read more in our [docs](/access-management/rbac). + +## Connector Improvements + +We also released a few notable improvements for our connectors: + +- In the Connector Builder, `Undo` and `Redo` have been added, which can also be accessed through cmd-z / cmd-shift-z hotkeys. This allows users to test without fear of losing their previous state within the same session. The Connector Builder is also now a full-screen experience with an improved header menu. Additionally, the `Test` button can now also be triggered by pressing cmd-enter, so users no longer need to manually click on `Test` while actively developing. + +- PyAirbyte now runs YAML sources with no additional installation needed, opening up more ways to sync data. + +- With CDK version 1.5.0, all low-code connectors will have a standardized way of reporting errors. The CDK automatically categorizes errors as config, system, or transient based on a default error mapping. + +- Python (and low-code) [CDK 2.0 updated Pydantic to 2.7](https://github.com/airbytehq/airbyte/pull/39524), which makes it 2x faster for small records. Source-S3 is the first source to benefit from this. + +- Our connector docs now contain Connector Quality Metric indicators. Icons display additional information targeted to help users understand the health of a connector. Sync Success Rate and Usage Rate are now scored 1-3 to better curate the Connector Catalog experience. We’ve also added how recently the connector and its CDK dependency have been updated. + +## Announcements +We announced we will formally deprecate support for Docker Compose deployments in favor of Kubernetes, and welcome beta testers [here](https://github.com/airbytehq/airbyte/discussions/40599) who want migration assistance. You can use [`abctl`](https://github.com/airbytehq/abctl) for local deployments starting today. + +Airbyte is planning to deprecate the Configuration API later this year, and we are looking for feedback on endpoints or data enhancements to improve the Airbyte API [here](https://github.com/airbytehq/airbyte/discussions/39433). + + diff --git a/docs/understanding-airbyte/database-data-catalog.md b/docs/understanding-airbyte/database-data-catalog.md index a9152131011c..0bcc1c6f65a3 100644 --- a/docs/understanding-airbyte/database-data-catalog.md +++ b/docs/understanding-airbyte/database-data-catalog.md @@ -8,7 +8,9 @@ - Each record represents a connector that Airbyte supports, e.g. Postgres. This table represents all the connectors that is supported by the current running platform. - The `actor_type` column tells us whether the record represents a Source or a Destination. - The `spec` column is a JSON blob. The schema of this JSON blob matches the [spec](airbyte-protocol.md#actor-specification) model in the Airbyte Protocol. Because the protocol object is JSON, this has to be a JSON blob. - - The `support_level` describes the support level of the connector (e.g. community, certified). + - The `support_level` describes the support level of the connector (e.g. `community`, `certified`, or `archived`). + - In the product UI, Marketplace contains connectors with `community` support level, and Integrations tab contains `certified` connectors. + - `support_level: archived` signals that this connector is no longer supported, and it's not available to install for any new connections. - The `docker_repository` field is the name of the docker image associated with the connector definition. `docker_image_tag` is the tag of the docker image and the version of the connector definition. - The `source_type` field is only used for Sources, and represents the category of the connector definition (e.g. API, Database). - The `resource_requirements` field sets a default resource requirement for any connector of this type. This overrides the default we set for all connector definitions, and it can be overridden by a connection-specific resource requirement. The column is a JSON blob with the schema defined in [ActorDefinitionResourceRequirements.yaml](https://github.com/airbytehq/airbyte/blob/master/airbyte-config-oss/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml) diff --git a/docs/using-airbyte/getting-started/add-a-destination.md b/docs/using-airbyte/getting-started/add-a-destination.md index 637c7bcde6ec..4a83fc52204b 100644 --- a/docs/using-airbyte/getting-started/add-a-destination.md +++ b/docs/using-airbyte/getting-started/add-a-destination.md @@ -16,7 +16,7 @@ Once you've signed up for Airbyte Cloud or logged in to your Airbyte Open Source You can use the provided search bar at the top of the page, or scroll down the list to find the destination you want to replicate data from. :::tip -You can filter the list of destinations by support level. Airbyte connectors are categorized in two support levels, Certified and Community. See our [Connector Support Levels](/integrations/connector-support-levels.md) page for more information on this topic. +You can filter the list of destinations by support level. Airbyte connectors are categorized in two support levels, Integrations and Marketplace Connectors. See our [Connector Support Levels](/integrations/connector-support-levels.md) page for more information on this topic. ::: diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js index 437a6eea7947..3e601019f617 100644 --- a/docusaurus/sidebars.js +++ b/docusaurus/sidebars.js @@ -693,6 +693,7 @@ module.exports = { type: "generated-index", }, items: [ + "release_notes/june_2024", "release_notes/may_2024", "release_notes/april_2024", "release_notes/march_2024", diff --git a/gradle.properties b/gradle.properties index c3f70a38a4c0..0b1ac35d8a0e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION=0.63.3 +VERSION=0.63.4 # NOTE: some of these values are overwritten in CI! # NOTE: if you want to override this for your local machine, set overrides in ~/.gradle/gradle.properties diff --git a/run-ab-platform.sh b/run-ab-platform.sh index 4618bc08c3d7..f2ebb8f1f260 100755 --- a/run-ab-platform.sh +++ b/run-ab-platform.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERSION=0.63.3 +VERSION=0.63.4 # Run away from anything even a little scary set -o nounset # -u exit if a variable is not set set -o errexit # -f exit for any command failure"