diff --git a/airbyte-api-server/README.md b/airbyte-api-server/README.md new file mode 100644 index 00000000000..bb449695713 --- /dev/null +++ b/airbyte-api-server/README.md @@ -0,0 +1,33 @@ +## Airbyte API Server + +**Purpose:** + +- Enables users to control Airbyte programmatically and use with Orchestration tools (ex: Airflow, Dagster, Prefect) +- Exists for Airbyte users to write applications against and enable [Powered by Airbyte](https://airbyte.com/embed-airbyte-connectors-with-api) ( + Headless version and UI version) + +**Documentation** + +Documentation for the API can be found at https://reference.airbyte.com/ and is powered by readme.io. The documentation currently only fully supports +Airbyte Cloud. We will have pages specific to OSS Airbtye Instances soon! + +The main differences will be configuration inputs for the source and destination create endpoints. +OAuth endpoints and workspace updates are not supported in OSS currently. + +**Local Airbyte Instance Usage** + +*Docker Compose* + +If your instance of Airbyte is running locally using docker-compose, you can access the Airbyte API of the local instance by spinning up with the +latest docker compose files. You can then make a call to `http://localhost:8006/v1/` or the health endpoint +at `http://localhost:8006/health`. Calls to the Airbyte API through docker compose will go through the airbyte-proxy, which requires basic auth with a +user and password supplied in the .env files. + +*Kubernetes* + +If you are running an instance of Airbyte locally using kubernetes, you can access the Airbyte API of the local instance by: + +1. Enabling the airbyte-api-server pod though helm with `helm install %release_name% charts/airbyte --set airbyte-api-server.enabled=true` + or `helm upgrade %release_name% charts/airbyte --set airbyte-api-server.enabled=true` if you already have an instance running. +2. Setting up a port forward to the airbyte-api-server kube svc by running `kubectl port-forward svc/airbyte-airbyte-api-server-svc 8006:80 &` +3. Making a call to `http://localhost:8006/v1/` or the health endpoint at `http://localhost:8006/health`. diff --git a/airbyte-api-server/build.gradle b/airbyte-api-server/build.gradle index 382d5bdd7d2..9cead40234f 100644 --- a/airbyte-api-server/build.gradle +++ b/airbyte-api-server/build.gradle @@ -8,6 +8,8 @@ plugins { } dependencies { + implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' + kapt(platform(libs.micronaut.bom)) kapt(libs.bundles.micronaut.annotation.processor) @@ -18,12 +20,18 @@ dependencies { annotationProcessor libs.bundles.micronaut.annotation.processor annotationProcessor libs.micronaut.jaxrs.processor + implementation project(':airbyte-analytics') implementation project(':airbyte-api') + implementation project(':airbyte-commons') + implementation project(':airbyte-config:config-models') + implementation 'com.cronutils:cron-utils:9.2.1' + implementation libs.bundles.jackson implementation platform(libs.micronaut.bom) implementation libs.bundles.micronaut implementation libs.bundles.micronaut.data.jdbc implementation libs.micronaut.jaxrs.server + implementation libs.micronaut.problem.json implementation libs.micronaut.security implementation libs.sentry.java @@ -42,6 +50,13 @@ dependencies { testImplementation libs.platform.testcontainers.postgresql testImplementation libs.mockwebserver testImplementation libs.mockito.inline + + implementation libs.airbyte.protocol + +} + +kapt { + correctErrorTypes true } Properties env = new Properties() diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/Application.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/Application.kt index 4e08e45d1d6..83353eaec08 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/Application.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/Application.kt @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server import io.micronaut.runtime.Micronaut.run diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/constants/ClientConfigs.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/constants/ClientConfigs.kt new file mode 100644 index 00000000000..51128b0c1f8 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/constants/ClientConfigs.kt @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.constants + +const val INTERNAL_API_HOST = "http://\${airbyte.internal.api.host}" +const val AUTH_HEADER = "Authorization" +const val ENDPOINT_API_USER_INFO_HEADER = "X-Endpoint-API-UserInfo" +const val ANALYTICS_HEADER = "X-Airbyte-Analytic-Source" +const val ANALYTICS_HEADER_VALUE = "airbyte-api" + +const val AIRBYTE_API_AUTH_HEADER_VALUE = "AIRBYTE_API_AUTH_HEADER_VALUE" +const val API_DOC_URL = "https://reference.airbyte.com" diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/constants/ServerConstants.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/constants/ServerConstants.kt new file mode 100644 index 00000000000..a889bfb5e9d --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/constants/ServerConstants.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.constants + +const val SOURCE_TYPE = "sourceType" +const val DESTINATION_TYPE = "destinationType" + +const val CONNECTIONS_PATH = "/v1/connections" +const val CONNECTIONS_WITH_ID_PATH = "$CONNECTIONS_PATH/{connectionId}" +const val STREAMS_PATH = "/v1/streams" +const val JOBS_PATH = "/v1/jobs" +const val JOBS_WITH_ID_PATH = "$JOBS_PATH/{jobId}" +const val SOURCES_PATH = "/v1/sources" +const val INITIATE_OAUTH_PATH = "$SOURCES_PATH/initiateOAuth" +const val SOURCES_WITH_ID_PATH = "$SOURCES_PATH/{sourceId}" +const val DESTINATIONS_PATH = "/v1/destinations" +const val DESTINATIONS_WITH_ID_PATH = "$DESTINATIONS_PATH/{destinationId}" +const val WORKSPACES_PATH = "/v1/workspaces" +const val WORKSPACES_WITH_ID_PATH = "$WORKSPACES_PATH/{workspaceId}" +const val WORKSPACES_WITH_ID_AND_OAUTH_PATH = "$WORKSPACES_WITH_ID_PATH/oauth_credentials" + +val POST = io.micronaut.http.HttpMethod.POST.name +val GET = io.micronaut.http.HttpMethod.GET.name +val PATCH = io.micronaut.http.HttpMethod.PATCH.name +val DELETE = io.micronaut.http.HttpMethod.DELETE.name +val PUT = io.micronaut.http.HttpMethod.PUT.name + +const val WORKSPACE_IDS = "workspaceIds" +const val INCLUDE_DELETED = "includeDeleted" + +const val OAUTH_CALLBACK_PATH = "/v1/oauth/callback" + +const val MESSAGE = "message" + +const val PARTIAL_UPDATE_OAUTH_KEY = "PARTIAL_UPDATE_OAUTH_KEY" + +const val HTTP_RESPONSE_BODY_DEBUG_MESSAGE = "HttpResponse body: " diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt new file mode 100644 index 00000000000..accd5a70157 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.controllers + +import io.airbyte.airbyte_api.generated.ConnectionsApi +import io.airbyte.airbyte_api.model.generated.ConnectionCreateRequest +import io.airbyte.airbyte_api.model.generated.ConnectionPatchRequest +import io.airbyte.airbyte_api.model.generated.ConnectionResponse +import io.airbyte.airbyte_api.model.generated.DestinationResponse +import io.airbyte.api.client.model.generated.AirbyteCatalog +import io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration +import io.airbyte.api.client.model.generated.DestinationSyncMode +import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRead +import io.airbyte.api.server.constants.CONNECTIONS_PATH +import io.airbyte.api.server.constants.CONNECTIONS_WITH_ID_PATH +import io.airbyte.api.server.constants.DELETE +import io.airbyte.api.server.constants.GET +import io.airbyte.api.server.constants.POST +import io.airbyte.api.server.constants.PUT +import io.airbyte.api.server.helpers.AirbyteCatalogHelper +import io.airbyte.api.server.helpers.TrackingHelper +import io.airbyte.api.server.helpers.getLocalUserInfoIfNull +import io.airbyte.api.server.services.ConnectionService +import io.airbyte.api.server.services.DestinationService +import io.airbyte.api.server.services.SourceService +import io.airbyte.api.server.services.UserService +import io.micronaut.http.annotation.Controller +import io.micronaut.http.annotation.Patch +import java.util.Objects +import java.util.UUID +import javax.ws.rs.core.Response + +@Controller(CONNECTIONS_PATH) +open class ConnectionsController( + private val connectionService: ConnectionService, + private val userService: UserService, + private val sourceService: SourceService, + private val destinationService: DestinationService, +) : ConnectionsApi { + override fun createConnection(connectionCreateRequest: ConnectionCreateRequest?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + val validUserInfo: String? = getLocalUserInfoIfNull(userInfo) + + TrackingHelper.callWithTracker({ + AirbyteCatalogHelper.validateCronConfiguration( + connectionCreateRequest!!.schedule, + ) + }, CONNECTIONS_PATH, POST, userId) + + // get destination response to retrieve workspace id as well as input for destination sync modes + val destinationResponse: DestinationResponse = + TrackingHelper.callWithTracker( + { destinationService.getDestination(connectionCreateRequest!!.destinationId, validUserInfo) }, + CONNECTIONS_PATH, + POST, + userId, + ) as DestinationResponse + + // get source schema for catalog id and airbyte catalog + val schemaResponse: SourceDiscoverSchemaRead = TrackingHelper.callWithTracker( + { sourceService.getSourceSchema(connectionCreateRequest!!.sourceId, false, validUserInfo) }, + CONNECTIONS_PATH, + POST, + userId, + ) + val catalogId = schemaResponse.catalogId + + val airbyteCatalogFromDiscoverSchema = schemaResponse.catalog + + // refer to documentation to understand what we need to do for the catalog + // https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#catalog + var configuredCatalog: AirbyteCatalog? = AirbyteCatalog() + + val validStreams: Map = AirbyteCatalogHelper.getValidStreams( + Objects.requireNonNull(airbyteCatalogFromDiscoverSchema), + ) + + // check user configs + if (AirbyteCatalogHelper.hasStreamConfigurations(connectionCreateRequest!!.configurations)) { + // validate user inputs + TrackingHelper.callWithTracker( + { + AirbyteCatalogHelper.validateStreams( + airbyteCatalogFromDiscoverSchema!!, + connectionCreateRequest.configurations, + ) + }, + CONNECTIONS_PATH, + POST, + userId, + ) + + // set user inputs + for (streamConfiguration in connectionCreateRequest.configurations.streams) { + val validStreamAndConfig = validStreams[streamConfiguration.name] + val schemaStream = validStreamAndConfig!!.stream + val schemaConfig = validStreamAndConfig.config + + val validDestinationSyncModes = TrackingHelper.callWithTracker( + { destinationService.getDestinationSyncModes(destinationResponse, validUserInfo) }, + CONNECTIONS_PATH, + POST, + userId, + ) as List + + // set user configs + TrackingHelper.callWithTracker( + { + AirbyteCatalogHelper.setAndValidateStreamConfig( + streamConfiguration, + validDestinationSyncModes, + schemaStream!!, + schemaConfig!!, + ) + }, + CONNECTIONS_PATH, + POST, + userId, + ) + configuredCatalog!!.addStreamsItem(validStreamAndConfig) + } + } else { + // no user supplied stream configs, return all streams with full refresh overwrite + configuredCatalog = airbyteCatalogFromDiscoverSchema + AirbyteCatalogHelper.setAllStreamsFullRefreshOverwrite(configuredCatalog!!) + } + + val finalConfiguredCatalog = configuredCatalog + val connectionResponse: Any = TrackingHelper.callWithTracker({ + connectionService.createConnection( + connectionCreateRequest, + catalogId!!, + finalConfiguredCatalog!!, + destinationResponse.workspaceId, + validUserInfo, + ) + }, CONNECTIONS_PATH, POST, userId)!! + TrackingHelper.trackSuccess( + CONNECTIONS_PATH, + POST, + userId, + destinationResponse.workspaceId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(connectionResponse) + .build() + } + + override fun deleteConnection(connectionId: UUID, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val connectionResponse: Any = TrackingHelper.callWithTracker( + { + connectionService.deleteConnection( + connectionId, + getLocalUserInfoIfNull(userInfo), + ) + }, + CONNECTIONS_WITH_ID_PATH, + DELETE, + userId, + )!! + TrackingHelper.trackSuccess( + CONNECTIONS_WITH_ID_PATH, + DELETE, + userId, + ) + return Response + .status(Response.Status.NO_CONTENT.statusCode) + .entity(connectionResponse) + .build() + } + + override fun getConnection(connectionId: UUID, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val connectionResponse: Any = TrackingHelper.callWithTracker({ + connectionService.getConnection( + connectionId, + getLocalUserInfoIfNull(userInfo), + ) + }, CONNECTIONS_PATH, GET, userId)!! + TrackingHelper.trackSuccess( + CONNECTIONS_WITH_ID_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(connectionResponse) + .build() + } + + override fun listConnections( + workspaceIds: MutableList?, + includeDeleted: Boolean?, + limit: Int?, + offset: Int?, + + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val safeWorkspaceIds = workspaceIds ?: emptyList() + val connections = TrackingHelper.callWithTracker({ + connectionService.listConnectionsForWorkspaces( + safeWorkspaceIds, + limit!!, + offset!!, + includeDeleted!!, + getLocalUserInfoIfNull(userInfo), + ) + }, CONNECTIONS_PATH, GET, userId)!! + TrackingHelper.trackSuccess( + CONNECTIONS_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(connections) + .build() + } + + @Patch + override fun patchConnection( + connectionId: UUID, + connectionPatchRequest: ConnectionPatchRequest, + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + val validUserInfo: String? = getLocalUserInfoIfNull(userInfo) + + // validate cron timing configurations + TrackingHelper.callWithTracker( + { + AirbyteCatalogHelper.validateCronConfiguration( + connectionPatchRequest.schedule, + ) + }, + CONNECTIONS_WITH_ID_PATH, + PUT, + userId, + ) + + val currentConnection: ConnectionResponse = + TrackingHelper.callWithTracker( + { connectionService.getConnection(connectionId, validUserInfo) }, + CONNECTIONS_WITH_ID_PATH, + PUT, + userId, + ) as ConnectionResponse + + // get destination response to retrieve workspace id as well as input for destination sync modes + val destinationResponse: DestinationResponse = + TrackingHelper.callWithTracker( + { destinationService.getDestination(currentConnection.destinationId, validUserInfo) }, + CONNECTIONS_WITH_ID_PATH, + PUT, + userId, + ) as DestinationResponse + + // get source schema for catalog id and airbyte catalog + val schemaResponse = TrackingHelper.callWithTracker( + { sourceService.getSourceSchema(currentConnection.sourceId, false, validUserInfo) }, + CONNECTIONS_PATH, + POST, + userId, + ) + val catalogId = schemaResponse.catalogId + + val airbyteCatalogFromDiscoverSchema = schemaResponse.catalog + + // refer to documentation to understand what we need to do for the catalog + // https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#catalog + var configuredCatalog: AirbyteCatalog? = AirbyteCatalog() + + val validStreams: Map = AirbyteCatalogHelper.getValidStreams( + Objects.requireNonNull(airbyteCatalogFromDiscoverSchema), + ) + + // check user configs + if (AirbyteCatalogHelper.hasStreamConfigurations(connectionPatchRequest.configurations)) { + // validate user inputs + TrackingHelper.callWithTracker( + { + AirbyteCatalogHelper.validateStreams( + airbyteCatalogFromDiscoverSchema!!, + connectionPatchRequest.configurations, + ) + }, + CONNECTIONS_PATH, + POST, + userId, + ) + + // set user inputs + for (streamConfiguration in connectionPatchRequest.configurations.streams) { + val validStreamAndConfig = validStreams[streamConfiguration.name] + val schemaStream = validStreamAndConfig!!.stream + val schemaConfig = validStreamAndConfig.config + + val validDestinationSyncModes = TrackingHelper.callWithTracker( + { destinationService.getDestinationSyncModes(destinationResponse, validUserInfo) }, + CONNECTIONS_PATH, + POST, + userId, + ) as List + + // set user configs + TrackingHelper.callWithTracker( + { + AirbyteCatalogHelper.setAndValidateStreamConfig( + streamConfiguration, + validDestinationSyncModes, + schemaStream!!, + schemaConfig!!, + ) + }, + CONNECTIONS_PATH, + POST, + userId, + ) + configuredCatalog!!.addStreamsItem(validStreamAndConfig) + } + } else { + // no user supplied stream configs, return all existing streams + configuredCatalog = null + } + + val finalConfiguredCatalog = configuredCatalog + val connectionResponse: Any = TrackingHelper.callWithTracker( + { + connectionService.updateConnection( + connectionId, + connectionPatchRequest, + catalogId!!, + finalConfiguredCatalog!!, + destinationResponse.workspaceId, + validUserInfo, + ) + }, + CONNECTIONS_PATH, + POST, + userId, + )!! + + TrackingHelper.trackSuccess( + CONNECTIONS_WITH_ID_PATH, + PUT, + userId, + destinationResponse.workspaceId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(connectionResponse) + .build() + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt new file mode 100644 index 00000000000..8d7dc183865 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.controllers + +import com.fasterxml.jackson.databind.node.ObjectNode +import io.airbyte.airbyte_api.generated.DestinationsApi +import io.airbyte.airbyte_api.model.generated.DestinationCreateRequest +import io.airbyte.airbyte_api.model.generated.DestinationPatchRequest +import io.airbyte.airbyte_api.model.generated.DestinationPutRequest +import io.airbyte.api.server.constants.DELETE +import io.airbyte.api.server.constants.DESTINATIONS_PATH +import io.airbyte.api.server.constants.DESTINATIONS_WITH_ID_PATH +import io.airbyte.api.server.constants.DESTINATION_TYPE +import io.airbyte.api.server.constants.GET +import io.airbyte.api.server.constants.PATCH +import io.airbyte.api.server.constants.POST +import io.airbyte.api.server.constants.PUT +import io.airbyte.api.server.helpers.TrackingHelper +import io.airbyte.api.server.helpers.getIdFromName +import io.airbyte.api.server.helpers.getLocalUserInfoIfNull +import io.airbyte.api.server.helpers.removeDestinationType +import io.airbyte.api.server.mappers.DESTINATION_NAME_TO_DEFINITION_ID +import io.airbyte.api.server.problems.UnprocessableEntityProblem +import io.airbyte.api.server.services.DestinationService +import io.airbyte.api.server.services.UserService +import io.micronaut.http.annotation.Controller +import io.micronaut.http.annotation.Patch +import java.util.UUID +import javax.ws.rs.core.Response + +@Controller(DESTINATIONS_PATH) +open class DestinationsController(private val destinationService: DestinationService, private val userService: UserService) : DestinationsApi { + override fun createDestination(destinationCreateRequest: DestinationCreateRequest, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val configurationJsonNode = destinationCreateRequest.configuration as ObjectNode + if (configurationJsonNode.findValue(DESTINATION_TYPE) == null) { + val unprocessableEntityProblem = UnprocessableEntityProblem() + TrackingHelper.trackFailuresIfAny( + DESTINATIONS_PATH, + POST, + userId, + unprocessableEntityProblem, + ) + throw unprocessableEntityProblem + } + val destinationName = + configurationJsonNode.findValue(DESTINATION_TYPE).toString().replace("\"", "") + val destinationDefinitionId: UUID = getIdFromName(DESTINATION_NAME_TO_DEFINITION_ID, destinationName) + + removeDestinationType(destinationCreateRequest) + + val destinationResponse: Any? = TrackingHelper.callWithTracker( + { + destinationService.createDestination( + destinationCreateRequest, + destinationDefinitionId, + getLocalUserInfoIfNull(userInfo), + ) + }, + DESTINATIONS_PATH, + POST, + userId, + ) + TrackingHelper.trackSuccess( + DESTINATIONS_PATH, + POST, + userId, + destinationCreateRequest.workspaceId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(destinationResponse) + .build() + } + + override fun deleteDestination(destinationId: UUID, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val destinationResponse: Any? = TrackingHelper.callWithTracker( + { + destinationService.deleteDestination( + destinationId, + getLocalUserInfoIfNull(userInfo), + ) + }, + DESTINATIONS_WITH_ID_PATH, + DELETE, + userId, + ) + TrackingHelper.trackSuccess( + DESTINATIONS_WITH_ID_PATH, + DELETE, + userId, + ) + return Response + .status(Response.Status.NO_CONTENT.statusCode) + .entity(destinationResponse) + .build() + } + + override fun getDestination(destinationId: UUID, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val destinationResponse: Any? = TrackingHelper.callWithTracker( + { + destinationService.getDestination( + destinationId, + getLocalUserInfoIfNull(userInfo), + ) + }, + DESTINATIONS_WITH_ID_PATH, + GET, + userId, + ) + TrackingHelper.trackSuccess( + DESTINATIONS_WITH_ID_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(destinationResponse) + .build() + } + + override fun listDestinations( + workspaceIds: MutableList?, + includeDeleted: Boolean?, + limit: Int?, + offset: Int?, + + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val safeWorkspaceIds = workspaceIds ?: emptyList() + val destinations: Any? = TrackingHelper.callWithTracker({ + destinationService.listDestinationsForWorkspaces( + safeWorkspaceIds, + includeDeleted!!, + limit!!, + offset!!, + getLocalUserInfoIfNull(userInfo), + ) + }, DESTINATIONS_PATH, GET, userId) + TrackingHelper.trackSuccess( + DESTINATIONS_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(destinations) + .build() + } + + @Patch + override fun patchDestination( + destinationId: UUID, + destinationPatchRequest: DestinationPatchRequest, + + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + removeDestinationType(destinationPatchRequest) + + val destinationResponse: Any = TrackingHelper.callWithTracker( + { + destinationService.partialUpdateDestination( + destinationId, + destinationPatchRequest, + getLocalUserInfoIfNull(userInfo), + ) + }, + DESTINATIONS_WITH_ID_PATH, + PATCH, + userId, + )!! + + TrackingHelper.trackSuccess( + DESTINATIONS_WITH_ID_PATH, + PATCH, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(destinationResponse) + .build() + } + + override fun putDestination( + destinationId: UUID, + destinationPutRequest: DestinationPutRequest, + + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + removeDestinationType(destinationPutRequest) + + val destinationResponse: Any? = TrackingHelper.callWithTracker( + { + destinationService.updateDestination( + destinationId, + destinationPutRequest, + getLocalUserInfoIfNull(userInfo), + ) + }, + DESTINATIONS_WITH_ID_PATH, + PUT, + userId, + ) + + TrackingHelper.trackSuccess( + DESTINATIONS_WITH_ID_PATH, + PUT, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(destinationResponse) + .build() + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Health.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/HealthController.kt similarity index 80% rename from airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Health.kt rename to airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/HealthController.kt index 089e2ba471a..05c8afee5ae 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Health.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/HealthController.kt @@ -1,4 +1,8 @@ -package io.airbyte.api.server.routes +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.controllers import io.micronaut.http.HttpResponse import io.micronaut.http.annotation.Controller @@ -10,11 +14,12 @@ import javax.ws.rs.GET * Health endpoint used by kubernetes and the gcp load balancer. */ @Controller("/health") -class Health { +class HealthController { @GET @ApiResponses( value = [ - ApiResponse(code = 200, message = "Successful operation"), ApiResponse( + ApiResponse(code = 200, message = "Successful operation"), + ApiResponse( code = 403, message = "The request is not authorized; see message for details.", ), diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt new file mode 100644 index 00000000000..2c1178421aa --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.controllers + +import io.airbyte.airbyte_api.generated.JobsApi +import io.airbyte.airbyte_api.model.generated.ConnectionResponse +import io.airbyte.airbyte_api.model.generated.JobCreateRequest +import io.airbyte.airbyte_api.model.generated.JobStatusEnum +import io.airbyte.airbyte_api.model.generated.JobTypeEnum +import io.airbyte.api.server.constants.DELETE +import io.airbyte.api.server.constants.GET +import io.airbyte.api.server.constants.JOBS_PATH +import io.airbyte.api.server.constants.JOBS_WITH_ID_PATH +import io.airbyte.api.server.constants.POST +import io.airbyte.api.server.filters.JobsFilter +import io.airbyte.api.server.helpers.TrackingHelper +import io.airbyte.api.server.helpers.getLocalUserInfoIfNull +import io.airbyte.api.server.problems.UnprocessableEntityProblem +import io.airbyte.api.server.services.ConnectionService +import io.airbyte.api.server.services.JobService +import io.airbyte.api.server.services.UserService +import io.micronaut.http.annotation.Controller +import java.time.OffsetDateTime +import java.util.UUID +import javax.ws.rs.core.Response + +@Controller(JOBS_PATH) +open class JobsController( + private val jobService: JobService, + private val userService: UserService, + private val connectionService: ConnectionService, +) : JobsApi { + override fun cancelJob(jobId: Long, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val jobResponse: Any? = TrackingHelper.callWithTracker( + { + jobService.cancelJob( + jobId, + getLocalUserInfoIfNull(userInfo), + ) + }, + JOBS_WITH_ID_PATH, + DELETE, + userId, + ) + + TrackingHelper.trackSuccess( + JOBS_WITH_ID_PATH, + DELETE, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(jobResponse) + .build() + } + + override fun createJob(jobCreateRequest: JobCreateRequest, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val connectionResponse: ConnectionResponse = + TrackingHelper.callWithTracker( + { + connectionService.getConnection( + jobCreateRequest.connectionId, + getLocalUserInfoIfNull(userInfo), + ) + }, + JOBS_PATH, + POST, + userId, + ) as ConnectionResponse + val workspaceId: UUID = connectionResponse.workspaceId + + return when (jobCreateRequest.jobType) { + JobTypeEnum.SYNC -> { + val jobResponse: Any = TrackingHelper.callWithTracker({ + jobService.sync( + jobCreateRequest.connectionId, + getLocalUserInfoIfNull(userInfo), + ) + }, JOBS_PATH, POST, userId)!! + TrackingHelper.trackSuccess( + JOBS_PATH, + POST, + userId, + workspaceId, + ) + Response + .status(Response.Status.OK.statusCode) + .entity(jobResponse) + .build() + } + + JobTypeEnum.RESET -> { + val jobResponse: Any = TrackingHelper.callWithTracker({ + jobService.reset( + jobCreateRequest.connectionId, + getLocalUserInfoIfNull(userInfo), + ) + }, JOBS_PATH, POST, userId)!! + TrackingHelper.trackSuccess( + JOBS_PATH, + POST, + userId, + workspaceId, + ) + Response + .status(Response.Status.OK.statusCode) + .entity(jobResponse) + .build() + } + + else -> { + val unprocessableEntityProblem = UnprocessableEntityProblem() + TrackingHelper.trackFailuresIfAny( + JOBS_PATH, + POST, + userId, + unprocessableEntityProblem, + ) + throw unprocessableEntityProblem + } + } + } + + override fun getJob(jobId: Long, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val jobResponse: Any? = TrackingHelper.callWithTracker( + { + jobService.getJobInfoWithoutLogs( + jobId, + getLocalUserInfoIfNull(userInfo), + ) + }, + JOBS_WITH_ID_PATH, + GET, + userId, + ) + + TrackingHelper.trackSuccess( + JOBS_WITH_ID_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(jobResponse) + .build() + } + + override fun listJobs( + connectionId: UUID?, + limit: Int?, + offset: Int?, + jobType: JobTypeEnum?, + workspaceIds: List?, + status: JobStatusEnum?, + createdAtStart: OffsetDateTime?, + createdAtEnd: OffsetDateTime?, + updatedAtStart: OffsetDateTime?, + updatedAtEnd: OffsetDateTime?, + + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + val jobsResponse: Any + val filter = JobsFilter( + createdAtStart, + createdAtEnd, + updatedAtStart, + updatedAtEnd, + limit, + offset, + jobType, + status, + ) + jobsResponse = ( + if (connectionId != null) { + TrackingHelper.callWithTracker( + { + jobService.getJobList( + connectionId, + filter, + getLocalUserInfoIfNull(userInfo), + ) + }, + JOBS_PATH, + GET, + userId, + ) + } else { + TrackingHelper.callWithTracker( + { + jobService.getJobList( + workspaceIds ?: emptyList(), + filter, + getLocalUserInfoIfNull(userInfo), + ) + }, + JOBS_PATH, + GET, + userId, + ) + } + )!! + + TrackingHelper.trackSuccess( + JOBS_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(jobsResponse) + .build() + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt new file mode 100644 index 00000000000..9ce3294b653 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.controllers + +import com.fasterxml.jackson.databind.node.ObjectNode +import io.airbyte.airbyte_api.generated.SourcesApi +import io.airbyte.airbyte_api.model.generated.InitiateOauthRequest +import io.airbyte.airbyte_api.model.generated.SourceCreateRequest +import io.airbyte.airbyte_api.model.generated.SourcePatchRequest +import io.airbyte.airbyte_api.model.generated.SourcePutRequest +import io.airbyte.api.server.constants.DELETE +import io.airbyte.api.server.constants.GET +import io.airbyte.api.server.constants.PATCH +import io.airbyte.api.server.constants.POST +import io.airbyte.api.server.constants.PUT +import io.airbyte.api.server.constants.SOURCES_PATH +import io.airbyte.api.server.constants.SOURCES_WITH_ID_PATH +import io.airbyte.api.server.constants.SOURCE_TYPE +import io.airbyte.api.server.helpers.TrackingHelper +import io.airbyte.api.server.helpers.getIdFromName +import io.airbyte.api.server.helpers.getLocalUserInfoIfNull +import io.airbyte.api.server.helpers.removeSourceTypeNode +import io.airbyte.api.server.mappers.SOURCE_NAME_TO_DEFINITION_ID +import io.airbyte.api.server.problems.UnprocessableEntityProblem +import io.airbyte.api.server.services.SourceService +import io.airbyte.api.server.services.UserService +import io.micronaut.http.annotation.Controller +import io.micronaut.http.annotation.Patch +import java.util.UUID +import javax.ws.rs.core.Response + +@Controller(SOURCES_PATH) +class SourcesController( + private val sourceService: SourceService, + private val userService: UserService, +) : SourcesApi { + override fun createSource(sourceCreateRequest: SourceCreateRequest?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val configurationJsonNode = sourceCreateRequest!!.configuration as ObjectNode + if (configurationJsonNode.findValue(SOURCE_TYPE) == null) { + throw UnprocessableEntityProblem() + } + val sourceName = configurationJsonNode.findValue(SOURCE_TYPE).toString().replace("\"", "") + val sourceDefinitionId: UUID = getIdFromName(SOURCE_NAME_TO_DEFINITION_ID, sourceName) + + removeSourceTypeNode(sourceCreateRequest) + + val sourceResponse: Any? = TrackingHelper.callWithTracker( + { + sourceService.createSource( + sourceCreateRequest, + sourceDefinitionId, + getLocalUserInfoIfNull(userInfo), + ) + }, + SOURCES_PATH, + POST, + userId, + ) + + TrackingHelper.trackSuccess( + SOURCES_PATH, + POST, + userId, + sourceCreateRequest.workspaceId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(sourceResponse) + .build() + } + + override fun deleteSource(sourceId: UUID?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val sourceResponse: Any? = TrackingHelper.callWithTracker( + { + sourceService.deleteSource( + sourceId!!, + getLocalUserInfoIfNull(userInfo), + ) + }, + SOURCES_WITH_ID_PATH, + DELETE, + userId, + ) + + TrackingHelper.trackSuccess( + SOURCES_WITH_ID_PATH, + DELETE, + userId, + ) + return Response + .status(Response.Status.NO_CONTENT.statusCode) + .entity(sourceResponse) + .build() + } + + override fun getSource(sourceId: UUID?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val sourceResponse: Any? = TrackingHelper.callWithTracker( + { + sourceService.getSource( + sourceId!!, + getLocalUserInfoIfNull(userInfo), + ) + }, + SOURCES_WITH_ID_PATH, + GET, + userId, + ) + + TrackingHelper.trackSuccess( + SOURCES_WITH_ID_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(sourceResponse) + .build() + } + + override fun initiateOAuth(initiateOauthRequest: InitiateOauthRequest?, userInfo: String?): Response { + return Response.status(Response.Status.NOT_IMPLEMENTED).build() + } + + override fun listSources( + workspaceIds: MutableList?, + includeDeleted: Boolean?, + limit: Int?, + offset: Int?, + + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val safeWorkspaceIds = workspaceIds ?: emptyList() + val sources: Any? = TrackingHelper.callWithTracker({ + sourceService.listSourcesForWorkspaces( + safeWorkspaceIds, + includeDeleted!!, + limit!!, + offset!!, + getLocalUserInfoIfNull(userInfo), + ) + }, SOURCES_PATH, GET, userId) + + TrackingHelper.trackSuccess( + SOURCES_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(sources) + .build() + } + + @Patch + override fun patchSource(sourceId: UUID?, sourcePatchRequest: SourcePatchRequest?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + removeSourceTypeNode(sourcePatchRequest!!) + + val sourceResponse: Any? = TrackingHelper.callWithTracker( + { + sourceService.partialUpdateSource( + sourceId!!, + sourcePatchRequest, + getLocalUserInfoIfNull(userInfo), + ) + }, + SOURCES_WITH_ID_PATH, + PATCH, + userId, + ) + + TrackingHelper.trackSuccess( + SOURCES_WITH_ID_PATH, + PATCH, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(sourceResponse) + .build() + } + + override fun putSource(sourceId: UUID?, sourcePutRequest: SourcePutRequest?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + removeSourceTypeNode(sourcePutRequest!!) + + val sourceResponse: Any? = TrackingHelper.callWithTracker( + { + sourceService.updateSource( + sourceId!!, + sourcePutRequest, + getLocalUserInfoIfNull(userInfo), + ) + }, + SOURCES_WITH_ID_PATH, + PUT, + userId, + ) + + TrackingHelper.trackSuccess( + SOURCES_WITH_ID_PATH, + PUT, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(sourceResponse) + .build() + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt new file mode 100644 index 00000000000..681b2ca5ef1 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.controllers + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import io.airbyte.airbyte_api.generated.StreamsApi +import io.airbyte.airbyte_api.model.generated.ConnectionSyncModeEnum +import io.airbyte.airbyte_api.model.generated.StreamProperties +import io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration +import io.airbyte.api.client.model.generated.DestinationSyncMode +import io.airbyte.api.client.model.generated.SyncMode +import io.airbyte.api.server.constants.GET +import io.airbyte.api.server.constants.STREAMS_PATH +import io.airbyte.api.server.helpers.TrackingHelper +import io.airbyte.api.server.helpers.getLocalUserInfoIfNull +import io.airbyte.api.server.problems.UnexpectedProblem +import io.airbyte.api.server.services.DestinationService +import io.airbyte.api.server.services.SourceService +import io.airbyte.api.server.services.UserService +import io.micronaut.http.HttpStatus +import io.micronaut.http.annotation.Controller +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.UUID +import javax.ws.rs.core.Response + +@Controller(STREAMS_PATH) +class StreamsController( + private val userService: UserService, + private val sourceService: SourceService, + private val destinationService: DestinationService, +) : StreamsApi { + + companion object { + private val log: org.slf4j.Logger? = LoggerFactory.getLogger(StreamsController::class.java) + } + + override fun getStreamProperties( + sourceId: UUID, + destinationId: UUID?, + ignoreCache: Boolean?, + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + val httpResponse = TrackingHelper.callWithTracker( + { + sourceService.getSourceSchema( + sourceId, + ignoreCache!!, + getLocalUserInfoIfNull(userInfo), + ) + }, + STREAMS_PATH, + GET, + userId, + ) + val destinationSyncModes = TrackingHelper.callWithTracker( + { + destinationService.getDestinationSyncModes( + destinationId!!, + getLocalUserInfoIfNull(userInfo), + ) + }, + STREAMS_PATH, + GET, + userId, + ) + val streamList = httpResponse.catalog!!.streams.stream() + .map { obj: AirbyteStreamAndConfiguration -> obj.stream } + .toList() + val listOfStreamProperties: MutableList = emptyList().toMutableList() + for (airbyteStream in streamList) { + val streamProperties = StreamProperties() + val sourceSyncModes = airbyteStream!!.supportedSyncModes!! + streamProperties.streamName = airbyteStream.name + streamProperties.syncModes = getValidSyncModes(sourceSyncModes, destinationSyncModes) + if (airbyteStream.defaultCursorField != null) { + streamProperties.defaultCursorField = airbyteStream.defaultCursorField + } + if (airbyteStream.sourceDefinedPrimaryKey != null) { + streamProperties.sourceDefinedPrimaryKey = airbyteStream.sourceDefinedPrimaryKey + } + streamProperties.sourceDefinedCursorField = airbyteStream.sourceDefinedCursor != null && airbyteStream.sourceDefinedCursor!! + streamProperties.propertyFields = getStreamFields(airbyteStream.jsonSchema) + listOfStreamProperties.add(streamProperties) + } + TrackingHelper.trackSuccess( + STREAMS_PATH, + GET, + userId, + ) + return Response + .status( + HttpStatus.OK.code, + ) + .entity(listOfStreamProperties) + .build() + } + + /** + * Parses a connectorSchema to retrieve all the possible stream fields. + * + * @param connectorSchema source or destination schema + * @return A list of stream fields, which are represented as list of strings since they can be + * nested fields. + */ + private fun getStreamFields(connectorSchema: JsonNode?): List> { + val yamlMapper = ObjectMapper(YAMLFactory()) + val streamFields: MutableList> = ArrayList() + val spec: JsonNode = try { + yamlMapper.readTree(connectorSchema!!.traverse()) + } catch (e: IOException) { + log?.error("Error getting stream fields from schema", e) + throw UnexpectedProblem(HttpStatus.INTERNAL_SERVER_ERROR) + } + val fields = spec.fields() + while (fields.hasNext()) { + val (key, paths) = fields.next() + if ("properties" == key) { + val propertyFields = paths.fields() + while (propertyFields.hasNext()) { + val (propertyName, nestedProperties) = propertyFields.next() + streamFields.add(listOf(propertyName)) + + // retrieve nested paths + for (entry in getStreamFields(nestedProperties)) { + if (entry.isEmpty()) { + continue + } + val streamFieldPath: MutableList = ArrayList(mutableListOf(propertyName)) + streamFieldPath.addAll(entry) + streamFields.add(streamFieldPath) + } + } + } + } + return streamFields + } + + private fun getValidSyncModes( + sourceSyncModes: List, + destinationSyncModes: List, + ): List { + val connectionSyncModes: MutableList = emptyList().toMutableList() + if (sourceSyncModes.contains(SyncMode.FULL_REFRESH)) { + if (destinationSyncModes.contains(DestinationSyncMode.APPEND)) { + connectionSyncModes.add(ConnectionSyncModeEnum.FULL_REFRESH_APPEND) + } + if (destinationSyncModes.contains(DestinationSyncMode.OVERWRITE)) { + connectionSyncModes.add(ConnectionSyncModeEnum.FULL_REFRESH_OVERWRITE) + } + } + if (sourceSyncModes.contains(SyncMode.INCREMENTAL)) { + if (destinationSyncModes.contains(DestinationSyncMode.APPEND)) { + connectionSyncModes.add(ConnectionSyncModeEnum.INCREMENTAL_APPEND) + } + if (destinationSyncModes.contains(DestinationSyncMode.APPEND_DEDUP)) { + connectionSyncModes.add(ConnectionSyncModeEnum.INCREMENTAL_DEDUPED_HISTORY) + } + } + return connectionSyncModes + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt new file mode 100644 index 00000000000..867766aeedd --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.controllers + +import io.airbyte.airbyte_api.generated.WorkspacesApi +import io.airbyte.airbyte_api.model.generated.WorkspaceCreateRequest +import io.airbyte.airbyte_api.model.generated.WorkspaceOAuthCredentialsRequest +import io.airbyte.airbyte_api.model.generated.WorkspaceResponse +import io.airbyte.airbyte_api.model.generated.WorkspaceUpdateRequest +import io.airbyte.api.server.constants.DELETE +import io.airbyte.api.server.constants.GET +import io.airbyte.api.server.constants.POST +import io.airbyte.api.server.constants.WORKSPACES_PATH +import io.airbyte.api.server.constants.WORKSPACES_WITH_ID_PATH +import io.airbyte.api.server.helpers.TrackingHelper +import io.airbyte.api.server.helpers.getLocalUserInfoIfNull +import io.airbyte.api.server.services.UserService +import io.airbyte.api.server.services.WorkspaceService +import io.micronaut.http.annotation.Controller +import java.util.UUID +import javax.ws.rs.core.Response + +@Controller(WORKSPACES_PATH) +class WorkspacesController( + private val workspaceService: WorkspaceService, + private val userService: UserService, +) : WorkspacesApi { + override fun createOrUpdateWorkspaceOAuthCredentials( + workspaceId: UUID?, + workspaceOAuthCredentialsRequest: WorkspaceOAuthCredentialsRequest?, + + userInfo: String?, + ): Response { + return Response.status(Response.Status.NOT_IMPLEMENTED).build() + } + + override fun createWorkspace(workspaceCreateRequest: WorkspaceCreateRequest?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val workspaceResponse: WorkspaceResponse = + TrackingHelper.callWithTracker( + { workspaceService.createWorkspace(workspaceCreateRequest!!, userInfo) }, + WORKSPACES_PATH, + POST, + userId, + ) as WorkspaceResponse + TrackingHelper.trackSuccess( + WORKSPACES_PATH, + POST, + userId, + workspaceResponse.workspaceId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(workspaceResponse) + .build() + } + + override fun deleteWorkspace(workspaceId: UUID?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val workspaceResponse: Any? = TrackingHelper.callWithTracker( + { + workspaceService.deleteWorkspace( + workspaceId!!, + getLocalUserInfoIfNull(userInfo), + ) + }, + WORKSPACES_WITH_ID_PATH, + DELETE, + userId, + ) + return Response + .status(Response.Status.NO_CONTENT.statusCode) + .entity(workspaceResponse) + .build() + } + + override fun getWorkspace(workspaceId: UUID?, userInfo: String?): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val workspaceResponse: Any? = TrackingHelper.callWithTracker( + { + workspaceService.getWorkspace( + workspaceId!!, + getLocalUserInfoIfNull(userInfo), + ) + }, + WORKSPACES_WITH_ID_PATH, + GET, + userId, + ) + TrackingHelper.trackSuccess( + WORKSPACES_WITH_ID_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(workspaceResponse) + .build() + } + + override fun listWorkspaces( + workspaceIds: MutableList?, + includeDeleted: Boolean?, + limit: Int?, + offset: Int?, + + userInfo: String?, + ): Response { + val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) + + val safeWorkspaceIds = workspaceIds ?: emptyList() + + val workspaces: Any? = TrackingHelper.callWithTracker( + { + workspaceService.listWorkspaces( + safeWorkspaceIds, + includeDeleted!!, + limit!!, + offset!!, + getLocalUserInfoIfNull(userInfo), + ) + }, + WORKSPACES_PATH, + GET, + userId, + ) + TrackingHelper.trackSuccess( + WORKSPACES_PATH, + GET, + userId, + ) + return Response + .status(Response.Status.OK.statusCode) + .entity(workspaces) + .build() + } + + override fun updateWorkspace( + workspaceId: UUID?, + workspaceUpdateRequest: WorkspaceUpdateRequest?, + + userInfo: String?, + ): Response { + return Response.status(Response.Status.NOT_IMPLEMENTED).build() + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/errorHandlers/AirbyteConversionErrorHandler.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/errorHandlers/AirbyteConversionErrorHandler.kt new file mode 100644 index 00000000000..e2bb7158a59 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/errorHandlers/AirbyteConversionErrorHandler.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.errorHandlers + +import com.fasterxml.jackson.databind.exc.InvalidFormatException +import com.fasterxml.jackson.databind.exc.ValueInstantiationException +import io.airbyte.api.server.problems.BadRequestProblem +import io.micronaut.context.annotation.Replaces +import io.micronaut.core.convert.exceptions.ConversionErrorException +import io.micronaut.http.HttpRequest +import io.micronaut.http.HttpResponse +import io.micronaut.http.server.exceptions.ConversionErrorHandler +import io.micronaut.http.server.exceptions.ExceptionHandler +import io.micronaut.http.server.exceptions.response.Error +import io.micronaut.http.server.exceptions.response.ErrorContext +import io.micronaut.http.server.exceptions.response.ErrorResponseProcessor +import jakarta.inject.Inject +import java.time.format.DateTimeParseException +import java.util.Optional + +/** + * Replaces the ConversionErrorHandler bean that micronaut ships with and allows us to more + * gracefully handle errors. Specifically deserialization errors cause us to end up here. The only + * difference between our conversion error handler and the default one is that we return a bad + * request problem whereas the default returns a generic problem that isn't suitable to show our + * users. Due to how and where the ConversionErrorException gets thrown in micronaut code, we're not + * really able to get more information about the deserialization error here. + */ +@Replaces(ConversionErrorHandler::class) +class AirbyteConversionErrorHandler + @Inject + constructor(responseProcessor: ErrorResponseProcessor<*>) : + ExceptionHandler> { + private val responseProcessor: ErrorResponseProcessor<*> + + init { + this.responseProcessor = responseProcessor + } + + override fun handle(request: HttpRequest<*>?, exception: ConversionErrorException): HttpResponse<*> { + var message: String + if (exception.cause is ValueInstantiationException) { + val exceptionCast = exception.cause as ValueInstantiationException + // Handles invalid enum values + val field: String = exceptionCast.path[0].fieldName + val originalMessage = + if (exceptionCast.cause == null) "Incorrectly formatted request body - Incorrectly formatted enum value" else exceptionCast.cause!!.message + message = String.format(originalMessage!!, field) + } else if (exception.cause is InvalidFormatException) { + val exceptionCast = exception.cause as InvalidFormatException + // Handles invalid format for things like UUID (not enough characters, etc) + val field: String = exceptionCast.path[0].fieldName + val value = exceptionCast.value as String + val type: String = exceptionCast.targetType.simpleName + message = String.format("Invalid value for field '%s': '%s' is not a valid %s type", field, value, type) + } else if (exception.cause is IllegalArgumentException) { + val exceptionCast = exception.cause as IllegalArgumentException + message = exceptionCast.message!! + val noEnumConstant = "No enum constant" + + // Hack for making a nice error message when enum validation fails. + // DefaultConversionService for String -> Enum calls Enum.valueOf which is different than how + // conversion works via micronaut-jackson-databind + // for POST bodies (the specific Enum Type's fromValue function). + // Unedited, this error looks something like this: "No enum constant + // io.airbyte.public_api_server.models.JobStatusEnum.test" + // Due to limiations around the data we have at this point, we can't tell the user which param they + // passed invalid data for. + if (message.contains(noEnumConstant)) { + message = message.replace("\\w+\\.".toRegex(), "").replace(noEnumConstant, "Invalid enum value: ") + } + } else if (exception.cause is DateTimeParseException) { + val exceptionCast = exception.cause as DateTimeParseException + message = String.format("Invalid datetime value passed: %s", exceptionCast.parsedString) + } else { + // If we've made it here, this is an error type we don't yet know how to handle. + message = "Incorrectly formatted request body" + } + return responseProcessor.processResponse( + ErrorContext.builder(request!!) + .cause(BadRequestProblem(message)) + .error(object : Error { + override fun getPath(): Optional { + return Optional.of('/'.toString() + exception.argument.name) + } + + override fun getMessage(): String { + return exception.message!! + } + }) + .build(), + HttpResponse.badRequest(), + ) + } + } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/errorHandlers/ConfigClientErrorHandler.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/errorHandlers/ConfigClientErrorHandler.kt new file mode 100644 index 00000000000..072a06cf8c8 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/errorHandlers/ConfigClientErrorHandler.kt @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +@file:Suppress("PackageName") + +package io.airbyte.api.server.errorHandlers + +import io.airbyte.airbyte_api.model.generated.ConnectionCreateRequest +import io.airbyte.api.server.constants.MESSAGE +import io.airbyte.api.server.problems.InvalidApiKeyProblem +import io.airbyte.api.server.problems.ResourceNotFoundProblem +import io.airbyte.api.server.problems.SyncConflictProblem +import io.airbyte.api.server.problems.UnexpectedProblem +import io.airbyte.api.server.problems.UnprocessableEntityProblem +import io.micronaut.http.HttpResponse +import io.micronaut.http.HttpStatus + +/** + * Maps config API client response statuses to problems. + */ +object ConfigClientErrorHandler { + /** + * Maps config API client response statuses to problems. + * + * @param response response from ConfigApiClient + * @param resourceId resource ID passed in with the request + */ + fun handleError(response: HttpResponse<*>, resourceId: String?) { + when (response.status) { + HttpStatus.NOT_FOUND -> throw ResourceNotFoundProblem(resourceId) + HttpStatus.CONFLICT -> { + val couldNotFulfillRequest = "Could not fulfill request" + val message: String = response.getBody(MutableMap::class.java) + .orElseGet { mutableMapOf(Pair(MESSAGE, couldNotFulfillRequest)) } + .getOrDefault(MESSAGE, couldNotFulfillRequest).toString() + throw SyncConflictProblem(message) + } + + HttpStatus.UNAUTHORIZED -> throw InvalidApiKeyProblem() + HttpStatus.UNPROCESSABLE_ENTITY -> { + val defaultErrorMessage = "The body of the request was not understood" + val message: String = response.getBody(MutableMap::class.java) + .orElseGet { mutableMapOf(Pair(MESSAGE, defaultErrorMessage)) } + .getOrDefault(MESSAGE, defaultErrorMessage).toString() + // Exclude the part of a schema validation message that's ugly if it's there + throw UnprocessableEntityProblem(message.split("\nSchema".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()[0]) + } + + else -> passThroughBadStatusCode(response) + } + } + + /** + * Maps config API client response statuses to problems during connection creation. + * + * @param response response from ConfigApiClient + * @param connectionCreate connection create inputs passed in with the request + */ + fun handleCreateConnectionError( + response: HttpResponse<*>, + connectionCreate: ConnectionCreateRequest, + ) { + when (response.status) { + HttpStatus.NOT_FOUND -> { + if (response.body.toString().contains(connectionCreate.getSourceId().toString())) { + throw ResourceNotFoundProblem(connectionCreate.getSourceId().toString()) + } else if (response.body.toString().contains(connectionCreate.getDestinationId().toString())) { + throw ResourceNotFoundProblem(connectionCreate.getDestinationId().toString()) + } + throw UnprocessableEntityProblem() + } + + HttpStatus.UNAUTHORIZED -> throw InvalidApiKeyProblem() + HttpStatus.UNPROCESSABLE_ENTITY -> throw UnprocessableEntityProblem() + else -> passThroughBadStatusCode(response) + } + } + + /** + * Throws an UnexpectedProblem if the response contains an error code 400 or above. + * + * @param response HttpResponse, most likely from the config api + */ + private fun passThroughBadStatusCode(response: HttpResponse<*>) { + if (response.status.code >= HttpStatus.BAD_REQUEST.code) { + throw UnexpectedProblem(response.status) + } + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/filters/BaseFilter.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/filters/BaseFilter.kt new file mode 100644 index 00000000000..b660e6153ff --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/filters/BaseFilter.kt @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.filters + +/** + * Base filter class for listing resources. + */ +open class BaseFilter( + val createdAtStart: java.time.OffsetDateTime?, + val createdAtEnd: java.time.OffsetDateTime?, + val updatedAtStart: java.time.OffsetDateTime?, + val updatedAtEnd: java.time.OffsetDateTime?, + val limit: Int? = 20, + val offset: Int? = 0, +) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/filters/JobsFilter.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/filters/JobsFilter.kt new file mode 100644 index 00000000000..3d94a8f4801 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/filters/JobsFilter.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.filters + +import io.airbyte.airbyte_api.model.generated.JobStatusEnum +import io.airbyte.airbyte_api.model.generated.JobTypeEnum +import io.airbyte.api.client.model.generated.JobStatus +import io.micronaut.core.annotation.Nullable +import java.time.OffsetDateTime + +/** + * Filters for jobs. Does some conversion. + */ +class JobsFilter( + createdAtStart: OffsetDateTime?, + createdAtEnd: OffsetDateTime?, + updatedAtStart: OffsetDateTime?, + updatedAtEnd: OffsetDateTime?, + limit: Int? = 20, + offset: Int? = 0, + jobType: JobTypeEnum?, + status: JobStatusEnum?, +) : + + BaseFilter(createdAtStart, createdAtEnd, updatedAtStart, updatedAtEnd, limit, offset) { + + val jobType: JobTypeEnum? + private val status: JobStatusEnum? + + init { + this.jobType = jobType + this.status = status + } + + /** + * Convert Airbyte API job status to config API job status. + */ + @Nullable + fun getConfigApiStatus(): JobStatus? { + return if (status == null) { + null + } else JobStatus.fromValue(status.toString()) + } +// @Nullable fun getJobType(): JobTypeEnum { +// return jobType +// } + } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/forwardingClient/ConfigApiClient.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/forwardingClient/ConfigApiClient.kt new file mode 100644 index 00000000000..a83f979144a --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/forwardingClient/ConfigApiClient.kt @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.forwardingClient + +import io.airbyte.api.client.model.generated.CompleteOAuthResponse +import io.airbyte.api.client.model.generated.CompleteSourceOauthRequest +import io.airbyte.api.client.model.generated.ConnectionCreate +import io.airbyte.api.client.model.generated.ConnectionIdRequestBody +import io.airbyte.api.client.model.generated.ConnectionRead +import io.airbyte.api.client.model.generated.ConnectionReadList +import io.airbyte.api.client.model.generated.ConnectionUpdate +import io.airbyte.api.client.model.generated.DestinationCreate +import io.airbyte.api.client.model.generated.DestinationDefinitionIdWithWorkspaceId +import io.airbyte.api.client.model.generated.DestinationDefinitionSpecificationRead +import io.airbyte.api.client.model.generated.DestinationIdRequestBody +import io.airbyte.api.client.model.generated.DestinationRead +import io.airbyte.api.client.model.generated.DestinationReadList +import io.airbyte.api.client.model.generated.DestinationUpdate +import io.airbyte.api.client.model.generated.JobIdRequestBody +import io.airbyte.api.client.model.generated.JobInfoRead +import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody +import io.airbyte.api.client.model.generated.JobListRequestBody +import io.airbyte.api.client.model.generated.JobReadList +import io.airbyte.api.client.model.generated.ListConnectionsForWorkspacesRequestBody +import io.airbyte.api.client.model.generated.ListResourcesForWorkspacesRequestBody +import io.airbyte.api.client.model.generated.OAuthConsentRead +import io.airbyte.api.client.model.generated.PartialDestinationUpdate +import io.airbyte.api.client.model.generated.PartialSourceUpdate +import io.airbyte.api.client.model.generated.SourceCreate +import io.airbyte.api.client.model.generated.SourceDefinitionIdWithWorkspaceId +import io.airbyte.api.client.model.generated.SourceDefinitionSpecificationRead +import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRead +import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody +import io.airbyte.api.client.model.generated.SourceIdRequestBody +import io.airbyte.api.client.model.generated.SourceOauthConsentRequest +import io.airbyte.api.client.model.generated.SourceRead +import io.airbyte.api.client.model.generated.SourceReadList +import io.airbyte.api.client.model.generated.SourceUpdate +import io.airbyte.api.client.model.generated.WorkspaceCreate +import io.airbyte.api.client.model.generated.WorkspaceIdRequestBody +import io.airbyte.api.client.model.generated.WorkspaceRead +import io.airbyte.api.client.model.generated.WorkspaceReadList +import io.airbyte.api.client.model.generated.WorkspaceUpdate +import io.airbyte.api.server.constants.ANALYTICS_HEADER +import io.airbyte.api.server.constants.ANALYTICS_HEADER_VALUE +import io.airbyte.api.server.constants.ENDPOINT_API_USER_INFO_HEADER +import io.airbyte.api.server.constants.INTERNAL_API_HOST +import io.micronaut.http.HttpHeaders +import io.micronaut.http.HttpResponse +import io.micronaut.http.MediaType +import io.micronaut.http.annotation.Body +import io.micronaut.http.annotation.Header +import io.micronaut.http.annotation.Post +import io.micronaut.http.client.annotation.Client + +/** + * The ConfigApiClient is a micronaut http client used to hit the internal airbyte config server. + * + * The X-Endpoint-API-UserInfo which populates endpointUserInfo is not used in OSS. + * In OSS the value can always be assumed to be null. + * + * Worth noting that status codes > 400 will throw an HttpClientResponseException EXCEPT 404s which + * will just return an HttpResponse with statusCode 404 + * https://docs.micronaut.io/latest/guide/index.html#clientError + */ +@Client(INTERNAL_API_HOST) +@Header(name = HttpHeaders.USER_AGENT, value = "Micronaut HTTP Client") +@Header(name = HttpHeaders.ACCEPT, value = MediaType.APPLICATION_JSON) +@Header(name = ANALYTICS_HEADER, value = ANALYTICS_HEADER_VALUE) +interface ConfigApiClient { + // Connections + @Post(value = "/api/v1/connections/create", processes = [MediaType.APPLICATION_JSON]) + fun createConnection( + @Body connectionId: ConnectionCreate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/connections/delete", processes = [MediaType.APPLICATION_JSON]) + fun deleteConnection( + @Body connectionIdRequestBody: ConnectionIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/connections/sync", processes = [MediaType.APPLICATION_JSON]) + fun sync( + @Body connectionId: ConnectionIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/connections/reset", processes = [MediaType.APPLICATION_JSON]) + fun reset( + @Body connectionId: ConnectionIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/connections/get", processes = [MediaType.APPLICATION_JSON]) + fun getConnection( + @Body connectionIdRequestBody: ConnectionIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/connections/update", processes = [MediaType.APPLICATION_JSON]) + fun updateConnection( + @Body connectionUpdate: ConnectionUpdate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + // OAuth + @Post(value = "/api/v1/source_oauths/get_consent_url", processes = [MediaType.APPLICATION_JSON]) + fun getSourceConsentUrl( + @Body consentRequest: SourceOauthConsentRequest, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/source_oauths/complete_oauth", processes = [MediaType.APPLICATION_JSON]) + fun completeSourceOAuth( + @Body completeSourceOauthRequest: CompleteSourceOauthRequest, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + // Sources + @Post(value = "/api/v1/sources/create", processes = [MediaType.APPLICATION_JSON]) + fun createSource( + @Body sourceCreate: SourceCreate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/sources/delete", processes = [MediaType.APPLICATION_JSON]) + fun deleteSource( + @Body sourceIdRequestBody: SourceIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/sources/get", processes = [MediaType.APPLICATION_JSON]) + fun getSource( + @Body sourceIdRequestBody: SourceIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/sources/partial_update", processes = [MediaType.APPLICATION_JSON]) + fun partialUpdateSource( + @Body partialSourceUpdate: PartialSourceUpdate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/sources/update", processes = [MediaType.APPLICATION_JSON]) + fun updateSource( + @Body sourceUpdate: SourceUpdate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/sources/discover_schema", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) + fun getSourceSchema( + @Body sourceId: SourceDiscoverSchemaRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post( + value = "/api/v1/source_definition_specifications/get", + processes = [MediaType.APPLICATION_JSON], + produces = [MediaType.APPLICATION_JSON], + ) + fun getSourceDefinitionSpecification( + @Body sourceDefinitionIdWithWorkspaceId: SourceDefinitionIdWithWorkspaceId, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + // Destinations + @Post(value = "/api/v1/destinations/create", processes = [MediaType.APPLICATION_JSON]) + fun createDestination( + @Body destinationCreate: DestinationCreate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/destinations/get", processes = [MediaType.APPLICATION_JSON]) + fun getDestination( + @Body destinationIdRequestBody: DestinationIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/destinations/update", processes = [MediaType.APPLICATION_JSON]) + fun updateDestination( + @Body destinationUpdate: DestinationUpdate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/destinations/partial_update", processes = [MediaType.APPLICATION_JSON]) + fun partialUpdateDestination( + @Body partialDestinationUpdate: PartialDestinationUpdate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/destinations/delete", processes = [MediaType.APPLICATION_JSON]) + fun deleteDestination( + @Body destinationIdRequestBody: DestinationIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/destination_definition_specifications/get", processes = [MediaType.APPLICATION_JSON]) + fun getDestinationSpec( + @Body destinationDefinitionIdWithWorkspaceId: DestinationDefinitionIdWithWorkspaceId, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + // Jobs + @Post(value = "/api/v1/jobs/get_without_logs", processes = [MediaType.APPLICATION_JSON]) + fun getJobInfoWithoutLogs( + @Body jobId: JobIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/jobs/list", processes = [MediaType.APPLICATION_JSON]) + fun getJobList( + @Body jobListRequestBody: JobListRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/jobs/list_for_workspaces", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) + fun getJobListForWorkspaces( + requestBody: JobListForWorkspacesRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/jobs/cancel", processes = [MediaType.APPLICATION_JSON]) + fun cancelJob( + @Body jobIdRequestBody: JobIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + // Workspaces + @Post(value = "/api/v1/workspaces/get", processes = [MediaType.APPLICATION_JSON]) + fun getWorkspace( + @Body workspaceIdRequestBody: WorkspaceIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/workspaces/list", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) + fun listAllWorkspaces( + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/workspaces/create", processes = [MediaType.APPLICATION_JSON]) + fun createWorkspace( + @Body workspaceCreate: WorkspaceCreate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/workspaces/update", processes = [MediaType.APPLICATION_JSON]) + fun updateWorkspace( + @Body workspaceUpdate: WorkspaceUpdate, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/workspaces/delete", processes = [MediaType.APPLICATION_JSON]) + fun deleteWorkspace( + @Body workspaceIdRequestBody: WorkspaceIdRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/connections/list_paginated", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) + fun listConnectionsForWorkspaces( + @Body listConnectionsForWorkspacesRequestBody: ListConnectionsForWorkspacesRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/sources/list_paginated", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) + fun listSourcesForWorkspaces( + @Body listResourcesForWorkspacesRequestBody: ListResourcesForWorkspacesRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/destinations/list_paginated", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) + fun listDestinationsForWorkspaces( + @Body listResourcesForWorkspacesRequestBody: ListResourcesForWorkspacesRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse + + @Post(value = "/api/v1/workspaces/list_paginated", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) + fun listWorkspaces( + @Body listResourcesForWorkspacesRequestBody: ListResourcesForWorkspacesRequestBody, + @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, + ): HttpResponse +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ActorConfigurationHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ActorConfigurationHelper.kt new file mode 100644 index 00000000000..3e40c98dff2 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ActorConfigurationHelper.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.helpers + +import com.fasterxml.jackson.databind.node.ObjectNode +import io.airbyte.api.common.ConfigurableActor +import io.airbyte.api.server.constants.DESTINATION_TYPE +import io.airbyte.api.server.constants.SOURCE_TYPE + +/** + * Removes the sourceType node from the actor's configuration. + * + * @param actor any actor model marked as a configurable actor via the x-implements extension in the + * api.yaml. + */ +fun removeSourceTypeNode(actor: ConfigurableActor) { + removeConfigurationNode(actor, SOURCE_TYPE) +} + +fun removeConfigurationNode(actor: ConfigurableActor, node: String) { + val configuration = actor.configuration as ObjectNode + configuration.remove(node) +} + +/** + * Removes the destinationType node from the actor's configuration. + * + * @param actor any actor model marked as a configurable actor via the x-implements extension in the + * api.yaml. + */ +fun removeDestinationType(actor: ConfigurableActor) { + removeConfigurationNode(actor, DESTINATION_TYPE) +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt new file mode 100644 index 00000000000..3be617477ba --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt @@ -0,0 +1,337 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.helpers + +import com.cronutils.model.Cron +import com.cronutils.model.CronType.QUARTZ +import com.cronutils.model.definition.CronDefinition +import com.cronutils.model.definition.CronDefinitionBuilder +import com.cronutils.parser.CronParser +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import io.airbyte.airbyte_api.model.generated.ConnectionSchedule +import io.airbyte.airbyte_api.model.generated.ConnectionSyncModeEnum +import io.airbyte.airbyte_api.model.generated.ScheduleTypeEnum +import io.airbyte.airbyte_api.model.generated.StreamConfiguration +import io.airbyte.airbyte_api.model.generated.StreamConfigurations +import io.airbyte.api.client.model.generated.AirbyteCatalog +import io.airbyte.api.client.model.generated.AirbyteStream +import io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration +import io.airbyte.api.client.model.generated.AirbyteStreamConfiguration +import io.airbyte.api.client.model.generated.DestinationSyncMode +import io.airbyte.api.client.model.generated.SyncMode +import io.airbyte.api.server.mappers.ConnectionReadMapper +import io.airbyte.api.server.problems.ConnectionConfigurationProblem +import io.airbyte.api.server.problems.ConnectionConfigurationProblem.Companion.duplicateStream +import io.airbyte.api.server.problems.ConnectionConfigurationProblem.Companion.invalidStreamName +import io.airbyte.api.server.problems.UnexpectedProblem +import io.micronaut.http.HttpStatus +import org.slf4j.LoggerFactory +import java.io.IOException + +/** + * Does everything necessary to both build and validate the AirbyteCatalog. + */ +object AirbyteCatalogHelper { + private val cronDefinition: CronDefinition = CronDefinitionBuilder.instanceDefinitionFor(QUARTZ) + private val parser: CronParser = CronParser(cronDefinition) + private val log = LoggerFactory.getLogger(AirbyteCatalogHelper.javaClass) + private const val MAX_LENGTH_OF_CRON = 7 + + /** + * Check whether stream configurations exist. + * + * @param streamConfigurations StreamConfigurations from conneciton create/update request + * @return true if they exist, false if they don't + */ + fun hasStreamConfigurations(streamConfigurations: StreamConfigurations?): Boolean { + return !streamConfigurations!!.streams.isNullOrEmpty() + } + + /** + * Just set a config to be full refresh overwrite. + * + * @param config config to be set + */ + fun setConfigDefaultFullRefreshOverwrite(config: AirbyteStreamConfiguration?) { + config!!.syncMode = SyncMode.FULL_REFRESH + config.destinationSyncMode = DestinationSyncMode.OVERWRITE + } + + /** + * Given an airbyte catalog, set all streams to be full refresh overwrite. + * + * @param airbyteCatalog The catalog to be modified + */ + fun setAllStreamsFullRefreshOverwrite(airbyteCatalog: AirbyteCatalog) { + for (schemaStreams in airbyteCatalog.streams) { + val config = schemaStreams.config!! + setConfigDefaultFullRefreshOverwrite(config) + } + } + + /** + * Given a reference catalog and a user's passed in streamConfigurations, ensure valid streams or + * throw a problem to be returned to the user. + * + * @param referenceCatalog - catalog, usually from discoverSourceSchema + * @param streamConfigurations - configurations passed in by the user. + * @return boolean so we can callWithTracker + */ + fun validateStreams(referenceCatalog: AirbyteCatalog, streamConfigurations: StreamConfigurations): Boolean { + val validStreams = getValidStreams(referenceCatalog) + val alreadyConfiguredStreams: MutableSet = HashSet() + for (streamConfiguration in streamConfigurations.streams) { + if (!validStreams.containsKey(streamConfiguration.name)) { + throw invalidStreamName(validStreams.keys) + } else if (alreadyConfiguredStreams.contains(streamConfiguration.name)) { + throw duplicateStream(streamConfiguration.name) + } + alreadyConfiguredStreams.add(streamConfiguration.name) + } + return true + } + + /** + * Given an AirbyteCatalog, return a map of valid streams where key == name and value == the stream + * config. + * + * @param airbyteCatalog Airbyte catalog to pull streams out of + * @return map of stream name: stream config + */ + fun getValidStreams(airbyteCatalog: AirbyteCatalog): Map { + val validStreams: MutableMap = HashMap() + for (schemaStream in airbyteCatalog.streams) { + validStreams[schemaStream.stream!!.name] = schemaStream + } + return validStreams + } + + /** + * Validate cron configuration for a given connectionschedule. + * + * @param connectionSchedule the schedule to validate + * @return boolean, but mostly so we can callwithTracker. + */ + fun validateCronConfiguration(connectionSchedule: ConnectionSchedule?): Boolean { + if (connectionSchedule != null) { + if (connectionSchedule.scheduleType != null && connectionSchedule.scheduleType === ScheduleTypeEnum.CRON) { + if (connectionSchedule.cronExpression == null) { + throw ConnectionConfigurationProblem.missingCronExpression() + } + try { + if (connectionSchedule.cronExpression.endsWith("UTC")) { + connectionSchedule.cronExpression = connectionSchedule.cronExpression.replace("UTC", "").trim() + } + val cron: Cron = parser.parse(connectionSchedule.cronExpression) + cron.validate() + val cronStrings: List = cron.asString().split("\\s+") + // Ensure first value is not `*`, could be seconds or minutes value + Integer.valueOf(cronStrings[0]) + if (cronStrings.size == MAX_LENGTH_OF_CRON) { + // Ensure minutes value is not `*` + Integer.valueOf(cronStrings[1]) + } + } catch (e: IllegalArgumentException) { + throw ConnectionConfigurationProblem.invalidCronExpression(connectionSchedule.cronExpression) + } + } + } + return true + // validate that the cron expression is not more often than every hour due to product specs + // check that the first seconds and hour values are not * + } + + /** + * Validates a stream's configurations and sets those configurations in the + * `AirbyteStreamConfiguration` object. Logic comes from + * https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#configuredairbytestream. + * + * @param streamConfiguration The configuration input of a specific stream provided by the caller. + * @param validDestinationSyncModes All the valid destination sync modes for a destination + * @param airbyteStream The immutable schema defined by the source + * @param config The configuration of a stream consumed by the config-api + * @return True if no exceptions. Needed so it can be used inside TrackingHelper.callWithTracker + */ + fun setAndValidateStreamConfig( + streamConfiguration: StreamConfiguration, + validDestinationSyncModes: List, + airbyteStream: AirbyteStream, + config: AirbyteStreamConfiguration, + ): Boolean { + // Set stream config as selected + config.selected = true + if (streamConfiguration.syncMode == null) { + setConfigDefaultFullRefreshOverwrite(config) + return true + } + + // validate that sync and destination modes are valid + val validCombinedSyncModes: Set = validCombinedSyncModes(airbyteStream.supportedSyncModes, validDestinationSyncModes) + if (!validCombinedSyncModes.contains(streamConfiguration.syncMode)) { + throw ConnectionConfigurationProblem.handleSyncModeProblem( + streamConfiguration.syncMode, + streamConfiguration.name, + validCombinedSyncModes, + ) + } + when (streamConfiguration.syncMode) { + ConnectionSyncModeEnum.FULL_REFRESH_APPEND -> { + config.syncMode = SyncMode.FULL_REFRESH + config.destinationSyncMode = DestinationSyncMode.APPEND + } + + ConnectionSyncModeEnum.INCREMENTAL_APPEND -> { + config.syncMode = SyncMode.INCREMENTAL + config.destinationSyncMode = DestinationSyncMode.APPEND + setAndValidateCursorField(streamConfiguration.cursorField, airbyteStream, config) + } + + ConnectionSyncModeEnum.INCREMENTAL_DEDUPED_HISTORY -> { + config.syncMode = SyncMode.INCREMENTAL + config.destinationSyncMode = DestinationSyncMode.APPEND_DEDUP + setAndValidateCursorField(streamConfiguration.cursorField, airbyteStream, config) + setAndValidatePrimaryKey(streamConfiguration.primaryKey, airbyteStream, config) + } + + else -> { + // always valid + setConfigDefaultFullRefreshOverwrite(config) + } + } + return true + } + + private fun setAndValidateCursorField( + cursorField: List?, + airbyteStream: AirbyteStream, + config: AirbyteStreamConfiguration, + ) { + if (airbyteStream.sourceDefinedCursor != null && airbyteStream.sourceDefinedCursor!!) { + if (!cursorField.isNullOrEmpty()) { + // if cursor given is not empty and is NOT the same as the default, throw error + if (java.util.Set.copyOf(cursorField) != java.util.Set.copyOf(airbyteStream.defaultCursorField)) { + throw ConnectionConfigurationProblem.sourceDefinedCursorFieldProblem(airbyteStream.name, airbyteStream.defaultCursorField!!) + } + } + config.cursorField = airbyteStream.defaultCursorField // this probably isn't necessary and should be already set + } else { + if (!cursorField.isNullOrEmpty()) { + // validate cursor field + val validCursorFields: List> = getStreamFields(airbyteStream.jsonSchema!!) + if (!validCursorFields.contains(cursorField)) { + throw ConnectionConfigurationProblem.invalidCursorField(airbyteStream.name, validCursorFields) + } + config.cursorField = cursorField + } else { + // no default or given cursor field + if (airbyteStream.defaultCursorField == null || airbyteStream.defaultCursorField!!.isEmpty()) { + throw ConnectionConfigurationProblem.missingCursorField(airbyteStream.name) + } + config.cursorField = airbyteStream.defaultCursorField // this probably isn't necessary and should be already set + } + } + } + + private fun setAndValidatePrimaryKey( + primaryKey: List>?, + airbyteStream: AirbyteStream, + config: AirbyteStreamConfiguration, + ) { + // if no source defined primary key + if (airbyteStream.sourceDefinedPrimaryKey == null || airbyteStream.sourceDefinedPrimaryKey!!.isEmpty()) { + if (!primaryKey.isNullOrEmpty()) { + // validate primary key + val validPrimaryKey: List> = getStreamFields(airbyteStream.jsonSchema!!) + + // todo maybe check that they don't provide the same primary key twice? + for (singlePrimaryKey in primaryKey) { + if (!validPrimaryKey.contains(singlePrimaryKey)) { // todo double check if the .contains() for list of strings works as intended + throw ConnectionConfigurationProblem.invalidPrimaryKey(airbyteStream.name, validPrimaryKey) + } + } + config.primaryKey = primaryKey + } else { + throw ConnectionConfigurationProblem.missingPrimaryKey(airbyteStream.name) + } + } else { + // source defined primary key exists + if (!primaryKey.isNullOrEmpty()) { + throw ConnectionConfigurationProblem.primaryKeyAlreadyDefined(airbyteStream.name) + } else { + config.primaryKey = airbyteStream.sourceDefinedPrimaryKey // this probably isn't necessary and should be already set + } + } + } + + /** + * Fetch a set off the valid combined sync modes given the valid source/destination sync modes. + * + * @param validSourceSyncModes - List of valid source sync modes + * @param validDestinationSyncModes - list of valid destination sync modes + * @return Set of valid ConnectionSyncModeEnum values + */ + fun validCombinedSyncModes( + validSourceSyncModes: List?, + validDestinationSyncModes: List, + ): Set { + val validCombinedSyncModes: MutableSet = HashSet() + for (sourceSyncMode in validSourceSyncModes!!) { + for (destinationSyncMode in validDestinationSyncModes) { + val combinedSyncMode: ConnectionSyncModeEnum? = + ConnectionReadMapper.syncModesToConnectionSyncModeEnum(sourceSyncMode, destinationSyncMode) + // This is true when the supported sync modes include full_refresh and the destination supports + // append_deduped + // or when the sync modes include incremental and the destination supports overwrite + if (combinedSyncMode != null) { + validCombinedSyncModes.add(combinedSyncMode) + } + } + } + return validCombinedSyncModes + } + + /** + * Parses a connectorSchema to retrieve all the possible stream fields. + * + * @param connectorSchema source or destination schema + * @return A list of stream fields, which are represented as list of strings since they can be + * nested fields. + */ + fun getStreamFields(connectorSchema: JsonNode): List> { + val yamlMapper = ObjectMapper(YAMLFactory()) + val streamFields: MutableList> = ArrayList() + val spec: JsonNode + spec = try { + yamlMapper.readTree(connectorSchema.traverse()) + } catch (e: IOException) { + log.error("Error getting stream fields from schema", e) + throw UnexpectedProblem(HttpStatus.INTERNAL_SERVER_ERROR) + } + val fields = spec.fields() + while (fields.hasNext()) { + val (key, paths) = fields.next() + if ("properties" == key) { + val propertyFields = paths.fields() + while (propertyFields.hasNext()) { + val (propertyName, nestedProperties) = propertyFields.next() + streamFields.add(java.util.List.of(propertyName)) + + // retrieve nested paths + for (entry in getStreamFields(nestedProperties)) { + if (entry.isEmpty()) { + continue + } + val streamFieldPath: MutableList = ArrayList(java.util.List.of(propertyName)) + streamFieldPath.addAll(entry) + streamFields.add(streamFieldPath) + } + } + } + } + return streamFields.toList() + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ApiTrackingHelpers.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ApiTrackingHelpers.kt new file mode 100644 index 00000000000..f50c8c78124 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ApiTrackingHelpers.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ +package io.airbyte.api.server.helpers + +import io.airbyte.analytics.Deployment +import io.airbyte.analytics.TrackingClientSingleton +import io.airbyte.commons.version.AirbyteVersion +import io.airbyte.config.Configs +import org.slf4j.LoggerFactory +import java.util.Optional +import java.util.UUID + +/** + * This class builds upon inherited tracking clients from OSS Airbyte. + *

+ * The {@link io.airbyte.analytics.LoggingTrackingClient} is pretty straightforward. + *

+ * The {@link io.airbyte.analytics.SegmentTrackingClient} is where things get slightly confusing. + *

+ * The Segment client expects an initialisation call to be made via the setupTrackingClient(UUID, + * AirbyteVersion, TrackingStrategy, Database) method. + *

+ */ +private val log = LoggerFactory.getLogger("ApiTrackingHelpers.kt") + +// Operation names +private val AIRBYTE_API_CALL = "Airbyte_API_Call" + +private val USER_ID = "user_id" +private val ENDPOINT = "endpoint" +private val OPERATION = "operation" +private val STATUS_CODE = "status_code" +private val API_VERSION = "api_version" +private val WORKSPACE = "workspace" + +fun setupTrackingClient( + airbyteVersion: AirbyteVersion?, + trackingStrategy: Configs.TrackingStrategy?, +) { + // fake a deployment UUID until we want to have the public api server talking to the database + // directly + val deploymentId = UUID.randomUUID() + log.info("setting up tracking client with deploymentId: $deploymentId") + TrackingClientSingleton.initializeWithoutDatabase( + trackingStrategy, + Deployment( + Configs.DeploymentMode.OSS, + deploymentId, + Configs.WorkerEnvironment.KUBERNETES, + ), + airbyteVersion, + ) +} + +fun track( + userId: UUID?, + endpointPath: String?, + httpOperation: String?, + httpStatusCode: Int, + apiVersion: String?, + workspaceId: Optional, +) { + val payload = mutableMapOf( + Pair(USER_ID, userId), + Pair(ENDPOINT, endpointPath), + Pair(OPERATION, httpOperation), + Pair(STATUS_CODE, httpStatusCode), + Pair(API_VERSION, apiVersion), + ) + if (workspaceId.isPresent) { + payload[WORKSPACE] = workspaceId.get().toString() + } + TrackingClientSingleton.get().track( + userId, + AIRBYTE_API_CALL, + payload as Map?, + ) +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ConnectionHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ConnectionHelper.kt new file mode 100644 index 00000000000..c14dc2adbbc --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ConnectionHelper.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.helpers + +import io.airbyte.airbyte_api.model.generated.NamespaceDefinitionEnum +import io.airbyte.airbyte_api.model.generated.NamespaceDefinitionEnumNoDefault +import io.airbyte.airbyte_api.model.generated.NonBreakingSchemaUpdatesBehaviorEnum +import io.airbyte.airbyte_api.model.generated.NonBreakingSchemaUpdatesBehaviorEnumNoDefault +import io.airbyte.api.client.model.generated.NamespaceDefinitionType +import io.airbyte.api.client.model.generated.NonBreakingChangesPreference + +/** + * Connection helpers. + */ +object ConnectionHelper { + /** + * Convert namespace definition enum -> NamespaceDefinitionType. + */ + fun convertNamespaceDefinitionEnum(namespaceDefinitionEnum: NamespaceDefinitionEnum): NamespaceDefinitionType { + return if (namespaceDefinitionEnum === NamespaceDefinitionEnum.CUSTOM_FORMAT) { + NamespaceDefinitionType.CUSTOMFORMAT + } else { + NamespaceDefinitionType.fromValue(namespaceDefinitionEnum.toString()) + } + } + + /** + * Convert namespace definition enum -> NamespaceDefinitionType. + */ + fun convertNamespaceDefinitionEnum(namespaceDefinitionEnum: NamespaceDefinitionEnumNoDefault): NamespaceDefinitionType { + return if (namespaceDefinitionEnum === NamespaceDefinitionEnumNoDefault.CUSTOM_FORMAT) { + NamespaceDefinitionType.CUSTOMFORMAT + } else { + NamespaceDefinitionType.fromValue(namespaceDefinitionEnum.toString()) + } + } + + /** + * Convert non-breaking schema updates behavior enum -> NonBreakingChangesPreference. + */ + fun convertNonBreakingSchemaUpdatesBehaviorEnum( + nonBreakingSchemaUpdatesBehaviorEnum: NonBreakingSchemaUpdatesBehaviorEnum, + ): NonBreakingChangesPreference { + return if (nonBreakingSchemaUpdatesBehaviorEnum === NonBreakingSchemaUpdatesBehaviorEnum.DISABLE_CONNECTION) { + NonBreakingChangesPreference.DISABLE + } else { + NonBreakingChangesPreference.fromValue(nonBreakingSchemaUpdatesBehaviorEnum.toString()) + } + } + + /** + * Convert non-breaking schema updates behavior enum -> NonBreakingChangesPreference. + */ + fun convertNonBreakingSchemaUpdatesBehaviorEnum( + nonBreakingSchemaUpdatesBehaviorEnum: NonBreakingSchemaUpdatesBehaviorEnumNoDefault, + ): NonBreakingChangesPreference { + return if (nonBreakingSchemaUpdatesBehaviorEnum === NonBreakingSchemaUpdatesBehaviorEnumNoDefault.DISABLE_CONNECTION) { + NonBreakingChangesPreference.DISABLE + } else { + NonBreakingChangesPreference.fromValue(nonBreakingSchemaUpdatesBehaviorEnum.toString()) + } + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/LocalUserInfoHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/LocalUserInfoHelper.kt new file mode 100644 index 00000000000..c709908b86d --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/LocalUserInfoHelper.kt @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.helpers + +import io.airbyte.api.server.constants.AIRBYTE_API_AUTH_HEADER_VALUE + +/** + * Not used for OSS, in OSS this will return null. + */ +fun getLocalUserInfoIfNull(userInfo: String?): String? { + return userInfo ?: System.getenv(AIRBYTE_API_AUTH_HEADER_VALUE) +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/NameToDefinitionMappingHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/NameToDefinitionMappingHelper.kt new file mode 100644 index 00000000000..4c179875159 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/NameToDefinitionMappingHelper.kt @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.helpers + +import io.airbyte.api.server.problems.UnknownValueProblem +import java.util.UUID + +fun getIdFromName(nameToDefinitionIdMap: Map, name: String?): UUID { + return nameToDefinitionIdMap[name] ?: throw UnknownValueProblem(name) +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/OAuthHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/OAuthHelper.kt new file mode 100644 index 00000000000..a8a05d22ad3 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/OAuthHelper.kt @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.helpers + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode +import io.airbyte.api.server.problems.InvalidRedirectUrlProblem +import io.airbyte.commons.json.Jsons +import io.airbyte.protocol.models.ConnectorSpecification +import org.slf4j.LoggerFactory +import java.io.IOException +import java.net.URI +import java.util.stream.Stream + +/** + * Helper for OAuth. + */ +object OAuthHelper { + private const val TEMP_OAUTH_STATE_KEY = "temp_oauth_state" + private const val HTTPS = "https" + private val OBJECT_MAPPER = ObjectMapper() + private const val PROPERTIES = "properties" + private val log = LoggerFactory.getLogger(OAuthHelper.javaClass) + + fun buildTempOAuthStateKey(state: String): String { + return "$TEMP_OAUTH_STATE_KEY.$state" + } + + /** + * Helper function to validate that a redirect URL is valid and if not, return the appropriate + * problem. + */ + fun validateRedirectUrl(redirectUrl: String?) { + if (redirectUrl == null) { + throw InvalidRedirectUrlProblem("Redirect URL cannot be null") + } + try { + val uri = URI.create(redirectUrl) + if (uri.scheme != HTTPS) { + throw InvalidRedirectUrlProblem("Redirect URL must use HTTPS") + } + } catch (e: IllegalArgumentException) { + log.error(e.message) + throw InvalidRedirectUrlProblem("Redirect URL must conform to RFC 2396 - https://www.ietf.org/rfc/rfc2396.txt") + } + } + + /** + * Test if a connector spec has oauth configuration. copied from OAuthConfigSUpplier + * + * @param spec to check + * @return true if it has an oauth config. otherwise, false. + */ + fun hasOAuthConfigSpecification(spec: ConnectorSpecification?): Boolean { + return spec != null && spec.advancedAuth != null && spec.advancedAuth.oauthConfigSpecification != null + } + + /** + * Test if a connector spec has legacy oauth configuration. copied from OAuthConfigSUpplier + * + * @param spec to check + * @return true if it has a legacy oauth config. otherwise, false. + */ + fun hasLegacyOAuthConfigSpecification(spec: ConnectorSpecification?): Boolean { + return spec != null && spec.authSpecification != null && spec.authSpecification.oauth2Specification != null + } + + /** + * Extract oauth config specification. + * + * @param specification - connector specification + * @return JsonNode of newly built OAuth spec. + */ + fun extractOAuthConfigSpecification(specification: ConnectorSpecification): JsonNode { + val oauthConfig: io.airbyte.protocol.models.OAuthConfigSpecification? = specification.advancedAuth.oauthConfigSpecification + val completeOAuthServerOutputSpecification: JsonNode = oauthConfig?.completeOauthServerOutputSpecification!! + val completeOAuthOutputSpecification: JsonNode = oauthConfig.completeOauthOutputSpecification!! + val constructedSpecNode: JsonNode = Jsons.emptyObject() + val oauthOutputPaths = extractFromCompleteOutputSpecification(completeOAuthOutputSpecification) + val oauthServerOutputPaths = extractFromCompleteOutputSpecification(completeOAuthServerOutputSpecification) + val required = Stream.concat(oauthOutputPaths.stream(), oauthServerOutputPaths.stream()).toList() + + // Try to get the original connector specification nodes so we retain documentation + for (paramPath in required) { + // Set required nodes + val paramName = paramPath[paramPath.size - 1] + var specNode: JsonNode = specification.connectionSpecification.get(PROPERTIES).findValue(paramName) + + // If we don't have one, resort to creating a naked node + if (specNode == null) { + val copyNode: Map<*, *> = java.util.Map.of("type", "string", "name", paramName) + specNode = Jsons.jsonNode(copyNode) + } + Jsons.setNestedValue(constructedSpecNode, alternatingList(PROPERTIES, paramPath), specNode) + } + + // Set title etc. + Jsons.setNestedValue(constructedSpecNode, listOf("title"), specification.connectionSpecification.get("title")) + return constructedSpecNode + } + + private fun extractFromCompleteOutputSpecification(outputSpecification: JsonNode): List> { + val properties = outputSpecification[PROPERTIES] + val paths = properties.findValues("path_in_connector_config") + return paths.stream().map> { node: JsonNode? -> + try { + return@map OBJECT_MAPPER.readerForListOf(String::class.java).readValue(node) as List + } catch (e: IOException) { + throw RuntimeException(e) + } + }.toList() + } + + /** + * Extract legacy oauth specification. + * + * @param specification - connector specification + * @return JsonNode of newly built OAuth spec. + */ + fun extractLegacyOAuthSpecification(specification: ConnectorSpecification): JsonNode { + val oauth2Specification: io.airbyte.protocol.models.OAuth2Specification? = specification.authSpecification.oauth2Specification + val rootNode: List = oauth2Specification?.rootObject!! + val specNodePath: List + val originalSpecNode: JsonNode + val ONE = 1 + if (rootNode.size > ONE) { + // Nested oneOf + // specNode = the index of array node specified in the spec + specNodePath = listOf(PROPERTIES, rootNode[0].toString()) + originalSpecNode = Jsons.navigateTo(specification.connectionSpecification, specNodePath).get("oneOf") + .get(rootNode[1].toString().toInt()) + } else if (rootNode.size == ONE) { + // just nested + specNodePath = listOf(PROPERTIES, rootNode[0].toString()) + originalSpecNode = Jsons.navigateTo(specification.connectionSpecification, specNodePath) + } else { + // unnested + specNodePath = listOf(PROPERTIES) + originalSpecNode = specification.connectionSpecification + } + val fields = originalSpecNode[PROPERTIES].fields() + val flatInitParams: MutableList? = oauth2Specification.oauthFlowInitParameters.stream() + .flatMap { obj: Collection<*> -> obj.stream() }.toList() + + // Remove fields that aren't in the init parameters + while (fields.hasNext()) { + val (key) = fields.next() + if (!flatInitParams!!.contains(key)) { + fields.remove() + } + } + + // create fields that aren't already in the original spec node + // In some cases (trello, instagram etc.) we don't need the oauth params in the connector, but we do + // need them to initiate oauth so they aren't + // in the spec, but we still need them + val requiredFields = OBJECT_MAPPER.valueToTree(flatInitParams) + + // Properly set required fields + (originalSpecNode as ObjectNode).putArray("required").addAll(requiredFields) + val constructedSpecNode: JsonNode = Jsons.emptyObject() + Jsons.setNestedValue(constructedSpecNode, specNodePath, originalSpecNode) + return constructedSpecNode + } + + /** + * Create a list with alternating elements of property, list[n]. Used to spoof a connector + * specification for splitting out secrets. + * + * @param property property to put in front of each list element + * @param list list to insert elements into + * @return new list with alternating elements + */ + private fun alternatingList( + property: String, + list: List, + ): List { + val result: MutableList = ArrayList(list.size * 2) + for (item in list) { + result.add(property) + result.add(item) + } + return result + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/TrackingHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/TrackingHelper.kt new file mode 100644 index 00000000000..1bd47ce736a --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/TrackingHelper.kt @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.helpers + +import io.micronaut.http.HttpStatus +import org.zalando.problem.AbstractThrowableProblem +import java.util.Optional +import java.util.UUID +import java.util.concurrent.Callable +import javax.ws.rs.core.Response + +/** + * Helper for segment tracking used by the public-api server. + * Todo: This should be a middleware through a micronaut annotation so that we do not need to add this wrapper functions around all our calls. + */ +object TrackingHelper { + private val API_VERSION = System.getenv("AIRBYTE_VERSION") ?: "unknown" + private fun trackSuccess(endpointPath: String, httpOperation: String, userId: UUID, workspaceId: Optional) { + val statusCode = Response.Status.OK.statusCode + track( + userId, + endpointPath, + httpOperation, + statusCode, + API_VERSION, + workspaceId, + ) + } + + /** + * Track success calls. + */ + fun trackSuccess(endpointPath: String?, httpOperation: String?, userId: UUID?) { + trackSuccess(endpointPath!!, httpOperation!!, userId!!, Optional.empty()) + } + + /** + * Track success calls with workspace id. + */ + fun trackSuccess(endpointPath: String?, httpOperation: String?, userId: UUID?, workspaceId: UUID) { + trackSuccess(endpointPath!!, httpOperation!!, userId!!, Optional.of(workspaceId)) + } + + /** + * Gets the status code from the problem if there was one thrown. + */ + fun trackFailuresIfAny(endpointPath: String?, httpOperation: String?, userId: UUID?, e: Exception?) { + var statusCode = 0 + if (e is AbstractThrowableProblem) { + statusCode = (e as AbstractThrowableProblem?)?.status?.statusCode ?: 500 + } else if (e != null) { + // also contains InvalidConsentUrlProblem + statusCode = HttpStatus.INTERNAL_SERVER_ERROR.code + } + + // only track if there was a failure + if (statusCode != 0) { + track( + userId, + endpointPath, + httpOperation, + statusCode, + API_VERSION, + Optional.empty(), + ) + } + } + + /** + * Tracks the problems thrown from the function being called. The function is usually a call to the + * config or cloud api servers. This tracker DOES NOT track successes. + * + * @param function usually a call to the config or cloud api + * @param endpoint the endpoint being called to be tracked into segment + * @param httpOperation the http operation to be tracked into segment e.g. GET, POST, DELETE + * @param userId the id of the user we want to track + * @return the output of the function. Will send segment tracking event for any exceptions caught + * from the function. + */ + fun callWithTracker(function: Callable, endpoint: String?, httpOperation: String?, userId: UUID?): T { + return try { + function.call() + } catch (e: Exception) { + trackFailuresIfAny(endpoint, httpOperation, userId, e) + throw e + } + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionCreateMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionCreateMapper.kt new file mode 100644 index 00000000000..2ec3324a621 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionCreateMapper.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.ConnectionCreateRequest +import io.airbyte.api.client.model.generated.AirbyteCatalog +import io.airbyte.api.client.model.generated.ConnectionCreate +import io.airbyte.api.client.model.generated.ConnectionScheduleData +import io.airbyte.api.client.model.generated.ConnectionScheduleDataCron +import io.airbyte.api.client.model.generated.ConnectionScheduleType +import io.airbyte.api.client.model.generated.ConnectionStatus +import io.airbyte.api.client.model.generated.Geography +import io.airbyte.api.server.helpers.ConnectionHelper +import java.util.UUID + +/** + * Mappers that help convert models from the public api to models from the config api. + */ +object ConnectionCreateMapper { + /** + * Converts a ConnectionCreateRequest object from the public api to a ConnectionCreate from the + * config api. Assumes validation has been done for all the connectino create request configurations + * including but not limited to cron expressions, streams, and their sync modes. + * + * @param connectionCreateRequest Input of a connection create from public api + * @return ConnectionCreate Response object to be sent to config api + */ + fun from( + connectionCreateRequest: ConnectionCreateRequest, + catalogId: UUID?, + configuredCatalog: AirbyteCatalog?, + ): ConnectionCreate { + val connectionCreateOss = ConnectionCreate() + connectionCreateOss.sourceId = connectionCreateRequest.sourceId + connectionCreateOss.destinationId = connectionCreateRequest.destinationId + connectionCreateOss.name = connectionCreateRequest.name + connectionCreateOss.nonBreakingChangesPreference = ConnectionHelper.convertNonBreakingSchemaUpdatesBehaviorEnum( + connectionCreateRequest.nonBreakingSchemaUpdatesBehavior, + ) + connectionCreateOss.namespaceDefinition = ConnectionHelper.convertNamespaceDefinitionEnum(connectionCreateRequest.namespaceDefinition) + if (connectionCreateRequest.namespaceFormat != null) { + connectionCreateOss.namespaceFormat = connectionCreateRequest.namespaceFormat + } + if (connectionCreateRequest.prefix != null) { + connectionCreateOss.setPrefix(connectionCreateRequest.prefix) + } + + // set geography + connectionCreateOss.setGeography(Geography.fromValue(connectionCreateRequest.dataResidency.toString())) + + // set schedule + if (connectionCreateRequest.schedule != null) { + connectionCreateOss.scheduleType = ConnectionScheduleType.fromValue(connectionCreateRequest.schedule.scheduleType.toString()) + val connectionScheduleDataCron = ConnectionScheduleDataCron() + connectionScheduleDataCron.cronExpression = connectionCreateRequest.schedule.cronExpression + connectionScheduleDataCron.setCronTimeZone("UTC") + val connectionScheduleData = ConnectionScheduleData() + connectionScheduleData.setCron(connectionScheduleDataCron) + connectionCreateOss.setScheduleData(connectionScheduleData) + } else { + connectionCreateOss.setScheduleType(ConnectionScheduleType.MANUAL) + } + + // set streams + if (catalogId != null) { + connectionCreateOss.setSourceCatalogId(catalogId) + } + if (configuredCatalog != null) { + connectionCreateOss.setSyncCatalog(configuredCatalog) + } + if (connectionCreateRequest.status != null) { + connectionCreateOss.setStatus(ConnectionStatus.fromValue(connectionCreateRequest.status.toString())) + } else { + connectionCreateOss.setStatus(ConnectionStatus.ACTIVE) + } + return connectionCreateOss + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionReadMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionReadMapper.kt new file mode 100644 index 00000000000..4e1eaf05f88 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionReadMapper.kt @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.ConnectionResponse +import io.airbyte.airbyte_api.model.generated.ConnectionScheduleResponse +import io.airbyte.airbyte_api.model.generated.ConnectionStatusEnum +import io.airbyte.airbyte_api.model.generated.ConnectionSyncModeEnum +import io.airbyte.airbyte_api.model.generated.GeographyEnum +import io.airbyte.airbyte_api.model.generated.NamespaceDefinitionEnum +import io.airbyte.airbyte_api.model.generated.NonBreakingSchemaUpdatesBehaviorEnum +import io.airbyte.airbyte_api.model.generated.ScheduleTypeWithBasicEnum +import io.airbyte.airbyte_api.model.generated.StreamConfiguration +import io.airbyte.airbyte_api.model.generated.StreamConfigurations +import io.airbyte.api.client.model.generated.ConnectionRead +import io.airbyte.api.client.model.generated.ConnectionScheduleType +import io.airbyte.api.client.model.generated.DestinationSyncMode +import io.airbyte.api.client.model.generated.NamespaceDefinitionType +import io.airbyte.api.client.model.generated.NonBreakingChangesPreference +import io.airbyte.api.client.model.generated.SyncMode +import java.util.UUID + +/** + * Mappers that help convert models from the config api to models from the public api. + */ +object ConnectionReadMapper { + /** + * Converts a ConnectionRead object from the config api to a ConnectionResponse. + * + * @param connectionRead Output of a connection create/get from config api + * @return ConnectionResponse Response object + */ + fun from(connectionRead: ConnectionRead, workspaceId: UUID?): ConnectionResponse { + val connectionResponse = ConnectionResponse() + connectionResponse.setConnectionId(connectionRead.connectionId) + connectionResponse.setName(connectionRead.name) + connectionResponse.setSourceId(connectionRead.sourceId) + connectionResponse.setDestinationId(connectionRead.destinationId) + connectionResponse.setWorkspaceId(workspaceId) + connectionResponse.setStatus(ConnectionStatusEnum.fromValue(connectionRead.status.value)) + if (connectionRead.geography != null) { + connectionResponse.setDataResidency(GeographyEnum.fromValue(connectionRead.geography!!.value)) + } + val connectionScheduleResponse = ConnectionScheduleResponse() + if (connectionRead.namespaceDefinition != null) { + connectionResponse.setNamespaceDefinition(convertNamespaceDefinitionType(connectionRead.namespaceDefinition)) + } + if (connectionRead.namespaceFormat != null) { + connectionResponse.setNamespaceFormat(connectionRead.namespaceFormat) + } + if (connectionRead.prefix != null) { + connectionResponse.setPrefix(connectionRead.prefix) + } + if (connectionRead.nonBreakingChangesPreference != null) { + connectionResponse.setNonBreakingSchemaUpdatesBehavior(convertNonBreakingChangesPreference(connectionRead.nonBreakingChangesPreference)) + } + + // connectionRead.getSchedule() is soon to be deprecated, but has not been cleaned up in the + // database yet + if (connectionRead.schedule != null) { + connectionScheduleResponse.setScheduleType(ScheduleTypeWithBasicEnum.BASIC) + // should this string just be a json object? + val basicTimingString = "Every " + connectionRead.schedule!!.units + " " + connectionRead.schedule!!.timeUnit.value + connectionScheduleResponse.setBasicTiming(basicTimingString) + } else if (connectionRead.schedule != null && connectionRead.scheduleType == null) { + connectionScheduleResponse.setScheduleType(ScheduleTypeWithBasicEnum.MANUAL) + } + if (connectionRead.scheduleType == ConnectionScheduleType.MANUAL) { + connectionScheduleResponse.setScheduleType(ScheduleTypeWithBasicEnum.MANUAL) + } else if (connectionRead.scheduleType == ConnectionScheduleType.CRON) { + connectionScheduleResponse.setScheduleType(ScheduleTypeWithBasicEnum.CRON) + if (connectionRead.scheduleData != null && connectionRead.scheduleData!!.cron != null) { + // should this string just be a json object? + val cronExpressionWithTimezone = connectionRead.scheduleData!!.cron!! + .cronExpression + " " + connectionRead.scheduleData!!.cron!!.cronTimeZone + connectionScheduleResponse.setCronExpression(cronExpressionWithTimezone) + } else { +// ConnectionReadMapper.log.error("CronExpression not found in ScheduleData for connection: {}", connectionRead.connectionId) + } + } else if (connectionRead.scheduleType == ConnectionScheduleType.BASIC) { + connectionScheduleResponse.setScheduleType(ScheduleTypeWithBasicEnum.BASIC) + if (connectionRead.scheduleData != null && connectionRead.scheduleData!!.basicSchedule != null) { + val schedule = connectionRead.scheduleData!!.basicSchedule + val basicTimingString = "Every " + schedule!!.units + " " + schedule.timeUnit.value + connectionScheduleResponse.setBasicTiming(basicTimingString) + } else { +// ConnectionReadMapper.log.error("BasicSchedule not found in ScheduleData for connection: {}", connectionRead.connectionId) + } + } + if (connectionRead.syncCatalog != null) { + val streamConfigurations = StreamConfigurations() + for (streamAndConfiguration in connectionRead.syncCatalog.streams) { + assert(streamAndConfiguration.config != null) + val connectionSyncMode: ConnectionSyncModeEnum? = syncModesToConnectionSyncModeEnum( + streamAndConfiguration.config!!.syncMode, + streamAndConfiguration.config!!.destinationSyncMode, + ) + streamConfigurations.addStreamsItem( + StreamConfiguration() + .name(streamAndConfiguration.stream!!.name) + .primaryKey(streamAndConfiguration.config!!.primaryKey) + .cursorField(streamAndConfiguration.config!!.cursorField) + .syncMode(connectionSyncMode), + ) + } + connectionResponse.setConfigurations(streamConfigurations) + } + connectionResponse.setSchedule(connectionScheduleResponse) + return connectionResponse + } + + private fun convertNamespaceDefinitionType(namespaceDefinitionType: NamespaceDefinitionType?): NamespaceDefinitionEnum { + return if (namespaceDefinitionType == NamespaceDefinitionType.CUSTOMFORMAT) { + NamespaceDefinitionEnum.CUSTOM_FORMAT + } else { + NamespaceDefinitionEnum.fromValue(namespaceDefinitionType.toString()) + } + } + + private fun convertNonBreakingChangesPreference(nonBreakingChangesPreference: NonBreakingChangesPreference?): NonBreakingSchemaUpdatesBehaviorEnum { + return if (nonBreakingChangesPreference == NonBreakingChangesPreference.DISABLE) { + NonBreakingSchemaUpdatesBehaviorEnum.DISABLE_CONNECTION + } else { + NonBreakingSchemaUpdatesBehaviorEnum.fromValue(nonBreakingChangesPreference.toString()) + } + } + + /** + * Map sync modes to combined sync modes. + * + * @param sourceSyncMode - source sync mode + * @param destinationSyncMode - destination sync mode + * @return - ConnectionSyncModeEnum value that corresponds. Null if there isn't one. + */ + fun syncModesToConnectionSyncModeEnum( + sourceSyncMode: SyncMode?, + destinationSyncMode: DestinationSyncMode?, + ): ConnectionSyncModeEnum? { + val mapper: MutableMap>? = + java.util.Map.of( + SyncMode.FULL_REFRESH, + mapOf( + Pair( + DestinationSyncMode.OVERWRITE, + ConnectionSyncModeEnum.FULL_REFRESH_OVERWRITE, + ), + Pair( + DestinationSyncMode.APPEND, + ConnectionSyncModeEnum.FULL_REFRESH_APPEND, + ), + ), + SyncMode.INCREMENTAL, + mapOf( + Pair( + DestinationSyncMode.APPEND, + ConnectionSyncModeEnum.INCREMENTAL_APPEND, + ), + Pair( + DestinationSyncMode.APPEND_DEDUP, + ConnectionSyncModeEnum.INCREMENTAL_DEDUPED_HISTORY, + ), + ), + ) + return if (sourceSyncMode == null || destinationSyncMode == null || mapper!![sourceSyncMode] == null) { + ConnectionSyncModeEnum.FULL_REFRESH_OVERWRITE + } else { + mapper[sourceSyncMode]!![destinationSyncMode] + } + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionUpdateMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionUpdateMapper.kt new file mode 100644 index 00000000000..f33c0804e4e --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionUpdateMapper.kt @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.ConnectionPatchRequest +import io.airbyte.airbyte_api.model.generated.ScheduleTypeEnum +import io.airbyte.api.client.model.generated.AirbyteCatalog +import io.airbyte.api.client.model.generated.ConnectionScheduleData +import io.airbyte.api.client.model.generated.ConnectionScheduleDataCron +import io.airbyte.api.client.model.generated.ConnectionScheduleType +import io.airbyte.api.client.model.generated.ConnectionStatus +import io.airbyte.api.client.model.generated.ConnectionUpdate +import io.airbyte.api.client.model.generated.Geography +import io.airbyte.api.server.helpers.ConnectionHelper +import java.util.UUID +import javax.validation.constraints.NotBlank + +/** + * Mappers that help convert models from the public api to models from the config api. + */ +object ConnectionUpdateMapper { + /** + * Converts a ConnectionPatchRequest object from the public api to a ConnectionUpdate from the + * config api. Assumes validation has been done for all the connection configurations including but + * not limited to cron expressions, streams, and their sync modes. + * + * @param connectionId connection Id + * @param connectionPatchRequest Input of a connection put from public api + * @return ConnectionCreate Response object to be sent to config api + */ + fun from( + connectionId: @NotBlank UUID?, + connectionPatchRequest: ConnectionPatchRequest, + catalogId: UUID?, + configuredCatalog: AirbyteCatalog?, + ): ConnectionUpdate { + val connectionUpdateOss = ConnectionUpdate() + connectionUpdateOss.connectionId(connectionId) + connectionUpdateOss.setName(connectionPatchRequest.getName()) + if (connectionPatchRequest.getNonBreakingSchemaUpdatesBehavior() != null) { + connectionUpdateOss.setNonBreakingChangesPreference( + ConnectionHelper.convertNonBreakingSchemaUpdatesBehaviorEnum(connectionPatchRequest.getNonBreakingSchemaUpdatesBehavior()), + ) + } + if (connectionPatchRequest.getNamespaceDefinition() != null) { + connectionUpdateOss.setNamespaceDefinition( + ConnectionHelper.convertNamespaceDefinitionEnum(connectionPatchRequest.getNamespaceDefinition()), + ) + } + if (connectionPatchRequest.getNamespaceFormat() != null) { + connectionUpdateOss.setNamespaceFormat(connectionPatchRequest.getNamespaceFormat()) + } + if (connectionPatchRequest.getPrefix() != null) { + connectionUpdateOss.setPrefix(connectionPatchRequest.getPrefix()) + } + + // set geography + if (connectionPatchRequest.getDataResidency() != null) { + connectionUpdateOss.setGeography(Geography.fromValue(connectionPatchRequest.getDataResidency().toString())) + } + + // set schedule + if (connectionPatchRequest.getSchedule() != null) { + connectionUpdateOss.setScheduleType(ConnectionScheduleType.fromValue(connectionPatchRequest.getSchedule().getScheduleType().toString())) + if (connectionPatchRequest.getSchedule().getScheduleType() !== ScheduleTypeEnum.MANUAL) { + // This should only be set if we're not manual + val connectionScheduleDataCron = ConnectionScheduleDataCron() + connectionScheduleDataCron.setCronExpression(connectionPatchRequest.getSchedule().getCronExpression()) + connectionScheduleDataCron.setCronTimeZone("UTC") + val connectionScheduleData = ConnectionScheduleData() + connectionScheduleData.setCron(connectionScheduleDataCron) + connectionUpdateOss.setScheduleData(connectionScheduleData) + } else { + connectionUpdateOss.setScheduleType(ConnectionScheduleType.MANUAL) + } + } + + // set streams + if (catalogId != null) { + connectionUpdateOss.setSourceCatalogId(catalogId) + } + if (configuredCatalog != null) { + connectionUpdateOss.setSyncCatalog(configuredCatalog) + } + if (connectionPatchRequest.getStatus() != null) { + connectionUpdateOss.setStatus(ConnectionStatus.fromValue(connectionPatchRequest.getStatus().toString())) + } + return connectionUpdateOss + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionsResponseMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionsResponseMapper.kt new file mode 100644 index 00000000000..6dbd2a17ea6 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/ConnectionsResponseMapper.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.ConnectionsResponse +import io.airbyte.api.client.model.generated.ConnectionRead +import io.airbyte.api.client.model.generated.ConnectionReadList +import io.airbyte.api.server.constants.CONNECTIONS_PATH +import io.airbyte.api.server.constants.INCLUDE_DELETED +import io.airbyte.api.server.constants.WORKSPACE_IDS +import java.util.UUID + +/** + * Mappers that help convert models from the config api to models from the public api. + */ +object ConnectionsResponseMapper { + /** + * Converts a ConnectionReadList object from the config api to a ConnectionsResponse object. + * + * @param connectionReadList Output of a connection list from config api + * @param workspaceIds workspaceIds requested by the user, if empty assume all workspaces requested + * @param includeDeleted did we include deleted workspaces or not? + * @param limit Number of JobResponses to be outputted + * @param offset Offset of the pagination + * @param apiHost Host url e.g. api.airbyte.com + * @return JobsResponse List of JobResponse along with a next and previous https requests + */ + fun from( + connectionReadList: ConnectionReadList, + workspaceIds: List, + includeDeleted: Boolean, + limit: Int, + offset: Int, + apiHost: String, + ): ConnectionsResponse { + val uriBuilder = PaginationMapper.getBuilder(apiHost, CONNECTIONS_PATH) + .queryParam(WORKSPACE_IDS, PaginationMapper.uuidListToQueryString(workspaceIds)) + .queryParam(INCLUDE_DELETED, includeDeleted) + val connectionsResponse = ConnectionsResponse() + connectionsResponse.setNext(PaginationMapper.getNextUrl(connectionReadList.connections, limit, offset, uriBuilder)) + connectionsResponse.setPrevious(PaginationMapper.getPreviousUrl(limit, offset, uriBuilder)) + connectionsResponse.setData( + connectionReadList.connections.map { connectionRead: ConnectionRead -> ConnectionReadMapper.from(connectionRead, connectionRead.workspaceId) }, + ) + return connectionsResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/DestinationReadMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/DestinationReadMapper.kt new file mode 100644 index 00000000000..9ba5ebf0515 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/DestinationReadMapper.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.DestinationResponse +import io.airbyte.api.client.model.generated.DestinationRead + +/** + * Mappers that help convert models from the config api to models from the public api. + */ +object DestinationReadMapper { + /** + * Converts a DestinationRead object from the config api to a DestinationResponse. + * + * @param destinationRead Output of a destination create/get from config api + * @return DestinationResponse Response object with destination details + */ + fun from(destinationRead: DestinationRead): DestinationResponse { + val destinationResponse = DestinationResponse() + destinationResponse.destinationId = destinationRead.destinationId + destinationResponse.name = destinationRead.name + destinationResponse.destinationType = DEFINITION_ID_TO_DESTINATION_NAME[destinationRead.destinationDefinitionId] + destinationResponse.workspaceId = destinationRead.workspaceId + destinationResponse.configuration = destinationRead.connectionConfiguration + return destinationResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/DestinationsResponseMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/DestinationsResponseMapper.kt new file mode 100644 index 00000000000..9aee5345152 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/DestinationsResponseMapper.kt @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.DestinationsResponse +import io.airbyte.api.client.model.generated.DestinationRead +import io.airbyte.api.client.model.generated.DestinationReadList +import io.airbyte.api.server.constants.DESTINATIONS_PATH +import io.airbyte.api.server.constants.INCLUDE_DELETED +import io.airbyte.api.server.constants.WORKSPACE_IDS +import java.util.UUID +import java.util.function.Function + +/** + * Maps config API DestinationReadLists to DestinationsResponse. + */ +object DestinationsResponseMapper { + /** + * Converts a SourceReadList object from the config api to a SourcesResponse object. + * + * @param destinationReadList Output of a destination list from config api + * @param workspaceIds workspaceIds we wanted to list + * @param includeDeleted did we include deleted workspaces or not? + * @param limit Number of responses to be outputted + * @param offset Offset of the pagination + * @param apiHost Host url e.g. api.airbyte.com + * @return DestinationsResponse List of DestinationResponse along with a next and previous https + * requests + */ + fun from( + destinationReadList: DestinationReadList, + workspaceIds: List, + includeDeleted: Boolean, + limit: Int, + offset: Int, + apiHost: String, + ): DestinationsResponse { + val uriBuilder = PaginationMapper.getBuilder(apiHost, DESTINATIONS_PATH) + .queryParam(WORKSPACE_IDS, PaginationMapper.uuidListToQueryString(workspaceIds)) + .queryParam(INCLUDE_DELETED, includeDeleted) + val destinationsResponse = DestinationsResponse() + destinationsResponse.setNext(PaginationMapper.getNextUrl(destinationReadList.destinations, limit, offset, uriBuilder)) + destinationsResponse.setPrevious(PaginationMapper.getPreviousUrl(limit, offset, uriBuilder)) + destinationsResponse.setData( + destinationReadList.destinations.stream() + .map(Function { obj: DestinationRead? -> DestinationReadMapper.from(obj!!) }) + .toList(), + ) + return destinationsResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/JobResponseMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/JobResponseMapper.kt new file mode 100644 index 00000000000..01b93a75b63 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/JobResponseMapper.kt @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.JobResponse +import io.airbyte.airbyte_api.model.generated.JobStatusEnum +import io.airbyte.airbyte_api.model.generated.JobTypeEnum +import io.airbyte.api.client.model.generated.JobConfigType +import io.airbyte.api.client.model.generated.JobInfoRead +import io.airbyte.api.client.model.generated.JobRead +import io.airbyte.api.client.model.generated.JobStatus +import io.airbyte.api.client.model.generated.JobWithAttemptsRead +import io.airbyte.api.server.mappers.JobsResponseMapper.ALLOWED_CONFIG_TYPES +import java.time.Duration +import java.time.Instant +import java.time.OffsetDateTime +import java.time.ZoneId +import java.util.Set +import java.util.UUID + +/** + * Mappers that help convert models from the config api to models from the public api. + */ +object JobResponseMapper { + private val TERMINAL_JOB_STATUS = Set.of(JobStatus.FAILED, JobStatus.CANCELLED, JobStatus.SUCCEEDED) + private val UTC = ZoneId.of("UTC") + + /** + * Converts a JobInfoRead object from the config api to a JobResponse object. + * + * @param jobInfoRead Output of a job create/get from config api + * @return JobResponse Response object which contains job id, status, and job type + */ + fun from(jobInfoRead: JobInfoRead): JobResponse { + val jobResponse: JobResponse = jobResponseFromJobReadMinusSyncedData(jobInfoRead.job) + if (jobInfoRead.attempts.size > 0) { + val lastAttempt = jobInfoRead.attempts[jobInfoRead.attempts.size - 1] + jobResponse.setBytesSynced(lastAttempt.attempt.bytesSynced) + jobResponse.setRowsSynced(lastAttempt.attempt.recordsSynced) + } + return jobResponse + } + + /** + * Converts a JobWithAttemptsRead object from the config api to a JobResponse object. + * + * @param jobWithAttemptsRead Output of a job get with attempts from config api + * @return JobResponse Response object which contains job id, status, and job type + */ + fun from(jobWithAttemptsRead: JobWithAttemptsRead): JobResponse { + val jobResponse: JobResponse = jobResponseFromJobReadMinusSyncedData(jobWithAttemptsRead.job) + if (jobWithAttemptsRead.attempts != null && jobWithAttemptsRead.attempts!!.size > 0) { + val lastAttempt = jobWithAttemptsRead.attempts!![jobWithAttemptsRead.attempts!!.size - 1] + jobResponse.setBytesSynced(lastAttempt.bytesSynced) + jobResponse.setRowsSynced(lastAttempt.recordsSynced) + } + return jobResponse + } + + /** + * Converts a JobRead object from the config api to a JobResponse object. + */ + private fun jobResponseFromJobReadMinusSyncedData(jobRead: JobRead?): JobResponse { + val jobResponse = JobResponse() + jobResponse.setJobId(jobRead!!.id) + jobResponse.setStatus(JobStatusEnum.fromValue(jobRead.status.toString())) + jobResponse.setConnectionId(UUID.fromString(jobRead.configId)) + when (jobRead.configType) { + JobConfigType.SYNC -> jobResponse.setJobType(JobTypeEnum.SYNC) + JobConfigType.RESET_CONNECTION -> jobResponse.setJobType(JobTypeEnum.RESET) + else -> { + assert(ALLOWED_CONFIG_TYPES.contains(jobRead.configType)) + } + } + // set to string for now since the jax-rs response entity turns offsetdatetime into epoch seconds + jobResponse.setStartTime(OffsetDateTime.ofInstant(Instant.ofEpochSecond(jobRead.createdAt), UTC).toString()) + if (TERMINAL_JOB_STATUS.contains(jobRead.status)) { + jobResponse.setLastUpdatedAt(OffsetDateTime.ofInstant(Instant.ofEpochSecond(jobRead.updatedAt), UTC).toString()) + } + + // duration is ISO_8601 formatted https://en.wikipedia.org/wiki/ISO_8601#Durations + if (jobRead.status != JobStatus.PENDING) { + jobResponse.setDuration(Duration.ofSeconds(jobRead.updatedAt - jobRead.createdAt).toString()) + } + return jobResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/JobsResponseMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/JobsResponseMapper.kt new file mode 100644 index 00000000000..ea1457b5009 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/JobsResponseMapper.kt @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.JobResponse +import io.airbyte.airbyte_api.model.generated.JobTypeEnum +import io.airbyte.airbyte_api.model.generated.JobsResponse +import io.airbyte.api.client.model.generated.JobConfigType +import io.airbyte.api.client.model.generated.JobReadList +import io.airbyte.api.client.model.generated.JobWithAttemptsRead +import io.airbyte.api.server.constants.JOBS_PATH +import java.util.UUID +import java.util.function.Function + +/** + * Mappers that help convert models from the config api to models from the public api. + */ +object JobsResponseMapper { + val ALLOWED_CONFIG_TYPES = java.util.List.of(JobConfigType.SYNC, JobConfigType.RESET_CONNECTION) + + /** + * Converts a JobReadList object from the config api to a JobsResponse object. + * + * @param jobsList Output of a job list from config api + * @param connectionId Id of the connection + * @param jobType Type of job e.g. sync or reset + * @param limit Number of JobResponses to be outputted + * @param offset Offset of the pagination + * @param apiHost Host url e.g. api.airbyte.com + * @return JobsResponse List of JobResponse along with a next and previous https requests + */ + fun from( + jobsList: JobReadList, + connectionId: UUID?, + jobType: JobTypeEnum?, + limit: Int, + offset: Int, + apiHost: String, + ): JobsResponse { + val jobs: List = jobsList.jobs.stream().filter { j: JobWithAttemptsRead -> + ALLOWED_CONFIG_TYPES.contains( + j.job!!.configType, + ) + }.map( + Function { obj: JobWithAttemptsRead? -> JobResponseMapper.from(obj!!) }, + ).toList() + val uriBuilder = PaginationMapper.getBuilder(apiHost, JOBS_PATH) + .queryParam("jobType", jobType) + .queryParam("connectionId", connectionId) + val jobsResponse = JobsResponse() + jobsResponse.setNext(PaginationMapper.getNextUrl(jobs, limit, offset, uriBuilder)) + jobsResponse.setPrevious(PaginationMapper.getPreviousUrl(limit, offset, uriBuilder)) + jobsResponse.setData(jobs) + return jobsResponse + } + + /** + * Converts a JobReadList object from the config api to a JobsResponse object. + * + * @param jobsList Output of a job list from config api + * @param workspaceIds workspace Ids to filter by + * @param jobType Type of job e.g. sync or reset + * @param limit Number of JobResponses to be outputted + * @param offset Offset of the pagination + * @param apiHost Host url e.g. api.airbyte.com + * @return JobsResponse List of JobResponse along with a next and previous https requests + */ + fun from( + jobsList: JobReadList, + workspaceIds: List?, + jobType: JobTypeEnum?, + limit: Int, + offset: Int, + apiHost: String, + ): JobsResponse { + val jobs: List = jobsList.jobs.stream().filter { j: JobWithAttemptsRead -> + ALLOWED_CONFIG_TYPES.contains( + j.job!!.configType, + ) + }.map( + Function { obj: JobWithAttemptsRead? -> JobResponseMapper.from(obj!!) }, + ).toList() + val uriBuilder = PaginationMapper.getBuilder(apiHost, JOBS_PATH) + .queryParam("jobType", jobType) + .queryParam("workspaceIds", workspaceIds) + val jobsResponse = JobsResponse() + jobsResponse.setNext(PaginationMapper.getNextUrl(jobs, limit, offset, uriBuilder)) + jobsResponse.setPrevious(PaginationMapper.getPreviousUrl(limit, offset, uriBuilder)) + jobsResponse.setData(jobs) + return jobsResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/NameToDefinitionMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/NameToDefinitionMapper.kt new file mode 100644 index 00000000000..c5a9b16a307 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/NameToDefinitionMapper.kt @@ -0,0 +1,723 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ +package io.airbyte.api.server.mappers + +import java.util.UUID + +/* + * The mappings in this file are used so that users can supply a connector name instead of definition id when referring + * to specific connectors via the Airbyte API. + */ +// todo: generate from https://connectors.airbyte.com/files/registries/v0/oss_registry.json instead of hardcoding +val DESTINATION_NAME_TO_DEFINITION_ID: Map = mapOf( + Pair("r2", UUID.fromString("0fb07be9-7c3b-4336-850d-5efc006152ee")), + Pair("amazon-sqs", UUID.fromString("0eeee7fb-518f-4045-bacc-9619e31c43ea")), + Pair("doris", UUID.fromString("05c161bf-ca73-4d48-b524-d392be417002")), + Pair("tidb", UUID.fromString("06ec60c7-7468-45c0-91ac-174f6e1a788b")), + Pair("csv", UUID.fromString("8be1cf83-fde1-477f-a4ad-318d23c9f3c6")), + Pair("kinesis", UUID.fromString("6d1d66d4-26ab-4602-8d32-f85894b04955")), + Pair("redis", UUID.fromString("d4d3fef9-e319-45c2-881a-bd02ce44cc9f")), + Pair("yugabytedb", UUID.fromString("2300fdcf-a532-419f-9f24-a014336e7966")), + Pair("elasticsearch", UUID.fromString("68f351a7-2745-4bef-ad7f-996b8e51bb8c")), + Pair("gcs", UUID.fromString("ca8f6566-e555-4b40-943a-545bf123117a")), + Pair("langchain", UUID.fromString("cf98d52c-ba5a-4dfd-8ada-c1baebfa6e73")), + Pair("mqtt", UUID.fromString("f3802bc4-5406-4752-9e8d-01e504ca8194")), + Pair("mysql", UUID.fromString("ca81ee7c-3163-4246-af40-094cc31e5e42")), + Pair("bigquery", UUID.fromString("22f6c74f-5699-40ff-833c-4a879ea40133")), + Pair("pubsub", UUID.fromString("356668e2-7e34-47f3-a3b0-67a8a481b692")), + Pair("s3", UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362")), + Pair("oracle", UUID.fromString("3986776d-2319-4de9-8af8-db14c0996e72")), + Pair("sqlite", UUID.fromString("b76be0a6-27dc-4560-95f6-2623da0bd7b6")), + Pair("azure-blob-storage", UUID.fromString("b4c5d105-31fd-4817-96b6-cb923bfc04cb")), + Pair("firebolt", UUID.fromString("18081484-02a5-4662-8dba-b270b582f321")), + Pair("dynamodb", UUID.fromString("8ccd8909-4e99-4141-b48d-4984b70b2d89")), + Pair("xata", UUID.fromString("2a51c92d-0fb4-4e54-94d2-cce631f24d1f")), + Pair("keen", UUID.fromString("81740ce8-d764-4ea7-94df-16bb41de36ae")), + Pair("starburst-galaxy", UUID.fromString("4528e960-6f7b-4412-8555-7e0097e1da17")), + Pair("duckdb", UUID.fromString("94bd199c-2ff0-4aa2-b98e-17f0acb72610")), + Pair("cassandra", UUID.fromString("707456df-6f4f-4ced-b5c6-03f73bcad1c5")), + Pair("clickhouse", UUID.fromString("ce0d828e-1dc4-496c-b122-2da42e637e48")), + Pair("kafka", UUID.fromString("9f760101-60ae-462f-9ee6-b7a9dafd454d")), + Pair("weaviate", UUID.fromString("7b7d7a0d-954c-45a0-bcfc-39a634b97736")), + Pair("typesense", UUID.fromString("36be8dc6-9851-49af-b776-9d4c30e4ab6a")), + Pair("mariadb-columnstore", UUID.fromString("294a4790-429b-40ae-9516-49826b9702e1")), + Pair("pulsar", UUID.fromString("2340cbba-358e-11ec-8d3d-0242ac130203")), + Pair("google-sheets", UUID.fromString("a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a")), + Pair("vertica", UUID.fromString("ca81ee7c-3163-9678-af40-094cc31e5e42")), + Pair("rockset", UUID.fromString("2c9d93a7-9a17-4789-9de9-f46f0097eb70")), + Pair("databend", UUID.fromString("302e4d8e-08d3-4098-acd4-ac67ca365b88")), + Pair("redshift", UUID.fromString("f7a7d195-377f-cf5b-70a5-be6b819019dc")), + Pair("iceberg", UUID.fromString("df65a8f3-9908-451b-aa9b-445462803560")), + Pair("convex", UUID.fromString("3eb4d99c-11fa-4561-a259-fc88e0c2f8f4")), + Pair("aws-datalake", UUID.fromString("99878c90-0fbd-46d3-9d98-ffde879d17fc")), + Pair("s3-glue", UUID.fromString("471e5cab-8ed1-49f3-ba11-79c687784737")), + Pair("mongodb", UUID.fromString("8b746512-8c2e-6ac1-4adc-b59faafd473c")), + Pair("timeplus", UUID.fromString("f70a8ece-351e-4790-b37b-cb790bcd6d54")), + Pair("mssql", UUID.fromString("d4353156-9217-4cad-8dd7-c108fd4f74cf")), + Pair("postgres", UUID.fromString("25c5221d-dce2-4163-ade9-739ef790f503")), + Pair("cumulio", UUID.fromString("e088acb6-9780-4568-880c-54c2dd7f431b")), + Pair("exasol", UUID.fromString("bb6071d9-6f34-4766-bec2-d1d4ed81a653")), + Pair("snowflake", UUID.fromString("424892c4-daac-4491-b35d-c6688ba547ba")), + Pair("firestore", UUID.fromString("27dc7500-6d1b-40b1-8b07-e2f2aea3c9f4")), + Pair("meilisearch", UUID.fromString("af7c921e-5892-4ff2-b6c1-4a5ab258fb7e")), + Pair("teradata", UUID.fromString("58e6f9da-904e-11ed-a1eb-0242ac120002")), + Pair("databricks", UUID.fromString("072d5540-f236-4294-ba7c-ade8fd918496")), + Pair("rabbitmq", UUID.fromString("e06ad785-ad6f-4647-b2e8-3027a5c59454")), + Pair("bigquery-denormalized", UUID.fromString("079d5540-f236-4294-ba7c-ade8fd918496")), + Pair("redpanda", UUID.fromString("825c5ee3-ed9a-4dd1-a2b6-79ed722f7b13")), + Pair("scylla", UUID.fromString("3dc6f384-cd6b-4be3-ad16-a41450899bf0")), + Pair("devmate-cloud", UUID.fromString("eebd85cf-60b2-4af6-9ba0-edeca01437b0")), + Pair("sftp-json", UUID.fromString("e9810f61-4bab-46d2-bb22-edfc902e0644")), + Pair("local-json", UUID.fromString("a625d593-bba5-4a1c-a53d-2d246268a816")), + Pair("selectdb", UUID.fromString("50a559a7-6323-4e33-8aa0-51dfd9dfadac")), + Pair("e2e-test", UUID.fromString("2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537")), +) + +val DEFINITION_ID_TO_DESTINATION_NAME: Map = mapOf( + Pair(UUID.fromString("0fb07be9-7c3b-4336-850d-5efc006152ee"), "r2"), + Pair(UUID.fromString("0eeee7fb-518f-4045-bacc-9619e31c43ea"), "amazon-sqs"), + Pair(UUID.fromString("05c161bf-ca73-4d48-b524-d392be417002"), "doris"), + Pair(UUID.fromString("06ec60c7-7468-45c0-91ac-174f6e1a788b"), "tidb"), + Pair(UUID.fromString("8be1cf83-fde1-477f-a4ad-318d23c9f3c6"), "csv"), + Pair(UUID.fromString("6d1d66d4-26ab-4602-8d32-f85894b04955"), "kinesis"), + Pair(UUID.fromString("d4d3fef9-e319-45c2-881a-bd02ce44cc9f"), "redis"), + Pair(UUID.fromString("2300fdcf-a532-419f-9f24-a014336e7966"), "yugabytedb"), + Pair(UUID.fromString("68f351a7-2745-4bef-ad7f-996b8e51bb8c"), "elasticsearch"), + Pair(UUID.fromString("ca8f6566-e555-4b40-943a-545bf123117a"), "gcs"), + Pair(UUID.fromString("cf98d52c-ba5a-4dfd-8ada-c1baebfa6e73"), "langchain"), + Pair(UUID.fromString("f3802bc4-5406-4752-9e8d-01e504ca8194"), "mqtt"), + Pair(UUID.fromString("ca81ee7c-3163-4246-af40-094cc31e5e42"), "mysql"), + Pair(UUID.fromString("22f6c74f-5699-40ff-833c-4a879ea40133"), "bigquery"), + Pair(UUID.fromString("356668e2-7e34-47f3-a3b0-67a8a481b692"), "pubsub"), + Pair(UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362"), "s3"), + Pair(UUID.fromString("3986776d-2319-4de9-8af8-db14c0996e72"), "oracle"), + Pair(UUID.fromString("b76be0a6-27dc-4560-95f6-2623da0bd7b6"), "sqlite"), + Pair(UUID.fromString("b4c5d105-31fd-4817-96b6-cb923bfc04cb"), "azure-blob-storage"), + Pair(UUID.fromString("18081484-02a5-4662-8dba-b270b582f321"), "firebolt"), + Pair(UUID.fromString("8ccd8909-4e99-4141-b48d-4984b70b2d89"), "dynamodb"), + Pair(UUID.fromString("2a51c92d-0fb4-4e54-94d2-cce631f24d1f"), "xata"), + Pair(UUID.fromString("81740ce8-d764-4ea7-94df-16bb41de36ae"), "keen"), + Pair(UUID.fromString("4528e960-6f7b-4412-8555-7e0097e1da17"), "starburst-galaxy"), + Pair(UUID.fromString("94bd199c-2ff0-4aa2-b98e-17f0acb72610"), "duckdb"), + Pair(UUID.fromString("707456df-6f4f-4ced-b5c6-03f73bcad1c5"), "cassandra"), + Pair(UUID.fromString("ce0d828e-1dc4-496c-b122-2da42e637e48"), "clickhouse"), + Pair(UUID.fromString("9f760101-60ae-462f-9ee6-b7a9dafd454d"), "kafka"), + Pair(UUID.fromString("7b7d7a0d-954c-45a0-bcfc-39a634b97736"), "weaviate"), + Pair(UUID.fromString("36be8dc6-9851-49af-b776-9d4c30e4ab6a"), "typesense"), + Pair(UUID.fromString("294a4790-429b-40ae-9516-49826b9702e1"), "mariadb-columnstore"), + Pair(UUID.fromString("2340cbba-358e-11ec-8d3d-0242ac130203"), "pulsar"), + Pair(UUID.fromString("a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a"), "google-sheets"), + Pair(UUID.fromString("ca81ee7c-3163-9678-af40-094cc31e5e42"), "vertica"), + Pair(UUID.fromString("2c9d93a7-9a17-4789-9de9-f46f0097eb70"), "rockset"), + Pair(UUID.fromString("302e4d8e-08d3-4098-acd4-ac67ca365b88"), "databend"), + Pair(UUID.fromString("f7a7d195-377f-cf5b-70a5-be6b819019dc"), "redshift"), + Pair(UUID.fromString("df65a8f3-9908-451b-aa9b-445462803560"), "iceberg"), + Pair(UUID.fromString("3eb4d99c-11fa-4561-a259-fc88e0c2f8f4"), "convex"), + Pair(UUID.fromString("99878c90-0fbd-46d3-9d98-ffde879d17fc"), "aws-datalake"), + Pair(UUID.fromString("471e5cab-8ed1-49f3-ba11-79c687784737"), "s3-glue"), + Pair(UUID.fromString("8b746512-8c2e-6ac1-4adc-b59faafd473c"), "mongodb"), + Pair(UUID.fromString("f70a8ece-351e-4790-b37b-cb790bcd6d54"), "timeplus"), + Pair(UUID.fromString("d4353156-9217-4cad-8dd7-c108fd4f74cf"), "mssql"), + Pair(UUID.fromString("25c5221d-dce2-4163-ade9-739ef790f503"), "postgres"), + Pair(UUID.fromString("e088acb6-9780-4568-880c-54c2dd7f431b"), "cumulio"), + Pair(UUID.fromString("bb6071d9-6f34-4766-bec2-d1d4ed81a653"), "exasol"), + Pair(UUID.fromString("424892c4-daac-4491-b35d-c6688ba547ba"), "snowflake"), + Pair(UUID.fromString("27dc7500-6d1b-40b1-8b07-e2f2aea3c9f4"), "firestore"), + Pair(UUID.fromString("af7c921e-5892-4ff2-b6c1-4a5ab258fb7e"), "meilisearch"), + Pair(UUID.fromString("58e6f9da-904e-11ed-a1eb-0242ac120002"), "teradata"), + Pair(UUID.fromString("072d5540-f236-4294-ba7c-ade8fd918496"), "databricks"), + Pair(UUID.fromString("e06ad785-ad6f-4647-b2e8-3027a5c59454"), "rabbitmq"), + Pair(UUID.fromString("079d5540-f236-4294-ba7c-ade8fd918496"), "bigquery-denormalized"), + Pair(UUID.fromString("825c5ee3-ed9a-4dd1-a2b6-79ed722f7b13"), "redpanda"), + Pair(UUID.fromString("3dc6f384-cd6b-4be3-ad16-a41450899bf0"), "scylla"), + Pair(UUID.fromString("eebd85cf-60b2-4af6-9ba0-edeca01437b0"), "devmate-cloud"), + Pair(UUID.fromString("e9810f61-4bab-46d2-bb22-edfc902e0644"), "sftp-json"), + Pair(UUID.fromString("a625d593-bba5-4a1c-a53d-2d246268a816"), "local-json"), + Pair(UUID.fromString("50a559a7-6323-4e33-8aa0-51dfd9dfadac"), "selectdb"), + Pair(UUID.fromString("2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537"), "e2e-test"), +) + +val SOURCE_NAME_TO_DEFINITION_ID: Map = mapOf( + Pair("partnerstack", UUID.fromString("d30fb809-6456-484d-8e2c-ee12e0f6888d")), + Pair("dixa", UUID.fromString("0b5c867e-1b12-4d02-ab74-97b2184ff6d7")), + Pair("amazon-sqs", UUID.fromString("983fd355-6bf3-4709-91b5-37afa391eeb6")), + Pair("google-analytics-data-api", UUID.fromString("3cc2eafd-84aa-4dca-93af-322d9dfeec1a")), + Pair("trustpilot", UUID.fromString("d7e23ea6-d741-4314-9209-a33c91a2e945")), + Pair("rocket-chat", UUID.fromString("921d9608-3915-450b-8078-0af18801ea1b")), + Pair("auth0", UUID.fromString("6c504e48-14aa-4221-9a72-19cf5ff1ae78")), + Pair("exchange-rates", UUID.fromString("e2b40e36-aa0e-4bed-b41b-bcea6fa348b1")), + Pair("iterable", UUID.fromString("2e875208-0c0b-4ee4-9e92-1cb3156ea799")), + Pair("airbyte-jenkins-source", UUID.fromString("d6f73702-d7a0-4e95-9758-b0fb1af0bfba")), + Pair("wrike", UUID.fromString("9c13f986-a13b-4988-b808-4705badf71c2")), + Pair("commcare", UUID.fromString("f39208dc-7e1c-48b8-919b-5006360cc27f")), + Pair("instagram", UUID.fromString("6acf6b55-4f1e-4fca-944e-1a3caef8aba8")), + Pair("chartmogul", UUID.fromString("b6604cbd-1b12-4c08-8767-e140d0fb0877")), + Pair("us-census", UUID.fromString("c4cfaeda-c757-489a-8aba-859fb08b6970")), + Pair("chargify", UUID.fromString("9b2d3607-7222-4709-9fa2-c2abdebbdd88")), + Pair("dockerhub", UUID.fromString("72d405a3-56d8-499f-a571-667c03406e43")), + Pair("tiktok-marketing", UUID.fromString("4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35")), + Pair("appsflyer", UUID.fromString("16447954-e6a8-4593-b140-43dea13bc457")), + Pair("merge", UUID.fromString("23240e9e-d14a-43bc-899f-72ea304d1994")), + Pair("orbit", UUID.fromString("95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a")), + Pair("drift", UUID.fromString("445831eb-78db-4b1f-8f1f-0d96ad8739e2")), + Pair("s3", UUID.fromString("69589781-7828-43c5-9f63-8925b1c1ccc2")), + Pair("zendesk-support", UUID.fromString("79c1aa37-dae3-42ae-b333-d1c105477715")), + Pair("github", UUID.fromString("ef69ef6e-aa7f-4af1-a01d-ef775033524e")), + Pair("mailchimp", UUID.fromString("b03a9f3e-22a5-11eb-adc1-0242ac120002")), + Pair("n8n", UUID.fromString("4a961f66-5e99-4430-8320-a73afe52f7a2")), + Pair("paystack", UUID.fromString("193bdcb8-1dd9-48d1-aade-91cadfd74f9b")), + Pair("zendesk-sunshine", UUID.fromString("325e0640-e7b3-4e24-b823-3361008f603f")), + Pair("recreation", UUID.fromString("25d7535d-91e0-466a-aa7f-af81578be277")), + Pair("delighted", UUID.fromString("cc88c43f-6f53-4e8a-8c4d-b284baaf9635")), + Pair("open-exchange-rates", UUID.fromString("77d5ca6b-d345-4dce-ba1e-1935a75778b8")), + Pair("strava", UUID.fromString("7a4327c4-315a-11ec-8d3d-0242ac130003")), + Pair("captain-data", UUID.fromString("fa290790-1dca-43e7-8ced-6a40b2a66099")), + Pair("twilio-taskrouter", UUID.fromString("2446953b-b794-429b-a9b3-c821ba992a48")), + Pair("apple-search-ads", UUID.fromString("e59c8416-c2fa-4bd3-9e95-52677ea281c1")), + Pair("klarna", UUID.fromString("60c24725-00ae-490c-991d-55b78c3197e0")), + Pair("linnworks", UUID.fromString("7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e")), + Pair("sftp", UUID.fromString("a827c52e-791c-4135-a245-e233c5255199")), + Pair("twilio", UUID.fromString("b9dc6155-672e-42ea-b10d-9f1f1fb95ab1")), + Pair("klaviyo", UUID.fromString("95e8cffd-b8c4-4039-968e-d32fb4a69bde")), + Pair("zenefits", UUID.fromString("8baba53d-2fe3-4e33-bc85-210d0eb62884")), + Pair("amazon-seller-partner", UUID.fromString("e55879a8-0ef8-4557-abcf-ab34c53ec460")), + Pair("yotpo", UUID.fromString("18139f00-b1ba-4971-8f80-8387b617cfd8")), + Pair("cockroachdb", UUID.fromString("9fa5862c-da7c-11eb-8d19-0242ac130003")), + Pair("instatus", UUID.fromString("1901024c-0249-45d0-bcac-31a954652927")), + Pair("freshdesk", UUID.fromString("ec4b9503-13cb-48ab-a4ab-6ade4be46567")), + Pair("facebook-pages", UUID.fromString("010eb12f-837b-4685-892d-0a39f76a98f5")), + Pair("dremio", UUID.fromString("d99e9ace-8621-46c2-9144-76ae4751d64b")), + Pair("vantage", UUID.fromString("28ce1fbd-1e15-453f-aa9f-da6c4d928e92")), + Pair("google-search-console", UUID.fromString("eb4c9e00-db83-4d63-a386-39cfa91012a8")), + Pair("plausible", UUID.fromString("603ba446-3d75-41d7-92f3-aba901f8b897")), + Pair("prestashop", UUID.fromString("d60a46d4-709f-4092-a6b7-2457f7d455f5")), + Pair("kyriba", UUID.fromString("547dc08e-ab51-421d-953b-8f3745201a8c")), + Pair("notion", UUID.fromString("6e00b415-b02e-4160-bf02-58176a0ae687")), + Pair("rss", UUID.fromString("0efee448-6948-49e2-b786-17db50647908")), + Pair("gainsight-px", UUID.fromString("0da3b186-8879-4e94-8738-55b48762f1e8")), + Pair("copper", UUID.fromString("44f3002f-2df9-4f6d-b21c-02cd3b47d0dc")), + Pair("postgres", UUID.fromString("decd338e-5647-4c0b-adf4-da0e75f5a750")), + Pair("mssql", UUID.fromString("b5ea17b1-f170-46dc-bc31-cc744ca984c1")), + Pair("talkdesk-explore", UUID.fromString("f00d2cf4-3c28-499a-ba93-b50b6f26359e")), + Pair("airbyte-customer-io-source", UUID.fromString("c47d6804-8b98-449f-970a-5ddb5cb5d7aa")), + Pair("gnews", UUID.fromString("ce38aec4-5a77-439a-be29-9ca44fd4e811")), + Pair("freshsales", UUID.fromString("eca08d79-7b92-4065-b7f3-79c14836ebe7")), + Pair("clockify", UUID.fromString("e71aae8a-5143-11ed-bdc3-0242ac120002")), + Pair("pinterest", UUID.fromString("5cb7e5fe-38c2-11ec-8d3d-0242ac130003")), + Pair("unleash", UUID.fromString("f77914a1-442b-4195-9355-8810a1f4ed3f")), + Pair("insightly", UUID.fromString("38f84314-fe6a-4257-97be-a8dcd942d693")), + Pair("lever-hiring", UUID.fromString("3981c999-bd7d-4afc-849b-e53dea90c948")), + Pair("linkedin-pages", UUID.fromString("af54297c-e8f8-4d63-a00d-a94695acc9d3")), + Pair("sonar-cloud", UUID.fromString("3ab1d7d0-1577-4ab9-bcc4-1ff6a4c2c9f2")), + Pair("babelforce", UUID.fromString("971c3e1e-78a5-411e-ad56-c4052b50876b")), + Pair("polygon-stock-api", UUID.fromString("5807d72f-0abc-49f9-8fa5-ae820007032b")), + Pair("azure-table", UUID.fromString("798ae795-5189-42b6-b64e-3cb91db93338")), + Pair("webflow", UUID.fromString("ef580275-d9a9-48bb-af5e-db0f5855be04")), + Pair("aircall", UUID.fromString("912eb6b7-a893-4a5b-b1c0-36ebbe2de8cd")), + Pair("faker", UUID.fromString("dfd88b22-b603-4c3d-aad7-3701784586b1")), + Pair("ip2whois", UUID.fromString("f23b7b7c-d705-49a3-9042-09add3b104a5")), + Pair("apify-dataset", UUID.fromString("47f17145-fe20-4ef5-a548-e29b048adf84")), + Pair("e2e-test", UUID.fromString("d53f9084-fa6b-4a5a-976c-5b8392f4ad8a")), + Pair("kustomer-singer", UUID.fromString("cd06e646-31bf-4dc8-af48-cbc6530fcad3")), + Pair("tmdb", UUID.fromString("6240848f-f795-45eb-8f5e-c7542822fc03")), + Pair("google-webfonts", UUID.fromString("a68fbcde-b465-4ab3-b2a6-b0590a875835")), + Pair("workramp", UUID.fromString("05b0bce2-4ec4-4534-bb1a-5d0127bd91b7")), + Pair("confluence", UUID.fromString("cf40a7f8-71f8-45ce-a7fa-fca053e4028c")), + Pair("coingecko-coins", UUID.fromString("9cdd4183-d0ba-40c3-aad3-6f46d4103974")), + Pair("pipedrive", UUID.fromString("d8286229-c680-4063-8c59-23b9b391c700")), + Pair("reply-io", UUID.fromString("8cc6537e-f8a6-423c-b960-e927af76116e")), + Pair("sap-fieldglass", UUID.fromString("ec5f3102-fb31-4916-99ae-864faf8e7e25")), + Pair("aws-cloudtrail", UUID.fromString("6ff047c0-f5d5-4ce5-8c81-204a830fa7e1")), + Pair("sendinblue", UUID.fromString("2e88fa20-a2f6-43cc-bba6-98a0a3f244fb")), + Pair("gong", UUID.fromString("32382e40-3b49-4b99-9c5c-4076501914e7")), + Pair("activecampaign", UUID.fromString("9f32dab3-77cb-45a1-9d33-347aa5fbe363")), + Pair("bamboo-hr", UUID.fromString("90916976-a132-4ce9-8bce-82a03dd58788")), + Pair("bigcommerce", UUID.fromString("59c5501b-9f95-411e-9269-7143c939adbd")), + Pair("quickbooks", UUID.fromString("cf9c4355-b171-4477-8f2d-6c5cc5fc8b7e")), + Pair("stripe", UUID.fromString("e094cb9a-26de-4645-8761-65c0c425d1de")), + Pair("mysql", UUID.fromString("435bb9a5-7887-4809-aa58-28c27df0d7ad")), + Pair("appfollow", UUID.fromString("b4375641-e270-41d3-9c20-4f9cecad87a8")), + Pair("linkedin-ads", UUID.fromString("137ece28-5434-455c-8f34-69dc3782f451")), + Pair("sftp-bulk", UUID.fromString("31e3242f-dee7-4cdc-a4b8-8e06c5458517")), + Pair("oura", UUID.fromString("2bf6c581-bec5-4e32-891d-de33036bd631")), + Pair("genesys", UUID.fromString("5ea4459a-8f1a-452a-830f-a65c38cc438d")), + Pair("oracle", UUID.fromString("b39a7370-74c3-45a6-ac3a-380d48520a83")), + Pair("outreach", UUID.fromString("3490c201-5d95-4783-b600-eaf07a4c7787")), + Pair("alloydb", UUID.fromString("1fa90628-2b9e-11ed-a261-0242ac120002")), + Pair("google-pagespeed-insights", UUID.fromString("1e9086ab-ddac-4c1d-aafd-ba43ff575fe4")), + Pair("launchdarkly", UUID.fromString("f96bb511-5e3c-48fc-b408-547953cd81a4")), + Pair("mailersend", UUID.fromString("2707d529-3c04-46eb-9c7e-40d4038df6f7")), + Pair("public-apis", UUID.fromString("a4617b39-3c14-44cd-a2eb-6e720f269235")), + Pair("spacex-api", UUID.fromString("62235e65-af7a-4138-9130-0bda954eb6a8")), + Pair("xkcd", UUID.fromString("80fddd16-17bd-4c0c-bf4a-80df7863fc9d")), + Pair("sendgrid", UUID.fromString("fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87")), + Pair("dynamodb", UUID.fromString("50401137-8871-4c5a-abb7-1f5fda35545a")), + Pair("kyve", UUID.fromString("60a1efcc-c31c-4c63-b508-5b48b6a9f4a6")), + Pair("braintree", UUID.fromString("63cea06f-1c75-458d-88fe-ad48c7cb27fd")), + Pair("adjust", UUID.fromString("d3b7fa46-111b-419a-998a-d7f046f6d66d")), + Pair("clickhouse", UUID.fromString("bad83517-5e54-4a3d-9b53-63e85fbd4d7c")), + Pair("secoda", UUID.fromString("da9fc6b9-8059-4be0-b204-f56e22e4d52d")), + Pair("coinmarketcap", UUID.fromString("239463f5-64bb-4d88-b4bd-18ce673fd572")), + Pair("ashby", UUID.fromString("4e8c9fa0-3634-499b-b948-11581b5c3efa")), + Pair("clickup-api", UUID.fromString("311a7a27-3fb5-4f7e-8265-5e4afe258b66")), + Pair("everhour", UUID.fromString("6babfc42-c734-4ef6-a817-6eca15f0f9b7")), + Pair("freshcaller", UUID.fromString("8a5d48f6-03bb-4038-a942-a8d3f175cca3")), + Pair("pexels-api", UUID.fromString("69d9eb65-8026-47dc-baf1-e4bf67901fd6")), + Pair("hellobaton", UUID.fromString("492b56d1-937c-462e-8076-21ad2031e784")), + Pair("younium", UUID.fromString("9c74c2d7-531a-4ebf-b6d8-6181f805ecdc")), + Pair("google-workspace-admin-reports", UUID.fromString("ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734")), + Pair("redshift", UUID.fromString("e87ffa8e-a3b5-f69c-9076-6011339de1f6")), + Pair("surveycto", UUID.fromString("dd4632f4-15e0-4649-9b71-41719fb1fdee")), + Pair("alpha-vantage", UUID.fromString("db385323-9333-4fec-bec3-9e0ca9326c90")), + Pair("lokalise", UUID.fromString("45e0b135-615c-40ac-b38e-e65b0944197f")), + Pair("intercom", UUID.fromString("d8313939-3782-41b0-be29-b3ca20d8dd3a")), + Pair("mongodb-v2", UUID.fromString("b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e")), + Pair("firebase-realtime-database", UUID.fromString("acb5f973-a565-441e-992f-4946f3e65662")), + Pair("recruitee", UUID.fromString("3b046ac7-d8d3-4eb3-b122-f96b2a16d8a8")), + Pair("orb", UUID.fromString("7f0455fb-4518-4ec0-b7a3-d808bf8081cc")), + Pair("chargebee", UUID.fromString("686473f1-76d9-4994-9cc7-9b13da46147c")), + Pair("harvest", UUID.fromString("fe2b4084-3386-4d3b-9ad6-308f61a6f1e6")), + Pair("hubplanner", UUID.fromString("8097ceb9-383f-42f6-9f92-d3fd4bcc7689")), + Pair("tyntec-sms", UUID.fromString("3c0c3cd1-b3e0-464a-9090-d3ceb5f92346")), + Pair("fullstory", UUID.fromString("263fd456-02d1-4a26-a35e-52ccaedad778")), + Pair("getlago", UUID.fromString("e1a3866b-d3b2-43b6-b6d7-8c1ee4d7f53f")), + Pair("qualaroo", UUID.fromString("b08e4776-d1de-4e80-ab5c-1e51dad934a2")), + Pair("mailjet-sms", UUID.fromString("6ec2acea-7fd1-4378-b403-41a666e0c028")), + Pair("teradata", UUID.fromString("aa8ba6fd-4875-d94e-fc8d-4e1e09aa2503")), + Pair("qonto", UUID.fromString("f7c0b910-5f66-11ed-9b6a-0242ac120002")), + Pair("intruder", UUID.fromString("3d15163b-11d8-412f-b808-795c9b2c3a3a")), + Pair("breezometer", UUID.fromString("7c37685e-8512-4901-addf-9afbef6c0de9")), + Pair("airbyte-pagerduty-source", UUID.fromString("2817b3f0-04e4-4c7a-9f32-7a5e8a83db95")), + Pair("pivotal-tracker", UUID.fromString("d60f5393-f99e-4310-8d05-b1876820f40e")), + Pair("timely", UUID.fromString("bc617b5f-1b9e-4a2d-bebe-782fd454a771")), + Pair("emailoctopus", UUID.fromString("46b25e70-c980-4590-a811-8deaf50ee09f")), + Pair("pocket", UUID.fromString("b0dd65f1-081f-4731-9c51-38e9e6aa0ebf")), + Pair("close-com", UUID.fromString("dfffecb7-9a13-43e9-acdc-b92af7997ca9")), + Pair("k6-cloud", UUID.fromString("e300ece7-b073-43a3-852e-8aff36a57f13")), + Pair("posthog", UUID.fromString("af6d50ee-dddf-4126-a8ee-7faee990774f")), + Pair("cart", UUID.fromString("bb1a6d31-6879-4819-a2bd-3eed299ea8e2")), + Pair("twitter", UUID.fromString("d7fd4f40-5e5a-4b8b-918f-a73077f8c131")), + Pair("weatherstack", UUID.fromString("5db8292c-5f5a-11ed-9b6a-0242ac120002")), + Pair("elasticsearch", UUID.fromString("7cf88806-25f5-4e1a-b422-b2fa9e1b0090")), + Pair("gcs", UUID.fromString("2a8c41ae-8c23-4be0-a73f-2ab10ca1a820")), + Pair("slack", UUID.fromString("c2281cee-86f9-4a86-bb48-d23286b4c7bd")), + Pair("looker", UUID.fromString("00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c")), + Pair("aha", UUID.fromString("81ca39dc-4534-4dd2-b848-b0cfd2c11fce")), + Pair("bigquery", UUID.fromString("bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c")), + Pair("airtable", UUID.fromString("14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212")), + Pair("rki-covid", UUID.fromString("d78e5de0-aa44-4744-aa4f-74c818ccfe19")), + Pair("whisky-hunter", UUID.fromString("e65f84c0-7598-458a-bfac-f770c381ff5d")), + Pair("lemlist", UUID.fromString("789f8e7a-2d28-11ec-8d3d-0242ac130003")), + Pair("punk-api", UUID.fromString("dbe9b7ae-7b46-4e44-a507-02a343cf7230")), + Pair("google-analytics-v4", UUID.fromString("eff3616a-f9c3-11eb-9a03-0242ac130003")), + Pair("azure-blob-storage", UUID.fromString("fdaaba68-4875-4ed9-8fcd-4ae1e0a25093")), + Pair("surveymonkey", UUID.fromString("badc5925-0485-42be-8caa-b34096cb71b5")), + Pair("postmarkapp", UUID.fromString("cde75ca1-1e28-4a0f-85bb-90c546de9f1f")), + Pair("zuora", UUID.fromString("3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5")), + Pair("microsoft-dataverse", UUID.fromString("9220e3de-3b60-4bb2-a46f-046d59ea235a")), + Pair("news-api", UUID.fromString("df38991e-f35b-4af2-996d-36817f614587")), + Pair("youtube-analytics", UUID.fromString("afa734e4-3571-11ec-991a-1e0031268139")), + Pair("callrail", UUID.fromString("dc98a6ad-2dd1-47b6-9529-2ec35820f9c6")), + Pair("courier", UUID.fromString("0541b2cd-2367-4986-b5f1-b79ff55439e4")), + Pair("airbyte-harness-source", UUID.fromString("6fe89830-d04d-401b-aad6-6552ffa5c4af")), + Pair("zapier-supported-storage", UUID.fromString("b8c917bc-7d1b-4828-995f-6726820266d0")), + Pair("kafka", UUID.fromString("d917a47b-8537-4d0d-8c10-36a9928d4265")), + Pair("pendo", UUID.fromString("b1ccb590-e84f-46c0-83a0-2048ccfffdae")), + Pair("zoho-crm", UUID.fromString("4942d392-c7b5-4271-91f9-3b4f4e51eb3e")), + Pair("mixpanel", UUID.fromString("12928b32-bf0a-4f1e-964f-07e12e37153a")), + Pair("vitally", UUID.fromString("6c6d8b0c-db35-4cd1-a7de-0ca8b080f5ac")), + Pair("typeform", UUID.fromString("e7eff203-90bf-43e5-a240-19ea3056c474")), + Pair("smartengage", UUID.fromString("21cc4a17-a011-4485-8a3e-e2341a91ab9f")), + Pair("dv-360", UUID.fromString("1356e1d9-977f-4057-ad4b-65f25329cf61")), + Pair("appstore-singer", UUID.fromString("2af123bf-0aaf-4e0d-9784-cb497f23741a")), + Pair("wikipedia-pageviews", UUID.fromString("87c58f70-6f7a-4f70-aba5-bab1a458f5ba")), + Pair("paypal-transaction", UUID.fromString("d913b0f2-cc51-4e55-a44c-8ba1697b9239")), + Pair("survey-sparrow", UUID.fromString("4a4d887b-0f2d-4b33-ab7f-9b01b9072804")), + Pair("amplitude", UUID.fromString("fa9f58c6-2d03-4237-aaa4-07d75e0c1396")), + Pair("smartsheets", UUID.fromString("374ebc65-6636-4ea0-925c-7d35999a8ffc")), + Pair("file", UUID.fromString("778daa7c-feaf-4db6-96f3-70fd645acc77")), + Pair("gocardless", UUID.fromString("ba15ac82-5c6a-4fb2-bf24-925c23a1180c")), + Pair("recharge", UUID.fromString("45d2e135-2ede-49e1-939f-3e3ec357a65e")), + Pair("statuspage", UUID.fromString("74cbd708-46c3-4512-9c93-abd5c3e9a94d")), + Pair("datadog", UUID.fromString("1cfc30c7-82db-43f4-9fd7-ac1b42312cda")), + Pair("configcat", UUID.fromString("4fd7565c-8b99-439b-80d0-2d965e1d958c")), + Pair("railz", UUID.fromString("9b6cc0c0-da81-4103-bbfd-5279e18a849a")), + Pair("waiteraid", UUID.fromString("03a53b13-794a-4d6b-8544-3b36ed8f3ce4")), + Pair("monday", UUID.fromString("80a54ea2-9959-4040-aac1-eee42423ec9b")), + Pair("snowflake", UUID.fromString("e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2")), + Pair("visma-economic", UUID.fromString("42495935-95de-4f5c-ae08-8fac00f6b308")), + Pair("smaily", UUID.fromString("781f8b1d-4e20-4842-a2c3-cd9b119d65fa")), + Pair("airbyte-victorops-source", UUID.fromString("7e20ce3e-d820-4327-ad7a-88f3927fd97a")), + Pair("zendesk-chat", UUID.fromString("40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4")), + Pair("braze", UUID.fromString("68b9c98e-0747-4c84-b05b-d30b47686725")), + Pair("retently", UUID.fromString("db04ecd1-42e7-4115-9cec-95812905c626")), + Pair("gridly", UUID.fromString("6cbea164-3237-433b-9abb-36d384ee4cbf")), + Pair("yandex-metrica", UUID.fromString("7865dce4-2211-4f6a-88e5-9d0fe161afe7")), + Pair("rd-station-marketing", UUID.fromString("fb141f29-be2a-450b-a4f2-2cd203a00f84")), + Pair("netsuite", UUID.fromString("4f2f093d-ce44-4121-8118-9d13b7bfccd0")), + Pair("google-ads", UUID.fromString("253487c0-2246-43ba-a21f-5116b20a2c50")), + Pair("nasa", UUID.fromString("1a8667d7-7978-43cd-ba4d-d32cbd478971")), + Pair("microsoft-teams", UUID.fromString("eaf50f04-21dd-4620-913b-2a83f5635227")), + Pair("shopify", UUID.fromString("9da77001-af33-4bcd-be46-6252bf9342b9")), + Pair("zendesk-sell", UUID.fromString("982eaa4c-bba1-4cce-a971-06a41f700b8c")), + Pair("persistiq", UUID.fromString("3052c77e-8b91-47e2-97a0-a29a22794b4b")), + Pair("db2", UUID.fromString("447e0381-3780-4b46-bb62-00a4e3c8b8e2")), + Pair("commercetools", UUID.fromString("008b2e26-11a3-11ec-82a8-0242ac130003")), + Pair("metabase", UUID.fromString("c7cb421b-942e-4468-99ee-e369bcabaec5")), + Pair("omnisend", UUID.fromString("e7f0c5e2-4815-48c4-90cf-f47124209835")), + Pair("okta", UUID.fromString("1d4fdb25-64fc-4569-92da-fcdca79a8372")), + Pair("primetric", UUID.fromString("f636c3c6-4077-45ac-b109-19fc62a283c1")), + Pair("trello", UUID.fromString("8da67652-004c-11ec-9a03-0242ac130003")), + Pair("datascope", UUID.fromString("8e1ae2d2-4790-44d3-9d83-75b3fc3940ff")), + Pair("tidb", UUID.fromString("0dad1a35-ccf8-4d03-b73e-6788c00b13ae")), + Pair("fauna", UUID.fromString("3825db3e-c94b-42ac-bd53-b5a9507ace2b")), + Pair("tvmaze-schedule", UUID.fromString("bd14b08f-9f43-400f-b2b6-7248b5c72561")), + Pair("tempo", UUID.fromString("d1aa448b-7c54-498e-ad95-263cbebcd2db")), + Pair("convertkit", UUID.fromString("be9ee02f-6efe-4970-979b-95f797a37188")), + Pair("salesloft", UUID.fromString("41991d12-d4b5-439e-afd0-260a31d4c53f")), + Pair("opsgenie", UUID.fromString("06bdb480-2598-40b8-8b0f-fc2e2d2abdda")), + Pair("salesforce", UUID.fromString("b117307c-14b6-41aa-9422-947e34922962")), + Pair("openweather", UUID.fromString("d8540a80-6120-485d-b7d6-272bca477d9b")), + Pair("shortio", UUID.fromString("2fed2292-5586-480c-af92-9944e39fe12d")), + Pair("zenloop", UUID.fromString("f1e4c7f6-db5c-4035-981f-d35ab4998794")), + Pair("woocommerce", UUID.fromString("2a2552ca-9f78-4c1c-9eb7-4d0dc66d72df")), + Pair("plaid", UUID.fromString("ed799e2b-2158-4c66-8da4-b40fe63bc72a")), + Pair("jira", UUID.fromString("68e63de2-bb83-4c7e-93fa-a8a9051e3993")), + Pair("nytimes", UUID.fromString("0fae6a9a-04eb-44d4-96e1-e02d3dbc1d83")), + Pair("fastbill", UUID.fromString("eb3e9c1c-0467-4eb7-a172-5265e04ccd0a")), + Pair("pardot", UUID.fromString("ad15c7ba-72a7-440b-af15-b9a963dc1a8a")), + Pair("the-guardian-api", UUID.fromString("d42bd69f-6bf0-4d0b-9209-16231af07a92")), + Pair("search-metrics", UUID.fromString("8d7ef552-2c0f-11ec-8d3d-0242ac130003")), + Pair("glassfrog", UUID.fromString("cf8ff320-6272-4faa-89e6-4402dc17e5d5")), + Pair("toggl", UUID.fromString("7e7c844f-2300-4342-b7d3-6dd7992593cd")), + Pair("firebolt", UUID.fromString("6f2ac653-8623-43c4-8950-19218c7caf3d")), + Pair("greenhouse", UUID.fromString("59f1e50a-331f-4f09-b3e8-2e8d4d355f44")), + Pair("zendesk-talk", UUID.fromString("c8630570-086d-4a40-99ae-ea5b18673071")), + Pair("senseforce", UUID.fromString("39de93cb-1511-473e-a673-5cbedb9436af")), + Pair("mailgun", UUID.fromString("5b9cb09e-1003-4f9c-983d-5779d1b2cd51")), + Pair("square", UUID.fromString("77225a51-cd15-4a13-af02-65816bd0ecf4")), + Pair("google-directory", UUID.fromString("d19ae824-e289-4b14-995a-0632eb46d246")), + Pair("bing-ads", UUID.fromString("47f25999-dd5e-4636-8c39-e7cea2453331")), + Pair("marketo", UUID.fromString("9e0556f4-69df-4522-a3fb-03264d36b348")), + Pair("gitlab", UUID.fromString("5e6175e5-68e1-4c17-bff9-56103bbb0d80")), + Pair("hubspot", UUID.fromString("36c891d9-4bd9-43ac-bad2-10e12756272c")), + Pair("google-sheets", UUID.fromString("71607ba1-c0ac-4799-8049-7f4b90dd50f7")), + Pair("snapchat-marketing", UUID.fromString("200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b")), + Pair("my-hours", UUID.fromString("722ba4bf-06ec-45a4-8dd5-72e4a5cf3903")), + Pair("coda", UUID.fromString("27f910fd-f832-4b2e-bcfd-6ab342e434d8")), + Pair("gutendex", UUID.fromString("bff9a277-e01d-420d-81ee-80f28a307318")), + Pair("mailerlite", UUID.fromString("dc3b9003-2432-4e93-a7f4-4620b0f14674")), + Pair("xero", UUID.fromString("6fd1e833-dd6e-45ec-a727-ab917c5be892")), + Pair("freshservice", UUID.fromString("9bb85338-ea95-4c93-b267-6be89125b267")), + Pair("facebook-marketing", UUID.fromString("e7778cfc-e97c-4458-9ecb-b4f2bba8946c")), + Pair("pokeapi", UUID.fromString("6371b14b-bc68-4236-bfbd-468e8df8e968")), + Pair("newsdata", UUID.fromString("60bd11d8-2632-4daa-a688-b47336d32093")), + Pair("convex", UUID.fromString("c332628c-f55c-4017-8222-378cfafda9b2")), + Pair("tplcentral", UUID.fromString("f9b6c538-ee12-42fe-8d4b-0c10f5955417")), + Pair("mailjet-mail", UUID.fromString("56582331-5de2-476b-b913-5798de77bbdf")), + Pair("ringcentral", UUID.fromString("213d69b9-da66-419e-af29-c23142d4af5f")), + Pair("sentry", UUID.fromString("cdaf146a-9b75-49fd-9dd2-9d64a0bb4781")), + Pair("coin-api", UUID.fromString("919984ef-53a2-479b-8ffe-9c1ddb9fc3f3")), + Pair("amazon-ads", UUID.fromString("c6b0a29e-1da9-4512-9002-7bfd0cba2246")), + Pair("zoom", UUID.fromString("cbfd9856-1322-44fb-bcf1-0b39b7a8e92e")), + Pair("asana", UUID.fromString("d0243522-dccf-4978-8ba0-37ed47a0bdbf")), + Pair("workable", UUID.fromString("ef3c99c6-9e90-43c8-9517-926cfd978517")), + Pair("recurly", UUID.fromString("cd42861b-01fc-4658-a8ab-5d11d0510f01")), + Pair("todoist", UUID.fromString("7d272065-c316-4c04-a433-cd4ee143f83e")), + Pair("pypi", UUID.fromString("88ecd3a8-5f5b-11ed-9b6a-0242ac120002")), + Pair("onesignal", UUID.fromString("bb6afd81-87d5-47e3-97c4-e2c2901b1cf8")), + Pair("flexport", UUID.fromString("f95337f1-2ad1-4baf-922f-2ca9152de630")), +) + +val DEFINITION_ID_TO_SOURCE_NAME: Map = mapOf( + Pair(UUID.fromString("d30fb809-6456-484d-8e2c-ee12e0f6888d"), "partnerstack"), + Pair(UUID.fromString("0b5c867e-1b12-4d02-ab74-97b2184ff6d7"), "dixa"), + Pair(UUID.fromString("983fd355-6bf3-4709-91b5-37afa391eeb6"), "amazon-sqs"), + Pair(UUID.fromString("3cc2eafd-84aa-4dca-93af-322d9dfeec1a"), "google-analytics-data-api"), + Pair(UUID.fromString("d7e23ea6-d741-4314-9209-a33c91a2e945"), "trustpilot"), + Pair(UUID.fromString("921d9608-3915-450b-8078-0af18801ea1b"), "rocket-chat"), + Pair(UUID.fromString("6c504e48-14aa-4221-9a72-19cf5ff1ae78"), "auth0"), + Pair(UUID.fromString("e2b40e36-aa0e-4bed-b41b-bcea6fa348b1"), "exchange-rates"), + Pair(UUID.fromString("2e875208-0c0b-4ee4-9e92-1cb3156ea799"), "iterable"), + Pair(UUID.fromString("d6f73702-d7a0-4e95-9758-b0fb1af0bfba"), "airbyte-jenkins-source"), + Pair(UUID.fromString("9c13f986-a13b-4988-b808-4705badf71c2"), "wrike"), + Pair(UUID.fromString("f39208dc-7e1c-48b8-919b-5006360cc27f"), "commcare"), + Pair(UUID.fromString("6acf6b55-4f1e-4fca-944e-1a3caef8aba8"), "instagram"), + Pair(UUID.fromString("b6604cbd-1b12-4c08-8767-e140d0fb0877"), "chartmogul"), + Pair(UUID.fromString("c4cfaeda-c757-489a-8aba-859fb08b6970"), "us-census"), + Pair(UUID.fromString("9b2d3607-7222-4709-9fa2-c2abdebbdd88"), "chargify"), + Pair(UUID.fromString("72d405a3-56d8-499f-a571-667c03406e43"), "dockerhub"), + Pair(UUID.fromString("4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35"), "tiktok-marketing"), + Pair(UUID.fromString("16447954-e6a8-4593-b140-43dea13bc457"), "appsflyer"), + Pair(UUID.fromString("23240e9e-d14a-43bc-899f-72ea304d1994"), "merge"), + Pair(UUID.fromString("95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a"), "orbit"), + Pair(UUID.fromString("445831eb-78db-4b1f-8f1f-0d96ad8739e2"), "drift"), + Pair(UUID.fromString("69589781-7828-43c5-9f63-8925b1c1ccc2"), "s3"), + Pair(UUID.fromString("79c1aa37-dae3-42ae-b333-d1c105477715"), "zendesk-support"), + Pair(UUID.fromString("ef69ef6e-aa7f-4af1-a01d-ef775033524e"), "github"), + Pair(UUID.fromString("b03a9f3e-22a5-11eb-adc1-0242ac120002"), "mailchimp"), + Pair(UUID.fromString("4a961f66-5e99-4430-8320-a73afe52f7a2"), "n8n"), + Pair(UUID.fromString("193bdcb8-1dd9-48d1-aade-91cadfd74f9b"), "paystack"), + Pair(UUID.fromString("325e0640-e7b3-4e24-b823-3361008f603f"), "zendesk-sunshine"), + Pair(UUID.fromString("25d7535d-91e0-466a-aa7f-af81578be277"), "recreation"), + Pair(UUID.fromString("cc88c43f-6f53-4e8a-8c4d-b284baaf9635"), "delighted"), + Pair(UUID.fromString("77d5ca6b-d345-4dce-ba1e-1935a75778b8"), "open-exchange-rates"), + Pair(UUID.fromString("7a4327c4-315a-11ec-8d3d-0242ac130003"), "strava"), + Pair(UUID.fromString("fa290790-1dca-43e7-8ced-6a40b2a66099"), "captain-data"), + Pair(UUID.fromString("2446953b-b794-429b-a9b3-c821ba992a48"), "twilio-taskrouter"), + Pair(UUID.fromString("e59c8416-c2fa-4bd3-9e95-52677ea281c1"), "apple-search-ads"), + Pair(UUID.fromString("60c24725-00ae-490c-991d-55b78c3197e0"), "klarna"), + Pair(UUID.fromString("7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e"), "linnworks"), + Pair(UUID.fromString("a827c52e-791c-4135-a245-e233c5255199"), "sftp"), + Pair(UUID.fromString("b9dc6155-672e-42ea-b10d-9f1f1fb95ab1"), "twilio"), + Pair(UUID.fromString("95e8cffd-b8c4-4039-968e-d32fb4a69bde"), "klaviyo"), + Pair(UUID.fromString("8baba53d-2fe3-4e33-bc85-210d0eb62884"), "zenefits"), + Pair(UUID.fromString("e55879a8-0ef8-4557-abcf-ab34c53ec460"), "amazon-seller-partner"), + Pair(UUID.fromString("18139f00-b1ba-4971-8f80-8387b617cfd8"), "yotpo"), + Pair(UUID.fromString("9fa5862c-da7c-11eb-8d19-0242ac130003"), "cockroachdb"), + Pair(UUID.fromString("1901024c-0249-45d0-bcac-31a954652927"), "instatus"), + Pair(UUID.fromString("ec4b9503-13cb-48ab-a4ab-6ade4be46567"), "freshdesk"), + Pair(UUID.fromString("010eb12f-837b-4685-892d-0a39f76a98f5"), "facebook-pages"), + Pair(UUID.fromString("d99e9ace-8621-46c2-9144-76ae4751d64b"), "dremio"), + Pair(UUID.fromString("28ce1fbd-1e15-453f-aa9f-da6c4d928e92"), "vantage"), + Pair(UUID.fromString("eb4c9e00-db83-4d63-a386-39cfa91012a8"), "google-search-console"), + Pair(UUID.fromString("603ba446-3d75-41d7-92f3-aba901f8b897"), "plausible"), + Pair(UUID.fromString("d60a46d4-709f-4092-a6b7-2457f7d455f5"), "prestashop"), + Pair(UUID.fromString("547dc08e-ab51-421d-953b-8f3745201a8c"), "kyriba"), + Pair(UUID.fromString("6e00b415-b02e-4160-bf02-58176a0ae687"), "notion"), + Pair(UUID.fromString("0efee448-6948-49e2-b786-17db50647908"), "rss"), + Pair(UUID.fromString("0da3b186-8879-4e94-8738-55b48762f1e8"), "gainsight-px"), + Pair(UUID.fromString("44f3002f-2df9-4f6d-b21c-02cd3b47d0dc"), "copper"), + Pair(UUID.fromString("decd338e-5647-4c0b-adf4-da0e75f5a750"), "postgres"), + Pair(UUID.fromString("b5ea17b1-f170-46dc-bc31-cc744ca984c1"), "mssql"), + Pair(UUID.fromString("f00d2cf4-3c28-499a-ba93-b50b6f26359e"), "talkdesk-explore"), + Pair(UUID.fromString("c47d6804-8b98-449f-970a-5ddb5cb5d7aa"), "airbyte-customer-io-source"), + Pair(UUID.fromString("ce38aec4-5a77-439a-be29-9ca44fd4e811"), "gnews"), + Pair(UUID.fromString("eca08d79-7b92-4065-b7f3-79c14836ebe7"), "freshsales"), + Pair(UUID.fromString("e71aae8a-5143-11ed-bdc3-0242ac120002"), "clockify"), + Pair(UUID.fromString("5cb7e5fe-38c2-11ec-8d3d-0242ac130003"), "pinterest"), + Pair(UUID.fromString("f77914a1-442b-4195-9355-8810a1f4ed3f"), "unleash"), + Pair(UUID.fromString("38f84314-fe6a-4257-97be-a8dcd942d693"), "insightly"), + Pair(UUID.fromString("3981c999-bd7d-4afc-849b-e53dea90c948"), "lever-hiring"), + Pair(UUID.fromString("af54297c-e8f8-4d63-a00d-a94695acc9d3"), "linkedin-pages"), + Pair(UUID.fromString("3ab1d7d0-1577-4ab9-bcc4-1ff6a4c2c9f2"), "sonar-cloud"), + Pair(UUID.fromString("971c3e1e-78a5-411e-ad56-c4052b50876b"), "babelforce"), + Pair(UUID.fromString("5807d72f-0abc-49f9-8fa5-ae820007032b"), "polygon-stock-api"), + Pair(UUID.fromString("798ae795-5189-42b6-b64e-3cb91db93338"), "azure-table"), + Pair(UUID.fromString("ef580275-d9a9-48bb-af5e-db0f5855be04"), "webflow"), + Pair(UUID.fromString("912eb6b7-a893-4a5b-b1c0-36ebbe2de8cd"), "aircall"), + Pair(UUID.fromString("dfd88b22-b603-4c3d-aad7-3701784586b1"), "faker"), + Pair(UUID.fromString("f23b7b7c-d705-49a3-9042-09add3b104a5"), "ip2whois"), + Pair(UUID.fromString("47f17145-fe20-4ef5-a548-e29b048adf84"), "apify-dataset"), + Pair(UUID.fromString("d53f9084-fa6b-4a5a-976c-5b8392f4ad8a"), "e2e-test"), + Pair(UUID.fromString("cd06e646-31bf-4dc8-af48-cbc6530fcad3"), "kustomer-singer"), + Pair(UUID.fromString("6240848f-f795-45eb-8f5e-c7542822fc03"), "tmdb"), + Pair(UUID.fromString("a68fbcde-b465-4ab3-b2a6-b0590a875835"), "google-webfonts"), + Pair(UUID.fromString("05b0bce2-4ec4-4534-bb1a-5d0127bd91b7"), "workramp"), + Pair(UUID.fromString("cf40a7f8-71f8-45ce-a7fa-fca053e4028c"), "confluence"), + Pair(UUID.fromString("9cdd4183-d0ba-40c3-aad3-6f46d4103974"), "coingecko-coins"), + Pair(UUID.fromString("d8286229-c680-4063-8c59-23b9b391c700"), "pipedrive"), + Pair(UUID.fromString("8cc6537e-f8a6-423c-b960-e927af76116e"), "reply-io"), + Pair(UUID.fromString("ec5f3102-fb31-4916-99ae-864faf8e7e25"), "sap-fieldglass"), + Pair(UUID.fromString("6ff047c0-f5d5-4ce5-8c81-204a830fa7e1"), "aws-cloudtrail"), + Pair(UUID.fromString("2e88fa20-a2f6-43cc-bba6-98a0a3f244fb"), "sendinblue"), + Pair(UUID.fromString("32382e40-3b49-4b99-9c5c-4076501914e7"), "gong"), + Pair(UUID.fromString("9f32dab3-77cb-45a1-9d33-347aa5fbe363"), "activecampaign"), + Pair(UUID.fromString("90916976-a132-4ce9-8bce-82a03dd58788"), "bamboo-hr"), + Pair(UUID.fromString("59c5501b-9f95-411e-9269-7143c939adbd"), "bigcommerce"), + Pair(UUID.fromString("cf9c4355-b171-4477-8f2d-6c5cc5fc8b7e"), "quickbooks"), + Pair(UUID.fromString("e094cb9a-26de-4645-8761-65c0c425d1de"), "stripe"), + Pair(UUID.fromString("435bb9a5-7887-4809-aa58-28c27df0d7ad"), "mysql"), + Pair(UUID.fromString("b4375641-e270-41d3-9c20-4f9cecad87a8"), "appfollow"), + Pair(UUID.fromString("137ece28-5434-455c-8f34-69dc3782f451"), "linkedin-ads"), + Pair(UUID.fromString("31e3242f-dee7-4cdc-a4b8-8e06c5458517"), "sftp-bulk"), + Pair(UUID.fromString("2bf6c581-bec5-4e32-891d-de33036bd631"), "oura"), + Pair(UUID.fromString("5ea4459a-8f1a-452a-830f-a65c38cc438d"), "genesys"), + Pair(UUID.fromString("b39a7370-74c3-45a6-ac3a-380d48520a83"), "oracle"), + Pair(UUID.fromString("3490c201-5d95-4783-b600-eaf07a4c7787"), "outreach"), + Pair(UUID.fromString("1fa90628-2b9e-11ed-a261-0242ac120002"), "alloydb"), + Pair(UUID.fromString("1e9086ab-ddac-4c1d-aafd-ba43ff575fe4"), "google-pagespeed-insights"), + Pair(UUID.fromString("f96bb511-5e3c-48fc-b408-547953cd81a4"), "launchdarkly"), + Pair(UUID.fromString("2707d529-3c04-46eb-9c7e-40d4038df6f7"), "mailersend"), + Pair(UUID.fromString("a4617b39-3c14-44cd-a2eb-6e720f269235"), "public-apis"), + Pair(UUID.fromString("62235e65-af7a-4138-9130-0bda954eb6a8"), "spacex-api"), + Pair(UUID.fromString("80fddd16-17bd-4c0c-bf4a-80df7863fc9d"), "xkcd"), + Pair(UUID.fromString("fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87"), "sendgrid"), + Pair(UUID.fromString("50401137-8871-4c5a-abb7-1f5fda35545a"), "dynamodb"), + Pair(UUID.fromString("60a1efcc-c31c-4c63-b508-5b48b6a9f4a6"), "kyve"), + Pair(UUID.fromString("63cea06f-1c75-458d-88fe-ad48c7cb27fd"), "braintree"), + Pair(UUID.fromString("d3b7fa46-111b-419a-998a-d7f046f6d66d"), "adjust"), + Pair(UUID.fromString("bad83517-5e54-4a3d-9b53-63e85fbd4d7c"), "clickhouse"), + Pair(UUID.fromString("da9fc6b9-8059-4be0-b204-f56e22e4d52d"), "secoda"), + Pair(UUID.fromString("239463f5-64bb-4d88-b4bd-18ce673fd572"), "coinmarketcap"), + Pair(UUID.fromString("4e8c9fa0-3634-499b-b948-11581b5c3efa"), "ashby"), + Pair(UUID.fromString("311a7a27-3fb5-4f7e-8265-5e4afe258b66"), "clickup-api"), + Pair(UUID.fromString("6babfc42-c734-4ef6-a817-6eca15f0f9b7"), "everhour"), + Pair(UUID.fromString("8a5d48f6-03bb-4038-a942-a8d3f175cca3"), "freshcaller"), + Pair(UUID.fromString("69d9eb65-8026-47dc-baf1-e4bf67901fd6"), "pexels-api"), + Pair(UUID.fromString("492b56d1-937c-462e-8076-21ad2031e784"), "hellobaton"), + Pair(UUID.fromString("9c74c2d7-531a-4ebf-b6d8-6181f805ecdc"), "younium"), + Pair(UUID.fromString("ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734"), "google-workspace-admin-reports"), + Pair(UUID.fromString("e87ffa8e-a3b5-f69c-9076-6011339de1f6"), "redshift"), + Pair(UUID.fromString("dd4632f4-15e0-4649-9b71-41719fb1fdee"), "surveycto"), + Pair(UUID.fromString("db385323-9333-4fec-bec3-9e0ca9326c90"), "alpha-vantage"), + Pair(UUID.fromString("45e0b135-615c-40ac-b38e-e65b0944197f"), "lokalise"), + Pair(UUID.fromString("d8313939-3782-41b0-be29-b3ca20d8dd3a"), "intercom"), + Pair(UUID.fromString("b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e"), "mongodb-v2"), + Pair(UUID.fromString("acb5f973-a565-441e-992f-4946f3e65662"), "firebase-realtime-database"), + Pair(UUID.fromString("3b046ac7-d8d3-4eb3-b122-f96b2a16d8a8"), "recruitee"), + Pair(UUID.fromString("7f0455fb-4518-4ec0-b7a3-d808bf8081cc"), "orb"), + Pair(UUID.fromString("686473f1-76d9-4994-9cc7-9b13da46147c"), "chargebee"), + Pair(UUID.fromString("fe2b4084-3386-4d3b-9ad6-308f61a6f1e6"), "harvest"), + Pair(UUID.fromString("8097ceb9-383f-42f6-9f92-d3fd4bcc7689"), "hubplanner"), + Pair(UUID.fromString("3c0c3cd1-b3e0-464a-9090-d3ceb5f92346"), "tyntec-sms"), + Pair(UUID.fromString("263fd456-02d1-4a26-a35e-52ccaedad778"), "fullstory"), + Pair(UUID.fromString("e1a3866b-d3b2-43b6-b6d7-8c1ee4d7f53f"), "getlago"), + Pair(UUID.fromString("b08e4776-d1de-4e80-ab5c-1e51dad934a2"), "qualaroo"), + Pair(UUID.fromString("6ec2acea-7fd1-4378-b403-41a666e0c028"), "mailjet-sms"), + Pair(UUID.fromString("aa8ba6fd-4875-d94e-fc8d-4e1e09aa2503"), "teradata"), + Pair(UUID.fromString("f7c0b910-5f66-11ed-9b6a-0242ac120002"), "qonto"), + Pair(UUID.fromString("3d15163b-11d8-412f-b808-795c9b2c3a3a"), "intruder"), + Pair(UUID.fromString("7c37685e-8512-4901-addf-9afbef6c0de9"), "breezometer"), + Pair(UUID.fromString("2817b3f0-04e4-4c7a-9f32-7a5e8a83db95"), "airbyte-pagerduty-source"), + Pair(UUID.fromString("d60f5393-f99e-4310-8d05-b1876820f40e"), "pivotal-tracker"), + Pair(UUID.fromString("bc617b5f-1b9e-4a2d-bebe-782fd454a771"), "timely"), + Pair(UUID.fromString("46b25e70-c980-4590-a811-8deaf50ee09f"), "emailoctopus"), + Pair(UUID.fromString("b0dd65f1-081f-4731-9c51-38e9e6aa0ebf"), "pocket"), + Pair(UUID.fromString("dfffecb7-9a13-43e9-acdc-b92af7997ca9"), "close-com"), + Pair(UUID.fromString("e300ece7-b073-43a3-852e-8aff36a57f13"), "k6-cloud"), + Pair(UUID.fromString("af6d50ee-dddf-4126-a8ee-7faee990774f"), "posthog"), + Pair(UUID.fromString("bb1a6d31-6879-4819-a2bd-3eed299ea8e2"), "cart"), + Pair(UUID.fromString("d7fd4f40-5e5a-4b8b-918f-a73077f8c131"), "twitter"), + Pair(UUID.fromString("5db8292c-5f5a-11ed-9b6a-0242ac120002"), "weatherstack"), + Pair(UUID.fromString("7cf88806-25f5-4e1a-b422-b2fa9e1b0090"), "elasticsearch"), + Pair(UUID.fromString("2a8c41ae-8c23-4be0-a73f-2ab10ca1a820"), "gcs"), + Pair(UUID.fromString("c2281cee-86f9-4a86-bb48-d23286b4c7bd"), "slack"), + Pair(UUID.fromString("00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c"), "looker"), + Pair(UUID.fromString("81ca39dc-4534-4dd2-b848-b0cfd2c11fce"), "aha"), + Pair(UUID.fromString("bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c"), "bigquery"), + Pair(UUID.fromString("14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212"), "airtable"), + Pair(UUID.fromString("d78e5de0-aa44-4744-aa4f-74c818ccfe19"), "rki-covid"), + Pair(UUID.fromString("e65f84c0-7598-458a-bfac-f770c381ff5d"), "whisky-hunter"), + Pair(UUID.fromString("789f8e7a-2d28-11ec-8d3d-0242ac130003"), "lemlist"), + Pair(UUID.fromString("dbe9b7ae-7b46-4e44-a507-02a343cf7230"), "punk-api"), + Pair(UUID.fromString("eff3616a-f9c3-11eb-9a03-0242ac130003"), "google-analytics-v4"), + Pair(UUID.fromString("fdaaba68-4875-4ed9-8fcd-4ae1e0a25093"), "azure-blob-storage"), + Pair(UUID.fromString("badc5925-0485-42be-8caa-b34096cb71b5"), "surveymonkey"), + Pair(UUID.fromString("cde75ca1-1e28-4a0f-85bb-90c546de9f1f"), "postmarkapp"), + Pair(UUID.fromString("3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5"), "zuora"), + Pair(UUID.fromString("9220e3de-3b60-4bb2-a46f-046d59ea235a"), "microsoft-dataverse"), + Pair(UUID.fromString("df38991e-f35b-4af2-996d-36817f614587"), "news-api"), + Pair(UUID.fromString("afa734e4-3571-11ec-991a-1e0031268139"), "youtube-analytics"), + Pair(UUID.fromString("dc98a6ad-2dd1-47b6-9529-2ec35820f9c6"), "callrail"), + Pair(UUID.fromString("0541b2cd-2367-4986-b5f1-b79ff55439e4"), "courier"), + Pair(UUID.fromString("6fe89830-d04d-401b-aad6-6552ffa5c4af"), "airbyte-harness-source"), + Pair(UUID.fromString("b8c917bc-7d1b-4828-995f-6726820266d0"), "zapier-supported-storage"), + Pair(UUID.fromString("d917a47b-8537-4d0d-8c10-36a9928d4265"), "kafka"), + Pair(UUID.fromString("b1ccb590-e84f-46c0-83a0-2048ccfffdae"), "pendo"), + Pair(UUID.fromString("4942d392-c7b5-4271-91f9-3b4f4e51eb3e"), "zoho-crm"), + Pair(UUID.fromString("12928b32-bf0a-4f1e-964f-07e12e37153a"), "mixpanel"), + Pair(UUID.fromString("6c6d8b0c-db35-4cd1-a7de-0ca8b080f5ac"), "vitally"), + Pair(UUID.fromString("e7eff203-90bf-43e5-a240-19ea3056c474"), "typeform"), + Pair(UUID.fromString("21cc4a17-a011-4485-8a3e-e2341a91ab9f"), "smartengage"), + Pair(UUID.fromString("1356e1d9-977f-4057-ad4b-65f25329cf61"), "dv-360"), + Pair(UUID.fromString("2af123bf-0aaf-4e0d-9784-cb497f23741a"), "appstore-singer"), + Pair(UUID.fromString("87c58f70-6f7a-4f70-aba5-bab1a458f5ba"), "wikipedia-pageviews"), + Pair(UUID.fromString("d913b0f2-cc51-4e55-a44c-8ba1697b9239"), "paypal-transaction"), + Pair(UUID.fromString("4a4d887b-0f2d-4b33-ab7f-9b01b9072804"), "survey-sparrow"), + Pair(UUID.fromString("fa9f58c6-2d03-4237-aaa4-07d75e0c1396"), "amplitude"), + Pair(UUID.fromString("374ebc65-6636-4ea0-925c-7d35999a8ffc"), "smartsheets"), + Pair(UUID.fromString("778daa7c-feaf-4db6-96f3-70fd645acc77"), "file"), + Pair(UUID.fromString("ba15ac82-5c6a-4fb2-bf24-925c23a1180c"), "gocardless"), + Pair(UUID.fromString("45d2e135-2ede-49e1-939f-3e3ec357a65e"), "recharge"), + Pair(UUID.fromString("74cbd708-46c3-4512-9c93-abd5c3e9a94d"), "statuspage"), + Pair(UUID.fromString("1cfc30c7-82db-43f4-9fd7-ac1b42312cda"), "datadog"), + Pair(UUID.fromString("4fd7565c-8b99-439b-80d0-2d965e1d958c"), "configcat"), + Pair(UUID.fromString("9b6cc0c0-da81-4103-bbfd-5279e18a849a"), "railz"), + Pair(UUID.fromString("03a53b13-794a-4d6b-8544-3b36ed8f3ce4"), "waiteraid"), + Pair(UUID.fromString("80a54ea2-9959-4040-aac1-eee42423ec9b"), "monday"), + Pair(UUID.fromString("e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2"), "snowflake"), + Pair(UUID.fromString("42495935-95de-4f5c-ae08-8fac00f6b308"), "visma-economic"), + Pair(UUID.fromString("781f8b1d-4e20-4842-a2c3-cd9b119d65fa"), "smaily"), + Pair(UUID.fromString("7e20ce3e-d820-4327-ad7a-88f3927fd97a"), "airbyte-victorops-source"), + Pair(UUID.fromString("40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4"), "zendesk-chat"), + Pair(UUID.fromString("68b9c98e-0747-4c84-b05b-d30b47686725"), "braze"), + Pair(UUID.fromString("db04ecd1-42e7-4115-9cec-95812905c626"), "retently"), + Pair(UUID.fromString("6cbea164-3237-433b-9abb-36d384ee4cbf"), "gridly"), + Pair(UUID.fromString("7865dce4-2211-4f6a-88e5-9d0fe161afe7"), "yandex-metrica"), + Pair(UUID.fromString("fb141f29-be2a-450b-a4f2-2cd203a00f84"), "rd-station-marketing"), + Pair(UUID.fromString("4f2f093d-ce44-4121-8118-9d13b7bfccd0"), "netsuite"), + Pair(UUID.fromString("253487c0-2246-43ba-a21f-5116b20a2c50"), "google-ads"), + Pair(UUID.fromString("1a8667d7-7978-43cd-ba4d-d32cbd478971"), "nasa"), + Pair(UUID.fromString("eaf50f04-21dd-4620-913b-2a83f5635227"), "microsoft-teams"), + Pair(UUID.fromString("9da77001-af33-4bcd-be46-6252bf9342b9"), "shopify"), + Pair(UUID.fromString("982eaa4c-bba1-4cce-a971-06a41f700b8c"), "zendesk-sell"), + Pair(UUID.fromString("3052c77e-8b91-47e2-97a0-a29a22794b4b"), "persistiq"), + Pair(UUID.fromString("447e0381-3780-4b46-bb62-00a4e3c8b8e2"), "db2"), + Pair(UUID.fromString("008b2e26-11a3-11ec-82a8-0242ac130003"), "commercetools"), + Pair(UUID.fromString("c7cb421b-942e-4468-99ee-e369bcabaec5"), "metabase"), + Pair(UUID.fromString("e7f0c5e2-4815-48c4-90cf-f47124209835"), "omnisend"), + Pair(UUID.fromString("1d4fdb25-64fc-4569-92da-fcdca79a8372"), "okta"), + Pair(UUID.fromString("f636c3c6-4077-45ac-b109-19fc62a283c1"), "primetric"), + Pair(UUID.fromString("8da67652-004c-11ec-9a03-0242ac130003"), "trello"), + Pair(UUID.fromString("8e1ae2d2-4790-44d3-9d83-75b3fc3940ff"), "datascope"), + Pair(UUID.fromString("0dad1a35-ccf8-4d03-b73e-6788c00b13ae"), "tidb"), + Pair(UUID.fromString("3825db3e-c94b-42ac-bd53-b5a9507ace2b"), "fauna"), + Pair(UUID.fromString("bd14b08f-9f43-400f-b2b6-7248b5c72561"), "tvmaze-schedule"), + Pair(UUID.fromString("d1aa448b-7c54-498e-ad95-263cbebcd2db"), "tempo"), + Pair(UUID.fromString("be9ee02f-6efe-4970-979b-95f797a37188"), "convertkit"), + Pair(UUID.fromString("41991d12-d4b5-439e-afd0-260a31d4c53f"), "salesloft"), + Pair(UUID.fromString("06bdb480-2598-40b8-8b0f-fc2e2d2abdda"), "opsgenie"), + Pair(UUID.fromString("b117307c-14b6-41aa-9422-947e34922962"), "salesforce"), + Pair(UUID.fromString("d8540a80-6120-485d-b7d6-272bca477d9b"), "openweather"), + Pair(UUID.fromString("2fed2292-5586-480c-af92-9944e39fe12d"), "shortio"), + Pair(UUID.fromString("f1e4c7f6-db5c-4035-981f-d35ab4998794"), "zenloop"), + Pair(UUID.fromString("2a2552ca-9f78-4c1c-9eb7-4d0dc66d72df"), "woocommerce"), + Pair(UUID.fromString("ed799e2b-2158-4c66-8da4-b40fe63bc72a"), "plaid"), + Pair(UUID.fromString("68e63de2-bb83-4c7e-93fa-a8a9051e3993"), "jira"), + Pair(UUID.fromString("0fae6a9a-04eb-44d4-96e1-e02d3dbc1d83"), "nytimes"), + Pair(UUID.fromString("eb3e9c1c-0467-4eb7-a172-5265e04ccd0a"), "fastbill"), + Pair(UUID.fromString("ad15c7ba-72a7-440b-af15-b9a963dc1a8a"), "pardot"), + Pair(UUID.fromString("d42bd69f-6bf0-4d0b-9209-16231af07a92"), "the-guardian-api"), + Pair(UUID.fromString("8d7ef552-2c0f-11ec-8d3d-0242ac130003"), "search-metrics"), + Pair(UUID.fromString("cf8ff320-6272-4faa-89e6-4402dc17e5d5"), "glassfrog"), + Pair(UUID.fromString("7e7c844f-2300-4342-b7d3-6dd7992593cd"), "toggl"), + Pair(UUID.fromString("6f2ac653-8623-43c4-8950-19218c7caf3d"), "firebolt"), + Pair(UUID.fromString("59f1e50a-331f-4f09-b3e8-2e8d4d355f44"), "greenhouse"), + Pair(UUID.fromString("c8630570-086d-4a40-99ae-ea5b18673071"), "zendesk-talk"), + Pair(UUID.fromString("39de93cb-1511-473e-a673-5cbedb9436af"), "senseforce"), + Pair(UUID.fromString("5b9cb09e-1003-4f9c-983d-5779d1b2cd51"), "mailgun"), + Pair(UUID.fromString("77225a51-cd15-4a13-af02-65816bd0ecf4"), "square"), + Pair(UUID.fromString("d19ae824-e289-4b14-995a-0632eb46d246"), "google-directory"), + Pair(UUID.fromString("47f25999-dd5e-4636-8c39-e7cea2453331"), "bing-ads"), + Pair(UUID.fromString("9e0556f4-69df-4522-a3fb-03264d36b348"), "marketo"), + Pair(UUID.fromString("5e6175e5-68e1-4c17-bff9-56103bbb0d80"), "gitlab"), + Pair(UUID.fromString("36c891d9-4bd9-43ac-bad2-10e12756272c"), "hubspot"), + Pair(UUID.fromString("71607ba1-c0ac-4799-8049-7f4b90dd50f7"), "google-sheets"), + Pair(UUID.fromString("200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b"), "snapchat-marketing"), + Pair(UUID.fromString("722ba4bf-06ec-45a4-8dd5-72e4a5cf3903"), "my-hours"), + Pair(UUID.fromString("27f910fd-f832-4b2e-bcfd-6ab342e434d8"), "coda"), + Pair(UUID.fromString("bff9a277-e01d-420d-81ee-80f28a307318"), "gutendex"), + Pair(UUID.fromString("dc3b9003-2432-4e93-a7f4-4620b0f14674"), "mailerlite"), + Pair(UUID.fromString("6fd1e833-dd6e-45ec-a727-ab917c5be892"), "xero"), + Pair(UUID.fromString("9bb85338-ea95-4c93-b267-6be89125b267"), "freshservice"), + Pair(UUID.fromString("e7778cfc-e97c-4458-9ecb-b4f2bba8946c"), "facebook-marketing"), + Pair(UUID.fromString("6371b14b-bc68-4236-bfbd-468e8df8e968"), "pokeapi"), + Pair(UUID.fromString("60bd11d8-2632-4daa-a688-b47336d32093"), "newsdata"), + Pair(UUID.fromString("c332628c-f55c-4017-8222-378cfafda9b2"), "convex"), + Pair(UUID.fromString("f9b6c538-ee12-42fe-8d4b-0c10f5955417"), "tplcentral"), + Pair(UUID.fromString("56582331-5de2-476b-b913-5798de77bbdf"), "mailjet-mail"), + Pair(UUID.fromString("213d69b9-da66-419e-af29-c23142d4af5f"), "ringcentral"), + Pair(UUID.fromString("cdaf146a-9b75-49fd-9dd2-9d64a0bb4781"), "sentry"), + Pair(UUID.fromString("919984ef-53a2-479b-8ffe-9c1ddb9fc3f3"), "coin-api"), + Pair(UUID.fromString("c6b0a29e-1da9-4512-9002-7bfd0cba2246"), "amazon-ads"), + Pair(UUID.fromString("cbfd9856-1322-44fb-bcf1-0b39b7a8e92e"), "zoom"), + Pair(UUID.fromString("d0243522-dccf-4978-8ba0-37ed47a0bdbf"), "asana"), + Pair(UUID.fromString("ef3c99c6-9e90-43c8-9517-926cfd978517"), "workable"), + Pair(UUID.fromString("cd42861b-01fc-4658-a8ab-5d11d0510f01"), "recurly"), + Pair(UUID.fromString("7d272065-c316-4c04-a433-cd4ee143f83e"), "todoist"), + Pair(UUID.fromString("88ecd3a8-5f5b-11ed-9b6a-0242ac120002"), "pypi"), + Pair(UUID.fromString("bb6afd81-87d5-47e3-97c4-e2c2901b1cf8"), "onesignal"), + Pair(UUID.fromString("f95337f1-2ad1-4baf-922f-2ca9152de630"), "flexport"), +) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/PaginationMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/PaginationMapper.kt new file mode 100644 index 00000000000..50a4b8f5dc2 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/PaginationMapper.kt @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.micronaut.core.util.CollectionUtils +import io.micronaut.http.uri.UriBuilder +import java.util.Optional +import java.util.UUID + +/** + * Pagination mapper for easily creating pagination previous/next strings. + */ +object PaginationMapper { + var LIMIT = "limit" + var OFFSET = "offset" + + /** + * Base URI builder we need to create. + * + * @param publicApiHost public API host + * @param path path of the endpoint for which you want to build pagination URLs + * @return URL builder for the base URL + */ + fun getBuilder(publicApiHost: String, path: String): UriBuilder { + return UriBuilder.of(publicApiHost).path(path) + } + + /** + * Get next offset. + * + * @param collection collection of items we just got + * @param limit current limit + * @param offset current offset + * @return offset or null which indicates we have no next offset to provide + */ + fun getNextOffset(collection: Collection<*>, limit: Int, offset: Int): Optional { + // If we have no more entries or we had no entries this page, just return empty - no next URL + return if (CollectionUtils.isEmpty(collection) || collection.size < limit) { + Optional.empty() + } else Optional.of(offset + limit) + } + + /** + * Gets previous offset based on passed in limit and offset. + * + * @param limit current limit + * @param offset current offset + * @return new "previous" offset + */ + fun getPreviousOffset(limit: Int, offset: Int): Int { + val previousOffset = offset - limit + return Math.max(previousOffset, 0) + } + + /** + * Get the full next URL. + * + * @param collection list of things we just got from the endpoint. + * @param limit current limit + * @param offset current offset + * @param uriBuilder the URL builder created from getBuilder + * @return a String URL that can be put into the response. + */ + fun getNextUrl(collection: Collection<*>, limit: Int, offset: Int, uriBuilder: UriBuilder): String { + val nextOffset = getNextOffset(collection, limit, offset) + return if (nextOffset.isPresent) { + uriBuilder.queryParam(LIMIT, limit) + .replaceQueryParam(OFFSET, nextOffset.get()).toString() + } else { + "" + } + } + + /** + * Get the full previous URL. + * + * @param limit current limit + * @param offset current offset + * @param uriBuilder the URL builder created from getBuilder + * @return a String URL that can be put into the response. + */ + fun getPreviousUrl(limit: Int, offset: Int, uriBuilder: UriBuilder): String { + return if (offset != 0) { + uriBuilder.queryParam(LIMIT, limit).replaceQueryParam( + OFFSET, + getPreviousOffset(limit, offset), + ).toString() + } else { + "" + } + } + + /** + * Helper to turn a list of UUIDs into a pagination acceptable string. + * + * @param uuids uuids to stringify + */ + fun uuidListToQueryString(uuids: List): String { + return java.lang.String.join(",", uuids.stream().map { obj: UUID -> obj.toString() }.toList()) + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourceDefinitionSpecificationReadMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourceDefinitionSpecificationReadMapper.kt new file mode 100644 index 00000000000..0985ec55f63 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourceDefinitionSpecificationReadMapper.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.api.client.model.generated.SourceDefinitionSpecificationRead + +/** + * Mappers that help convert models from the config api to models from the public api. + */ +object SourceDefinitionSpecificationReadMapper { + /** + * Converts a SourceDefinitionRead object from the config api to an object without including job + * info. + * + * @param sourceDefinitionSpecificationRead Output of a connection create/get from config api + * @return SourceDefinitionSpecificationRead Response object with everything except jobInfo + */ + fun from(sourceDefinitionSpecificationRead: SourceDefinitionSpecificationRead): SourceDefinitionSpecificationRead { + sourceDefinitionSpecificationRead.setJobInfo(null) + return sourceDefinitionSpecificationRead + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourceReadMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourceReadMapper.kt new file mode 100644 index 00000000000..f809ca6aa89 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourceReadMapper.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.SourceResponse +import io.airbyte.api.client.model.generated.SourceRead + +/** + * Mappers that help convert models from the config api to models from the public api. + */ +object SourceReadMapper { + /** + * Converts a SourceRead object from the config api to a SourceResponse. + * + * @param sourceRead Output of a source create/get from config api + * @return SourceResponse Response object with source details + */ + fun from(sourceRead: SourceRead): SourceResponse { + val sourceResponse = SourceResponse() + sourceResponse.sourceId = sourceRead.sourceId + sourceResponse.name = sourceRead.name + sourceResponse.sourceType = DEFINITION_ID_TO_SOURCE_NAME.get(sourceRead.sourceDefinitionId) + sourceResponse.workspaceId = sourceRead.workspaceId + sourceResponse.configuration = sourceRead.connectionConfiguration + return sourceResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourcesResponseMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourcesResponseMapper.kt new file mode 100644 index 00000000000..e4b522bd1fa --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/SourcesResponseMapper.kt @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.SourcesResponse +import io.airbyte.api.client.model.generated.SourceRead +import io.airbyte.api.client.model.generated.SourceReadList +import io.airbyte.api.server.constants.INCLUDE_DELETED +import io.airbyte.api.server.constants.SOURCES_PATH +import io.airbyte.api.server.constants.WORKSPACE_IDS +import java.util.UUID +import java.util.function.Function + +/** + * Maps config API SourceReadList to SourcesResponse. + */ +object SourcesResponseMapper { + /** + * Converts a SourceReadList object from the config api to a SourcesResponse object. + * + * @param sourceReadList Output of a source list from config api + * @param workspaceIds workspaceIds we wanted to list + * @param includeDeleted did we include deleted workspaces or not? + * @param limit Number of responses to be outputted + * @param offset Offset of the pagination + * @param apiHost Host url e.g. api.airbyte.com + * @return SourcesResponse List of SourceResponse along with a next and previous https requests + */ + fun from( + sourceReadList: SourceReadList, + workspaceIds: List, + includeDeleted: Boolean, + limit: Int, + offset: Int, + apiHost: String, + ): SourcesResponse { + val uriBuilder = PaginationMapper.getBuilder(apiHost, SOURCES_PATH) + .queryParam(WORKSPACE_IDS, PaginationMapper.uuidListToQueryString(workspaceIds)) + .queryParam(INCLUDE_DELETED, includeDeleted) + val sourcesResponse = SourcesResponse() + sourcesResponse.setNext(PaginationMapper.getNextUrl(sourceReadList.sources, limit, offset, uriBuilder)) + sourcesResponse.setPrevious(PaginationMapper.getPreviousUrl(limit, offset, uriBuilder)) + sourcesResponse.setData( + sourceReadList.sources.stream() + .map(Function { obj: SourceRead? -> SourceReadMapper.from(obj!!) }) + .toList(), + ) + return sourcesResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/WorkspaceResponseMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/WorkspaceResponseMapper.kt new file mode 100644 index 00000000000..c4503037888 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/WorkspaceResponseMapper.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.GeographyEnum +import io.airbyte.airbyte_api.model.generated.WorkspaceResponse +import io.airbyte.api.client.model.generated.WorkspaceRead + +/** + * Mappers that help convert models from the config api to models from the public api. + */ +object WorkspaceResponseMapper { + /** + * Converts a WorkspaceRead object from the config api to an object with just a WorkspaceId. + * + * @param workspaceRead Output of a workspace create/get from config api + * @return WorkspaceResponse Response object which contains the workspace id + */ + fun from(workspaceRead: WorkspaceRead): WorkspaceResponse { + val workspaceResponse = WorkspaceResponse() + workspaceResponse.workspaceId = workspaceRead.workspaceId + workspaceResponse.name = workspaceRead.name + if (workspaceRead.defaultGeography != null) { + workspaceResponse.dataResidency = GeographyEnum.fromValue(workspaceRead.defaultGeography!!.toString()) + } + return workspaceResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/WorkspacesResponseMapper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/WorkspacesResponseMapper.kt new file mode 100644 index 00000000000..89305c3afe1 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/mappers/WorkspacesResponseMapper.kt @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.mappers + +import io.airbyte.airbyte_api.model.generated.WorkspacesResponse +import io.airbyte.api.client.model.generated.WorkspaceRead +import io.airbyte.api.client.model.generated.WorkspaceReadList +import io.airbyte.api.server.constants.INCLUDE_DELETED +import io.airbyte.api.server.constants.WORKSPACES_PATH +import io.airbyte.api.server.constants.WORKSPACE_IDS +import io.airbyte.api.server.mappers.WorkspaceResponseMapper.from +import java.util.UUID + +/** + * Maps config API WorkspaceReadList to WorkspacesResponse. + */ +object WorkspacesResponseMapper { + /** + * Converts a WorkspaceReadList object from the config api to a WorkspacesResponse object. + * + * @param workspaceReadList Output of a workspace list from config api + * @param workspaceIds workspaceIds we wanted to list + * @param includeDeleted did we include deleted workspaces or not? + * @param limit Number of responses to be outputted + * @param offset Offset of the pagination + * @param apiHost Host url e.g. api.airbyte.com + * @return SourcesResponse List of SourceResponse along with a next and previous https requests + */ + fun from( + workspaceReadList: WorkspaceReadList, + workspaceIds: List, + includeDeleted: Boolean, + limit: Int, + offset: Int, + apiHost: String, + ): WorkspacesResponse { + val uriBuilder = PaginationMapper.getBuilder(apiHost, WORKSPACES_PATH) + .queryParam(WORKSPACE_IDS, PaginationMapper.uuidListToQueryString(workspaceIds)) + .queryParam(INCLUDE_DELETED, includeDeleted) + val workspacesResponse = WorkspacesResponse() + workspacesResponse.next = PaginationMapper.getNextUrl(workspaceReadList.workspaces, limit, offset, uriBuilder) + workspacesResponse.previous = PaginationMapper.getPreviousUrl(limit, offset, uriBuilder) + workspacesResponse.data = workspaceReadList.workspaces.stream() + .map { obj: WorkspaceRead? -> from(obj!!) } + .toList() + return workspacesResponse + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/BadRequestProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/BadRequestProblem.kt new file mode 100644 index 00000000000..a057347c797 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/BadRequestProblem.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * Bad request problem for generic 400 errors. + */ +class BadRequestProblem(message: String?) : AbstractThrowableProblem( + TYPE, + TITLE, + HttpStatusType(HttpStatus.BAD_REQUEST), + message, +) { + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors#bad-request") + private const val TITLE = "bad-request" + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt new file mode 100644 index 00000000000..284822c62a3 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.airbyte_api.model.generated.ConnectionSyncModeEnum +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI +import javax.validation.Valid + +/** + * Thrown when a configuration for a connection is not valid. + */ +class ConnectionConfigurationProblem private constructor(message: String) : AbstractThrowableProblem( + TYPE, + TITLE, + HttpStatusType(HttpStatus.BAD_REQUEST), + "The body of the request contains an invalid connection configuration. $message", +) { + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors") + private const val TITLE = "bad-request" + fun handleSyncModeProblem( + connectionSyncMode: @Valid ConnectionSyncModeEnum?, + streamName: String, + validSyncModes: Set, + ): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "Cannot set sync mode to $connectionSyncMode for stream $streamName. Valid sync modes are: $validSyncModes", + ) + } + + fun invalidStreamName(validStreamNames: Collection): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "Invalid stream found. The list of valid streams include: $validStreamNames.", + ) + } + + fun duplicateStream(streamName: String): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem("Duplicate stream found in configuration for: $streamName.") + } + + fun sourceDefinedCursorFieldProblem(streamName: String, cursorField: List): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "Cursor Field " + cursorField + " is already defined by source for stream: " + streamName + + ". Do not include a cursor field configuration for this stream.", + ) + } + + fun missingCursorField(streamName: String): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "No default cursor field for stream: $streamName. Please include a cursor field configuration for this stream.", + ) + } + + fun invalidCursorField(streamName: String, validFields: List?>): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "Invalid cursor field for stream: $streamName. The list of valid cursor fields include: $validFields.", + ) + } + + fun missingPrimaryKey(streamName: String): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "No default primary key for stream: $streamName. Please include a primary key configuration for this stream.", + ) + } + + fun primaryKeyAlreadyDefined(streamName: String): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "Primary key for stream: $streamName is already pre-defined. Please do NOT include a primary key configuration for this stream.", + ) + } + + fun invalidPrimaryKey(streamName: String, validFields: List?>): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "Invalid cursor field for stream: $streamName. The list of valid primary keys fields: $validFields.", + ) + } + + fun invalidCronExpression(cronExpression: String): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "The cron expression " + cronExpression + + " is not valid or is less than the one hour minimum. The seconds and minutes values cannot be `*`.", + ) + } + + fun missingCronExpression(): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem("Missing cron expression in the schedule.") + } + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ForbiddenProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ForbiddenProblem.kt new file mode 100644 index 00000000000..c57cca288f3 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ForbiddenProblem.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * For throwing forbidden errors. + */ +class ForbiddenProblem(message: String?) : AbstractThrowableProblem( + TYPE, + TITLE, + HttpStatusType(HttpStatus.FORBIDDEN), + message, +) { + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors#forbidden") + private const val TITLE = "forbidden" + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/InvalidApiKeyProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/InvalidApiKeyProblem.kt new file mode 100644 index 00000000000..63dcb4334b9 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/InvalidApiKeyProblem.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * Thrown when API key in Authorization header is invalid. + */ +class InvalidApiKeyProblem : AbstractThrowableProblem() { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors#invalid-api-key") + private val TITLE = "invalid-api-key" + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/InvalidRedirectUrlProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/InvalidRedirectUrlProblem.kt new file mode 100644 index 00000000000..09beac3cda4 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/InvalidRedirectUrlProblem.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * Problem to indicate an invalid URL format. + */ +class InvalidRedirectUrlProblem(url: String?) : AbstractThrowableProblem( + TYPE, + TITLE, + HttpStatusType(HttpStatus.UNPROCESSABLE_ENTITY), + String.format("Redirect URL format not understood: %s", url), +) { + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors#invalid-redirect-url") + private const val TITLE = "invalid-redirect-url" + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ResourceNotFoundProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ResourceNotFoundProblem.kt new file mode 100644 index 00000000000..10d3c938450 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ResourceNotFoundProblem.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * Thrown when user attempts to interact with a resource that can't be found in the db. + */ +class ResourceNotFoundProblem(resourceId: String?) : AbstractThrowableProblem( + TYPE, + TITLE, + HttpStatusType(HttpStatus.BAD_REQUEST), + String.format("Could not find a resource for: %s", resourceId), +) { + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors#resource-not-found") + private const val TITLE = "resource-not-found" + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/SyncConflictProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/SyncConflictProblem.kt new file mode 100644 index 00000000000..7453092cf73 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/SyncConflictProblem.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * Thrown when a user attempts to start a sync run while one is already running. + */ +class SyncConflictProblem(message: String?) : AbstractThrowableProblem( + TYPE, + TITLE, + HttpStatusType(HttpStatus.CONFLICT), + message, +) { + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors#try-again-later") + private const val TITLE = "try-again-later" + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnexpectedProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnexpectedProblem.kt new file mode 100644 index 00000000000..19f32bf214a --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnexpectedProblem.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * UnexpectedProblem allows us to pass through the httpStatus from configApi. + */ +class UnexpectedProblem : AbstractThrowableProblem { + private var statusCode: Int + + constructor(httpStatus: HttpStatus) : super( + TYPE, + TITLE, + HttpStatusType(httpStatus), + "An unexpected problem has occurred. If this is an error that needs to be addressed, please submit a pull request or github issue.", + ) { + statusCode = httpStatus.code + } + + constructor(httpStatus: HttpStatus, message: String?) : super( + TYPE, + TITLE, + HttpStatusType(httpStatus), + message, + ) { + statusCode = httpStatus.code + } + + constructor(title: String?, httpStatus: HttpStatus, message: String?) : super( + TYPE, + title, + HttpStatusType(httpStatus), + message, + ) { + statusCode = httpStatus.code + } + + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors") + private const val TITLE = "unexpected-problem" + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnknownValueProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnknownValueProblem.kt new file mode 100644 index 00000000000..1d95aa0bcb1 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnknownValueProblem.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * Thrown when user sends an invalid input. + */ +class UnknownValueProblem(value: String?) : AbstractThrowableProblem( + TYPE, + TITLE, + HttpStatusType(HttpStatus.BAD_REQUEST), + String.format("Submitted value could not be found: %s", value), +) { + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors") + private const val TITLE = "value-not-found" + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnprocessableEntityProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnprocessableEntityProblem.kt new file mode 100644 index 00000000000..24956c9a659 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/UnprocessableEntityProblem.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.problems + +import io.airbyte.api.server.constants.API_DOC_URL +import io.micronaut.http.HttpStatus +import io.micronaut.problem.HttpStatusType +import org.zalando.problem.AbstractThrowableProblem +import org.zalando.problem.Exceptional +import java.io.Serial +import java.net.URI + +/** + * Thrown when request body cannot be processed correctly. + */ +class UnprocessableEntityProblem : AbstractThrowableProblem { + constructor() : super( + TYPE, + TITLE, + HttpStatusType(HttpStatus.UNPROCESSABLE_ENTITY), + "The body of the request was not understood", + ) + + constructor(message: String?) : super( + TYPE, + TITLE, + HttpStatusType(HttpStatus.UNPROCESSABLE_ENTITY), + message, + ) + + companion object { + @Serial + private val serialVersionUID = 1L + private val TYPE = URI.create("$API_DOC_URL/reference/errors#unprocessable-entity") + private const val TITLE = "unprocessable-entity" + } + + override fun getCause(): Exceptional? { + return null + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/ConnectionsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/ConnectionsController.kt deleted file mode 100644 index eb0a190de64..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/ConnectionsController.kt +++ /dev/null @@ -1,37 +0,0 @@ -package io.airbyte.api.server.routes - -import io.airbyte.airbyte_api.generated.ConnectionsApi -import io.airbyte.airbyte_api.model.generated.ConnectionCreateRequest -import io.airbyte.airbyte_api.model.generated.ConnectionPatchRequest -import io.airbyte.api.server.services.ConnectionService -import io.micronaut.http.annotation.Controller -import java.util.UUID -import javax.ws.rs.core.Response - -@Controller("/v1/connections") -open class ConnectionsController(connectionService: ConnectionService) : ConnectionsApi { - override fun createConnection(connectionCreateRequest: ConnectionCreateRequest?): Response { - TODO("Not yet implemented") - } - - override fun deleteConnection(connectionId: UUID?): Response { - TODO("Not yet implemented") - } - - override fun getConnection(connectionId: UUID?): Response { - TODO("Not yet implemented") - } - - override fun listConnections( - workspaceIds: MutableList?, - includeDeleted: Boolean?, - limit: Int?, - offset: Int?, - ): Response { - TODO("Not yet implemented") - } - - override fun patchConnection(connectionId: UUID?, connectionPatchRequest: ConnectionPatchRequest?): Response { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Destinations.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Destinations.kt deleted file mode 100644 index e32ac922ca1..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Destinations.kt +++ /dev/null @@ -1,37 +0,0 @@ -package io.airbyte.api.server.routes - -import io.airbyte.airbyte_api.generated.DestinationsApi -import io.airbyte.airbyte_api.model.generated.DestinationCreateRequest -import io.airbyte.airbyte_api.model.generated.DestinationPatchRequest -import io.airbyte.airbyte_api.model.generated.DestinationPutRequest -import io.airbyte.api.server.services.DestinationService -import io.micronaut.http.annotation.Controller -import java.util.UUID -import javax.ws.rs.core.Response - -@Controller("/v1/destinations") -open class Destinations(private var destinationService: DestinationService) : DestinationsApi { - override fun createDestination(destinationCreateRequest: DestinationCreateRequest?): Response { - TODO("Not yet implemented") - } - - override fun deleteDestination(destinationId: UUID?): Response { - TODO("Not yet implemented") - } - - override fun getDestination(destinationId: UUID?): Response { - TODO("Not yet implemented") - } - - override fun listDestinations(workspaceIds: MutableList?, includeDeleted: Boolean?, limit: Int?, offset: Int?): Response { - TODO("Not yet implemented") - } - - override fun patchDestination(destinationId: UUID?, destinationPatchRequest: DestinationPatchRequest?): Response { - TODO("Not yet implemented") - } - - override fun putDestination(destinationId: UUID?, destinationPutRequest: DestinationPutRequest?): Response { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Jobs.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Jobs.kt deleted file mode 100644 index a6c5b56a9c9..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Jobs.kt +++ /dev/null @@ -1,28 +0,0 @@ -package io.airbyte.api.server.routes - -import io.airbyte.airbyte_api.generated.JobsApi -import io.airbyte.airbyte_api.model.generated.JobCreateRequest -import io.airbyte.airbyte_api.model.generated.JobTypeEnum -import io.airbyte.api.server.services.JobService -import io.micronaut.http.annotation.Controller -import java.util.UUID -import javax.ws.rs.core.Response - -@Controller("/v1/jobs") -open class Jobs(private var jobService: JobService) : JobsApi { - override fun cancelJob(jobId: Long?): Response { - TODO("Not yet implemented") - } - - override fun createJob(jobCreateRequest: JobCreateRequest?): Response { - TODO("Not yet implemented") - } - - override fun getJob(jobId: Long?): Response { - TODO("Not yet implemented") - } - - override fun listJobs(connectionId: UUID?, limit: Int?, offset: Int?, jobType: JobTypeEnum?, workspaceIds: MutableList?): Response { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Sources.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Sources.kt deleted file mode 100644 index b861bf8fc8d..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Sources.kt +++ /dev/null @@ -1,42 +0,0 @@ -package io.airbyte.api.server.routes - -import io.airbyte.airbyte_api.generated.SourcesApi -import io.airbyte.airbyte_api.model.generated.InitiateOauthRequest -import io.airbyte.airbyte_api.model.generated.SourceCreateRequest -import io.airbyte.airbyte_api.model.generated.SourcePatchRequest -import io.airbyte.airbyte_api.model.generated.SourcePutRequest -import io.airbyte.api.server.services.SourceService -import io.micronaut.http.annotation.Controller -import java.util.UUID -import javax.ws.rs.core.Response - -@Controller("/v1/sources") -class Sources(sourceService: SourceService) : SourcesApi { - override fun createSource(sourceCreateRequest: SourceCreateRequest?): Response { - TODO("Not yet implemented") - } - - override fun deleteSource(sourceId: UUID?): Response { - TODO("Not yet implemented") - } - - override fun getSource(sourceId: UUID?): Response { - TODO("Not yet implemented") - } - - override fun initiateOAuth(initiateOauthRequest: InitiateOauthRequest?): Response { - TODO("Not yet implemented") - } - - override fun listSources(workspaceIds: MutableList?, includeDeleted: Boolean?, limit: Int?, offset: Int?): Response { - TODO("Not yet implemented") - } - - override fun patchSource(sourceId: UUID?, sourcePatchRequest: SourcePatchRequest?): Response { - TODO("Not yet implemented") - } - - override fun putSource(sourceId: UUID?, sourcePutRequest: SourcePutRequest?): Response { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Workspaces.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Workspaces.kt deleted file mode 100644 index 380ac54c05f..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/routes/Workspaces.kt +++ /dev/null @@ -1,40 +0,0 @@ -package io.airbyte.api.server.routes - -import io.airbyte.airbyte_api.generated.WorkspacesApi -import io.airbyte.airbyte_api.model.generated.WorkspaceCreateRequest -import io.airbyte.airbyte_api.model.generated.WorkspaceOAuthCredentialsRequest -import io.airbyte.airbyte_api.model.generated.WorkspaceUpdateRequest -import io.airbyte.api.server.services.WorkspaceService -import io.micronaut.http.annotation.Controller -import java.util.UUID -import javax.ws.rs.core.Response - -@Controller("/v1/workspaces") -class Workspaces(workspaceService: WorkspaceService) : WorkspacesApi { - override fun createOrUpdateWorkspaceOAuthCredentials( - workspaceId: UUID?, - workspaceOAuthCredentialsRequest: WorkspaceOAuthCredentialsRequest?, - ): Response { - TODO("Not yet implemented") - } - - override fun createWorkspace(workspaceCreateRequest: WorkspaceCreateRequest?): Response { - TODO("Not yet implemented") - } - - override fun deleteWorkspace(workspaceId: UUID?): Response { - TODO("Not yet implemented") - } - - override fun getWorkspace(workspaceId: UUID?): Response { - TODO("Not yet implemented") - } - - override fun listWorkspaces(workspaceIds: MutableList?, includeDeleted: Boolean?, limit: Int?, offset: Int?): Response { - TODO("Not yet implemented") - } - - override fun updateWorkspace(workspaceId: UUID?, workspaceUpdateRequest: WorkspaceUpdateRequest?): Response { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/ConnectionService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/ConnectionService.kt index e0a5285746d..2c03434c1d7 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/ConnectionService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/ConnectionService.kt @@ -1,38 +1,259 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server.services +import com.fasterxml.jackson.databind.ObjectMapper +import io.airbyte.airbyte_api.model.generated.ConnectionCreateRequest +import io.airbyte.airbyte_api.model.generated.ConnectionPatchRequest import io.airbyte.airbyte_api.model.generated.ConnectionResponse import io.airbyte.airbyte_api.model.generated.ConnectionsResponse -import io.airbyte.api.model.generated.ConnectionCreate -import io.airbyte.api.model.generated.ConnectionUpdate +import io.airbyte.airbyte_api.model.generated.SourceResponse +import io.airbyte.api.client.model.generated.AirbyteCatalog +import io.airbyte.api.client.model.generated.ConnectionCreate +import io.airbyte.api.client.model.generated.ConnectionIdRequestBody +import io.airbyte.api.client.model.generated.ConnectionRead +import io.airbyte.api.client.model.generated.ConnectionReadList +import io.airbyte.api.client.model.generated.ConnectionUpdate +import io.airbyte.api.client.model.generated.ListConnectionsForWorkspacesRequestBody +import io.airbyte.api.client.model.generated.Pagination +import io.airbyte.api.server.constants.HTTP_RESPONSE_BODY_DEBUG_MESSAGE +import io.airbyte.api.server.errorHandlers.ConfigClientErrorHandler +import io.airbyte.api.server.forwardingClient.ConfigApiClient +import io.airbyte.api.server.mappers.ConnectionCreateMapper +import io.airbyte.api.server.mappers.ConnectionReadMapper +import io.airbyte.api.server.mappers.ConnectionUpdateMapper +import io.airbyte.api.server.mappers.ConnectionsResponseMapper +import io.airbyte.api.server.problems.UnexpectedProblem +import io.micronaut.context.annotation.Secondary +import io.micronaut.context.annotation.Value +import io.micronaut.http.HttpResponse +import io.micronaut.http.HttpStatus +import io.micronaut.http.client.exceptions.HttpClientResponseException +import jakarta.inject.Singleton +import org.slf4j.LoggerFactory import java.util.Collections +import java.util.Objects import java.util.UUID interface ConnectionService { + fun createConnection( - connectionCreate: ConnectionCreate, - endpointUserInfo: String, + connectionCreateRequest: ConnectionCreateRequest, + catalogId: UUID, + configuredCatalog: AirbyteCatalog, + workspaceId: UUID, + userInfo: String?, ): ConnectionResponse fun deleteConnection( connectionId: UUID, - endpointUserInfo: String, - ): ConnectionResponse + userInfo: String?, + ) fun getConnection( connectionId: UUID, - endpointUserInfo: String, + userInfo: String?, ): ConnectionResponse fun updateConnection( - connectionUpdate: ConnectionUpdate, - endpointUserInfo: String, + connectionId: UUID, + connectionPatchRequest: ConnectionPatchRequest, + catalogId: UUID, + configuredCatalog: AirbyteCatalog, + workspaceId: UUID, + userInfo: String?, ): ConnectionResponse fun listConnectionsForWorkspaces( - workspaceIds: MutableList = Collections.emptyList(), + workspaceIds: List = Collections.emptyList(), limit: Int = 20, offset: Int = 0, includeDeleted: Boolean = false, - endpointUserInfo: String, + + userInfo: String?, ): ConnectionsResponse } + +@Singleton +@Secondary +class ConnectionServiceImpl( + private val configApiClient: ConfigApiClient, + private val userService: UserService, + private val sourceService: SourceService, +) : ConnectionService { + + companion object { + private val log = LoggerFactory.getLogger(ConnectionServiceImpl::class.java) + } + + @Value("\${airbyte.api.host}") + var publicApiHost: String? = null + + /** + * Creates a connection. + */ + override fun createConnection( + connectionCreateRequest: ConnectionCreateRequest, + catalogId: UUID, + configuredCatalog: AirbyteCatalog, + workspaceId: UUID, + userInfo: String?, + + ): ConnectionResponse { + val connectionCreateOss: ConnectionCreate = + ConnectionCreateMapper.from(connectionCreateRequest, catalogId, configuredCatalog) + + // this is kept as a string to easily parse the error response to determine if a source or a + // destination id is invalid + + // this is kept as a string to easily parse the error response to determine if a source or a + // destination id is invalid + val response = try { + configApiClient.createConnection(connectionCreateOss, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for createConnection: ", e) + e.response as HttpResponse + } + + ConfigClientErrorHandler.handleCreateConnectionError(response, connectionCreateRequest) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + + val objectMapper = ObjectMapper() + return try { + ConnectionReadMapper.from( + objectMapper.readValue( + Objects.requireNonNull(response.body()), + ConnectionRead::class.java, + ), + workspaceId, + ) + } catch (e: Exception) { + log.error("Error while reading response and converting to Connection read: ", e) + throw UnexpectedProblem(HttpStatus.INTERNAL_SERVER_ERROR) + } + } + + /** + * Deletes a connection by ID. + */ + override fun deleteConnection(connectionId: UUID, userInfo: String?) { + val connectionIdRequestBody = ConnectionIdRequestBody().connectionId(connectionId) + val response = try { + configApiClient.deleteConnection(connectionIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for connection delete: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, connectionId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + } + + /** + * Gets a connection by ID. + */ + override fun getConnection(connectionId: UUID, userInfo: String?): ConnectionResponse { + val connectionIdRequestBody = ConnectionIdRequestBody() + connectionIdRequestBody.connectionId = connectionId + + val response = try { + configApiClient.getConnection(connectionIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for getConnection: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, connectionId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + + // get workspace id from source id + val sourceResponse: SourceResponse = sourceService.getSource(response.body()!!.sourceId, userInfo) + + return ConnectionReadMapper.from( + response.body()!!, + sourceResponse.workspaceId, + ) + } + + /** + * Updates a connection with patch semantics. + */ + override fun updateConnection( + connectionId: UUID, + connectionPatchRequest: ConnectionPatchRequest, + catalogId: UUID, + configuredCatalog: AirbyteCatalog, + workspaceId: UUID, + userInfo: String?, + ): ConnectionResponse { + val connectionUpdate: ConnectionUpdate = + ConnectionUpdateMapper.from( + connectionId, + connectionPatchRequest, + catalogId, + configuredCatalog, + ) + + // this is kept as a string to easily parse the error response to determine if a source or a + // destination id is invalid + val response = try { + configApiClient.updateConnection(connectionUpdate, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for updateConnection: ", e) + e.response as HttpResponse + } + + ConfigClientErrorHandler.handleError(response, connectionId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + + val objectMapper = ObjectMapper() + return try { + ConnectionReadMapper.from( + objectMapper.readValue( + Objects.requireNonNull(response.body()), + ConnectionRead::class.java, + ), + workspaceId, + ) + } catch (e: java.lang.Exception) { + log.error("Error while reading response and converting to Connection read: ", e) + throw UnexpectedProblem(HttpStatus.INTERNAL_SERVER_ERROR) + } + } + + /** + * Lists connections for a set of workspace IDs or all workspaces if none are provided. + */ + override fun listConnectionsForWorkspaces( + workspaceIds: List, + limit: Int, + offset: Int, + includeDeleted: Boolean, + userInfo: String?, + ): ConnectionsResponse { + val pagination: Pagination = Pagination().pageSize(limit).rowOffset(offset) + val workspaceIdsToQuery = workspaceIds.ifEmpty { userService.getAllWorkspaceIdsForUser(null, userInfo) } + + val listConnectionsForWorkspacesRequestBody = ListConnectionsForWorkspacesRequestBody() + .workspaceIds(workspaceIdsToQuery) + .includeDeleted(includeDeleted) + .pagination(pagination) + + val response = try { + configApiClient.listConnectionsForWorkspaces(listConnectionsForWorkspacesRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for listConnectionsForWorkspaces: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, workspaceIds.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return ConnectionsResponseMapper.from( + response.body()!!, + workspaceIds, + includeDeleted, + limit, + offset, + publicApiHost!!, + ) + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/DestinationService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/DestinationService.kt index 6b18b3aa402..01dac7c5e79 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/DestinationService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/DestinationService.kt @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server.services import io.airbyte.airbyte_api.model.generated.DestinationCreateRequest @@ -5,38 +9,242 @@ import io.airbyte.airbyte_api.model.generated.DestinationPatchRequest import io.airbyte.airbyte_api.model.generated.DestinationPutRequest import io.airbyte.airbyte_api.model.generated.DestinationResponse import io.airbyte.airbyte_api.model.generated.DestinationsResponse +import io.airbyte.api.client.model.generated.DestinationCreate +import io.airbyte.api.client.model.generated.DestinationDefinitionIdWithWorkspaceId +import io.airbyte.api.client.model.generated.DestinationDefinitionSpecificationRead +import io.airbyte.api.client.model.generated.DestinationIdRequestBody +import io.airbyte.api.client.model.generated.DestinationRead +import io.airbyte.api.client.model.generated.DestinationReadList +import io.airbyte.api.client.model.generated.DestinationSyncMode +import io.airbyte.api.client.model.generated.DestinationUpdate +import io.airbyte.api.client.model.generated.ListResourcesForWorkspacesRequestBody +import io.airbyte.api.client.model.generated.Pagination +import io.airbyte.api.client.model.generated.PartialDestinationUpdate +import io.airbyte.api.server.constants.HTTP_RESPONSE_BODY_DEBUG_MESSAGE +import io.airbyte.api.server.errorHandlers.ConfigClientErrorHandler +import io.airbyte.api.server.forwardingClient.ConfigApiClient +import io.airbyte.api.server.helpers.getIdFromName +import io.airbyte.api.server.mappers.DESTINATION_NAME_TO_DEFINITION_ID +import io.airbyte.api.server.mappers.DestinationReadMapper +import io.airbyte.api.server.mappers.DestinationsResponseMapper +import io.micronaut.context.annotation.Secondary +import io.micronaut.context.annotation.Value +import io.micronaut.http.HttpResponse +import io.micronaut.http.client.exceptions.HttpClientResponseException +import jakarta.inject.Singleton +import org.slf4j.LoggerFactory import java.util.UUID -import javax.validation.constraints.NotBlank interface DestinationService { + fun createDestination( - destinationCreateRequest: @NotBlank DestinationCreateRequest?, - destinationDefinitionId: @NotBlank UUID?, - userInfo: String, + destinationCreateRequest: DestinationCreateRequest, + destinationDefinitionId: UUID, + userInfo: String?, ): DestinationResponse - fun getDestination(destinationId: @NotBlank UUID?, userInfo: String): DestinationResponse + fun getDestination(destinationId: UUID, userInfo: String?): DestinationResponse fun updateDestination( destinationId: UUID, destinationPutRequest: DestinationPutRequest, - userInfo: String, + userInfo: String?, ): DestinationResponse fun partialUpdateDestination( destinationId: UUID, destinationPatchRequest: DestinationPatchRequest, - userInfo: String, + userInfo: String?, ): DestinationResponse - fun deleteDestination(connectionId: @NotBlank UUID, userInfo: String) + fun deleteDestination(connectionId: UUID, userInfo: String?) fun listDestinationsForWorkspaces( + workspaceIds: List, + includeDeleted: Boolean = false, + limit: Int = 20, + offset: Int = 0, + + userInfo: String?, + ): DestinationsResponse? + + fun getDestinationSyncModes(destinationId: UUID, userInfo: String?): List + fun getDestinationSyncModes(destinationResponse: DestinationResponse, userInfo: String?): List +} + +@Singleton +@Secondary +class DestinationServiceImpl(private val configApiClient: ConfigApiClient, private val userService: UserService) : DestinationService { + + companion object { + private val log = LoggerFactory.getLogger(DestinationServiceImpl::class.java) + } + + @Value("\${airbyte.api.host}") + var publicApiHost: String? = null + + /** + * Creates a destination. + */ + override fun createDestination( + destinationCreateRequest: DestinationCreateRequest, + destinationDefinitionId: UUID, + userInfo: String?, + ): DestinationResponse { + val destinationCreateOss = DestinationCreate() + destinationCreateOss.name = destinationCreateRequest.name + destinationCreateOss.destinationDefinitionId = destinationDefinitionId + destinationCreateOss.workspaceId = destinationCreateRequest.workspaceId + destinationCreateOss.connectionConfiguration = destinationCreateRequest.configuration + + val response = try { + configApiClient.createDestination(destinationCreateOss, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for createDestination: ", e) + e.response as HttpResponse + } + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + ConfigClientErrorHandler.handleError(response, destinationCreateRequest.workspaceId.toString()) + return DestinationReadMapper.from(response.body()!!) + } + + /** + * Gets a destination by ID. + */ + override fun getDestination(destinationId: UUID, userInfo: String?): DestinationResponse { + val destinationIdRequestBody = DestinationIdRequestBody() + destinationIdRequestBody.destinationId = destinationId + + log.info("getDestination request: $destinationIdRequestBody") + val response = try { + configApiClient.getDestination(destinationIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for getDestination: ", e) + e.response as HttpResponse + } + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + ConfigClientErrorHandler.handleError(response, destinationId.toString()) + return DestinationReadMapper.from(response.body()!!) + } + + /** + * Updates a destination by ID. + */ + override fun updateDestination(destinationId: UUID, destinationPutRequest: DestinationPutRequest, userInfo: String?): DestinationResponse { + val destinationUpdate = DestinationUpdate() + .destinationId(destinationId) + .connectionConfiguration(destinationPutRequest.configuration) + .name(destinationPutRequest.name) + + val response = try { + configApiClient.updateDestination(destinationUpdate, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for updateDestination: ", e) + e.response as HttpResponse + } + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + ConfigClientErrorHandler.handleError(response, destinationId.toString()) + return DestinationReadMapper.from(response.body()!!) + } + + /** + * Partially updates a destination with patch semantics. + */ + override fun partialUpdateDestination( + destinationId: UUID, + destinationPatchRequest: DestinationPatchRequest, + userInfo: String?, + ): DestinationResponse { + val partialDestinationUpdate = PartialDestinationUpdate() + .destinationId(destinationId) + .connectionConfiguration(destinationPatchRequest.configuration) + .name(destinationPatchRequest.name) + + val response = try { + configApiClient.partialUpdateDestination(partialDestinationUpdate, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for partialUpdateDestination: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, destinationId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return DestinationReadMapper.from(response.body()!!) + } + + /** + * Deletes updates a destination by ID. + */ + override fun deleteDestination(connectionId: UUID, userInfo: String?) { + val destinationIdRequestBody = DestinationIdRequestBody().destinationId(connectionId) + val response = try { + configApiClient.deleteDestination(destinationIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for destination delete: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, connectionId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + } + + /** + * Lists destinations by workspace IDs or all destinations if no workspace IDs are provided. + */ + override fun listDestinationsForWorkspaces( workspaceIds: List, includeDeleted: Boolean, limit: Int, offset: Int, - authorization: String, - userInfo: String, - ): DestinationsResponse? + userInfo: String?, + ): DestinationsResponse { + val pagination: Pagination = Pagination().pageSize(limit).rowOffset(offset) + val workspaceIdsToQuery = workspaceIds.ifEmpty { userService.getAllWorkspaceIdsForUser(null, userInfo) } + val listResourcesForWorkspacesRequestBody = ListResourcesForWorkspacesRequestBody() + listResourcesForWorkspacesRequestBody.includeDeleted = includeDeleted + listResourcesForWorkspacesRequestBody.pagination = pagination + listResourcesForWorkspacesRequestBody.workspaceIds = workspaceIdsToQuery + + val response = try { + configApiClient.listDestinationsForWorkspaces(listResourcesForWorkspacesRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for listWorkspaces: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, workspaceIds.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return DestinationsResponseMapper.from( + response.body()!!, + workspaceIds, + includeDeleted, + limit, + offset, + publicApiHost!!, + ) + } + + override fun getDestinationSyncModes(destinationId: UUID, userInfo: String?): List { + val destinationResponse: DestinationResponse = getDestination(destinationId, userInfo) + return getDestinationSyncModes(destinationResponse, userInfo) + } + + override fun getDestinationSyncModes( + destinationResponse: DestinationResponse, + userInfo: String?, + ): List { + val destinationDefinitionId: UUID = + getIdFromName(DESTINATION_NAME_TO_DEFINITION_ID, destinationResponse.destinationType) + val destinationDefinitionIdWithWorkspaceId = DestinationDefinitionIdWithWorkspaceId() + destinationDefinitionIdWithWorkspaceId.destinationDefinitionId = destinationDefinitionId + destinationDefinitionIdWithWorkspaceId.workspaceId = destinationResponse.workspaceId + var response: HttpResponse + try { + response = configApiClient.getDestinationSpec(destinationDefinitionIdWithWorkspaceId, userInfo) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for getDestinationSpec: ", e) + response = e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, destinationResponse.destinationId.toString()) + val destinationDefinitionSpecificationRead = response.body.get() + return destinationDefinitionSpecificationRead.supportedDestinationSyncModes!! + } } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt index e1aac56108d..a4533fd098a 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt @@ -1,34 +1,229 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server.services import io.airbyte.airbyte_api.model.generated.JobResponse import io.airbyte.airbyte_api.model.generated.JobTypeEnum import io.airbyte.airbyte_api.model.generated.JobsResponse +import io.airbyte.api.client.model.generated.ConnectionIdRequestBody +import io.airbyte.api.client.model.generated.JobConfigType +import io.airbyte.api.client.model.generated.JobIdRequestBody +import io.airbyte.api.client.model.generated.JobInfoRead +import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody +import io.airbyte.api.client.model.generated.JobListRequestBody +import io.airbyte.api.client.model.generated.JobReadList +import io.airbyte.api.client.model.generated.Pagination +import io.airbyte.api.server.constants.HTTP_RESPONSE_BODY_DEBUG_MESSAGE +import io.airbyte.api.server.errorHandlers.ConfigClientErrorHandler +import io.airbyte.api.server.filters.JobsFilter +import io.airbyte.api.server.forwardingClient.ConfigApiClient +import io.airbyte.api.server.mappers.JobResponseMapper +import io.airbyte.api.server.mappers.JobsResponseMapper +import io.airbyte.api.server.problems.UnexpectedProblem +import io.airbyte.api.server.problems.UnprocessableEntityProblem +import io.micronaut.context.annotation.Secondary +import io.micronaut.context.annotation.Value +import io.micronaut.http.HttpResponse +import io.micronaut.http.HttpStatus +import io.micronaut.http.client.exceptions.HttpClientResponseException +import io.micronaut.http.client.exceptions.ReadTimeoutException +import jakarta.inject.Singleton +import org.slf4j.LoggerFactory +import java.util.Objects import java.util.UUID -import javax.validation.constraints.NotBlank interface JobService { - fun sync(connectionId: @NotBlank UUID, userInfo: String): JobResponse - fun reset(connectionId: @NotBlank UUID, userInfo: String): JobResponse + fun sync(connectionId: UUID, userInfo: String?): JobResponse + + fun reset(connectionId: UUID, userInfo: String?): JobResponse - fun cancelJob(jobId: Long, userInfo: String): JobResponse + fun cancelJob(jobId: Long, userInfo: String?): JobResponse - fun getJobInfoWithoutLogs(jobId: @NotBlank Long, userInfo: String): JobResponse + fun getJobInfoWithoutLogs(jobId: Long, userInfo: String?): JobResponse fun getJobList( - connectionId: @NotBlank UUID, - jobType: JobTypeEnum, - limit: Int, - offset: Int, - userInfo: String, + connectionId: UUID, + jobsFilter: JobsFilter, + userInfo: String?, ): JobsResponse fun getJobList( workspaceIds: List, - jobType: JobTypeEnum, - limit: Int, - offset: Int, - authorization: String, - userInfo: String, + jobsFilter: JobsFilter, + + userInfo: String?, ): JobsResponse } + +@Singleton +@Secondary +class JobServiceImpl(val configApiClient: ConfigApiClient, val userService: UserService) : JobService { + + companion object { + private val log = LoggerFactory.getLogger(JobServiceImpl::class.java) + } + + @Value("\${airbyte.api.host}") + var publicApiHost: String? = null + + /** + * Starts a sync job for the given connection ID. + */ + override fun sync(connectionId: UUID, userInfo: String?): JobResponse { + val connectionIdRequestBody = ConnectionIdRequestBody().connectionId(connectionId) + val response = try { + configApiClient.sync(connectionIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for job sync: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, connectionId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return JobResponseMapper.from(Objects.requireNonNull(response.body())) + } + + /** + * Starts a reset job for the given connection ID. + */ + override fun reset(connectionId: UUID, userInfo: String?): JobResponse { + val connectionIdRequestBody = ConnectionIdRequestBody().connectionId(connectionId) + val response = try { + configApiClient.reset(connectionIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for job reset: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, connectionId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return JobResponseMapper.from(Objects.requireNonNull(response.body())) + } + + /** + * Cancels a job by ID. + */ + override fun cancelJob(jobId: Long, userInfo: String?): JobResponse { + val jobIdRequestBody = JobIdRequestBody().id(jobId) + val response = try { + configApiClient.cancelJob(jobIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for cancelJob: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, jobId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return JobResponseMapper.from(Objects.requireNonNull(response.body())) + } + + /** + * Gets job info without logs as they're sometimes large enough to make the response size exceed the server max. + */ + override fun getJobInfoWithoutLogs(jobId: Long, userInfo: String?): JobResponse { + val jobIdRequestBody = JobIdRequestBody().id(jobId) + val response = try { + configApiClient.getJobInfoWithoutLogs(jobIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error( + "Config api response error for getJobInfoWithoutLogs: $jobId", + e, + ) + e.response as HttpResponse + } catch (e: ReadTimeoutException) { + log.error( + "Config api read timeout error for getJobInfoWithoutLogs: $jobId", + e, + ) + throw UnexpectedProblem(HttpStatus.REQUEST_TIMEOUT) + } + + ConfigClientErrorHandler.handleError(response, jobId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return JobResponseMapper.from(Objects.requireNonNull(response.body())) + } + + /** + * Lists jobs by connection ID and job type. + */ + override fun getJobList(connectionId: UUID, jobsFilter: JobsFilter, userInfo: String?): JobsResponse { + val configTypes: List = getJobConfigTypes(jobsFilter.jobType) + val jobListRequestBody = JobListRequestBody() + .configId(connectionId.toString()) + .configTypes(configTypes) + .pagination(Pagination().pageSize(jobsFilter.limit).rowOffset(jobsFilter.offset)) + .status(jobsFilter.getConfigApiStatus()) + .createdAtStart(jobsFilter.createdAtStart) + .createdAtEnd(jobsFilter.createdAtEnd) + .updatedAtStart(jobsFilter.updatedAtStart) + .updatedAtEnd(jobsFilter.updatedAtEnd) + + val response = try { + configApiClient.getJobList(jobListRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for getJobList: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, connectionId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return JobsResponseMapper.from( + response.body()!!, + connectionId, + jobsFilter.jobType, + jobsFilter.limit!!, + jobsFilter.offset!!, + publicApiHost!!, + ) + } + + /** + * list jobs by workspace ID and job type. + */ + override fun getJobList(workspaceIds: List, jobsFilter: JobsFilter, userInfo: String?): JobsResponse { + val configTypes = getJobConfigTypes(jobsFilter.jobType) + + // Get relevant workspace Ids + val workspaceIdsToQuery = workspaceIds.ifEmpty { userService.getAllWorkspaceIdsForUser(null, userInfo) } + + val requestBody = JobListForWorkspacesRequestBody() + .workspaceIds(workspaceIdsToQuery) + .configTypes(configTypes) + .pagination(Pagination().pageSize(jobsFilter.limit).rowOffset(jobsFilter.offset)) + .status(jobsFilter.getConfigApiStatus()) + .createdAtStart(jobsFilter.createdAtStart) + .createdAtEnd(jobsFilter.createdAtEnd) + .updatedAtStart(jobsFilter.updatedAtStart) + .updatedAtEnd(jobsFilter.updatedAtEnd) + + val response = try { + configApiClient.getJobListForWorkspaces(requestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for getJobList: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, workspaceIds.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return JobsResponseMapper.from( + response.body()!!, + workspaceIds, + jobsFilter.jobType, + jobsFilter.limit!!, + jobsFilter.offset!!, + publicApiHost!!, + ) + } + + private fun getJobConfigTypes(jobType: JobTypeEnum?): List { + val configTypes: MutableList = ArrayList() + if (jobType == null) { + configTypes.addAll(listOf(JobConfigType.SYNC, JobConfigType.RESET_CONNECTION)) + } else { + when (jobType) { + JobTypeEnum.SYNC -> configTypes.add(JobConfigType.SYNC) + JobTypeEnum.RESET -> configTypes.add(JobConfigType.RESET_CONNECTION) + else -> throw UnprocessableEntityProblem() + } + } + return configTypes + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/OAuthService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/OAuthService.kt index 3913cef0e94..98e022c8834 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/OAuthService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/OAuthService.kt @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server.services import com.fasterxml.jackson.databind.JsonNode @@ -8,12 +12,13 @@ import java.util.UUID import javax.validation.constraints.NotBlank interface OAuthService { + fun getSourceConsentUrl( workspaceId: @NotBlank UUID, definitionId: @NotBlank UUID, redirectUrl: @NotBlank String, oauthInputConfiguration: JsonNode, - userInfo: String, + userInfo: String?, ): OAuthConsentRead fun completeSourceOAuthReturnSecret( @@ -22,7 +27,7 @@ interface OAuthService { redirectUrl: @NotBlank String, queryParameters: @NotBlank MutableMap, oauthInputConfiguration: JsonNode, - userInfo: String, + userInfo: String?, ): CompleteOAuthResponse fun setWorkspaceOverrideOAuthParams( @@ -30,6 +35,6 @@ interface OAuthService { actorType: ActorTypeEnum, definitionId: UUID, oauthCredentialsConfiguration: JsonNode, - userInfo: String, + userInfo: String?, ) } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceDefinitionService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceDefinitionService.kt new file mode 100644 index 00000000000..214da68f424 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceDefinitionService.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.services + +import io.airbyte.api.client.model.generated.SourceDefinitionIdWithWorkspaceId +import io.airbyte.api.client.model.generated.SourceDefinitionSpecificationRead +import io.airbyte.api.server.constants.HTTP_RESPONSE_BODY_DEBUG_MESSAGE +import io.airbyte.api.server.errorHandlers.ConfigClientErrorHandler +import io.airbyte.api.server.forwardingClient.ConfigApiClient +import io.airbyte.api.server.mappers.SourceDefinitionSpecificationReadMapper +import io.micronaut.context.annotation.Secondary +import io.micronaut.http.HttpResponse +import io.micronaut.http.client.exceptions.HttpClientResponseException +import jakarta.inject.Singleton +import org.slf4j.LoggerFactory +import java.util.Objects +import java.util.UUID + +interface SourceDefinitionService { + + fun getSourceDefinitionSpecification( + sourceDefinitionId: UUID, + workspaceId: UUID, + userInfo: String?, + ): SourceDefinitionSpecificationRead? +} + +@Singleton +@Secondary +class SourceDefinitionServiceImpl(private val configApiClient: ConfigApiClient) : SourceDefinitionService { + + companion object { + private val log = LoggerFactory.getLogger(SourceDefinitionServiceImpl::class.java) + } + + override fun getSourceDefinitionSpecification(sourceDefinitionId: UUID, workspaceId: UUID, userInfo: String?): SourceDefinitionSpecificationRead? { + val sourceDefinitionIdWithWorkspaceId = SourceDefinitionIdWithWorkspaceId().sourceDefinitionId(sourceDefinitionId).workspaceId(workspaceId) + + var response: HttpResponse + try { + response = configApiClient.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId, userInfo!!) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for cancelJob: ", e) + response = e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, sourceDefinitionId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return SourceDefinitionSpecificationReadMapper.from( + Objects.requireNonNull( + response.body(), + ), + ) + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceService.kt index 9854b80c560..da119621bc3 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceService.kt @@ -1,27 +1,251 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server.services import io.airbyte.airbyte_api.model.generated.SourceCreateRequest import io.airbyte.airbyte_api.model.generated.SourcePatchRequest import io.airbyte.airbyte_api.model.generated.SourcePutRequest import io.airbyte.airbyte_api.model.generated.SourceResponse +import io.airbyte.airbyte_api.model.generated.SourcesResponse +import io.airbyte.api.client.model.generated.ListResourcesForWorkspacesRequestBody +import io.airbyte.api.client.model.generated.Pagination +import io.airbyte.api.client.model.generated.PartialSourceUpdate +import io.airbyte.api.client.model.generated.SourceCreate import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRead +import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody +import io.airbyte.api.client.model.generated.SourceIdRequestBody +import io.airbyte.api.client.model.generated.SourceRead +import io.airbyte.api.client.model.generated.SourceReadList +import io.airbyte.api.client.model.generated.SourceUpdate +import io.airbyte.api.server.constants.HTTP_RESPONSE_BODY_DEBUG_MESSAGE +import io.airbyte.api.server.errorHandlers.ConfigClientErrorHandler +import io.airbyte.api.server.forwardingClient.ConfigApiClient +import io.airbyte.api.server.mappers.SourceReadMapper +import io.airbyte.api.server.mappers.SourcesResponseMapper +import io.airbyte.api.server.problems.UnexpectedProblem +import io.micronaut.context.annotation.Secondary +import io.micronaut.context.annotation.Value +import io.micronaut.http.HttpResponse +import io.micronaut.http.HttpStatus +import io.micronaut.http.client.exceptions.HttpClientResponseException +import io.micronaut.http.client.exceptions.ReadTimeoutException +import jakarta.inject.Singleton +import org.slf4j.LoggerFactory import java.util.UUID -import javax.validation.constraints.NotBlank interface SourceService { + fun createSource( - sourceCreateRequest: @NotBlank SourceCreateRequest, - sourceDefinitionId: @NotBlank UUID, - userInfo: String, + sourceCreateRequest: SourceCreateRequest, + sourceDefinitionId: UUID, + userInfo: String?, ): SourceResponse - fun updateSource(sourceId: UUID, sourcePutRequest: SourcePutRequest, userInfo: String): SourceResponse + fun updateSource(sourceId: UUID, sourcePutRequest: SourcePutRequest, userInfo: String?): SourceResponse + + fun partialUpdateSource(sourceId: UUID, sourcePatchRequest: SourcePatchRequest, userInfo: String?): SourceResponse + + fun deleteSource(sourceId: UUID, userInfo: String?) + + fun getSource(sourceId: UUID, userInfo: String?): SourceResponse + + fun getSourceSchema(sourceId: UUID, disableCache: Boolean, userInfo: String?): SourceDiscoverSchemaRead + + fun listSourcesForWorkspaces( + workspaceIds: List, + includeDeleted: Boolean = false, + limit: Int = 20, + offset: Int = 0, + + userInfo: String?, + ): SourcesResponse +} + +@Singleton +@Secondary +class SourceServiceImpl( + private val configApiClient: ConfigApiClient, + private val userService: UserServiceImpl, +) : SourceService { + + companion object { + private val log = LoggerFactory.getLogger(SourceServiceImpl::class.java) + } + + @Value("\${airbyte.api.host}") + var publicApiHost: String? = null + + /** + * Creates a source. + */ + override fun createSource(sourceCreateRequest: SourceCreateRequest, sourceDefinitionId: UUID, userInfo: String?): SourceResponse { + val sourceCreateOss = SourceCreate() + sourceCreateOss.name = sourceCreateRequest.name + sourceCreateOss.sourceDefinitionId = sourceDefinitionId + sourceCreateOss.workspaceId = sourceCreateRequest.workspaceId + sourceCreateOss.connectionConfiguration = sourceCreateRequest.configuration + sourceCreateOss.secretId = sourceCreateRequest.secretId + + val response = try { + configApiClient.createSource(sourceCreateOss, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for createSource: ", e) + e.response as HttpResponse + } + + ConfigClientErrorHandler.handleError(response, sourceCreateRequest.workspaceId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return SourceReadMapper.from(response.body()!!) + } + + /** + * Updates a source fully with full replacement of configuration. + */ + override fun updateSource(sourceId: UUID, sourcePutRequest: SourcePutRequest, userInfo: String?): SourceResponse { + val sourceUpdate = SourceUpdate() + .sourceId(sourceId) + .connectionConfiguration(sourcePutRequest.configuration) + .name(sourcePutRequest.name) + + val response = try { + configApiClient.updateSource(sourceUpdate, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for updateSource: ", e) + e.response as HttpResponse + } + + ConfigClientErrorHandler.handleError(response, sourceId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return SourceReadMapper.from(response.body()!!) + } + + /** + * Updates a source allowing patch semantics including within the configuration. + */ + override fun partialUpdateSource(sourceId: UUID, sourcePatchRequest: SourcePatchRequest, userInfo: String?): SourceResponse { + val sourceUpdate = PartialSourceUpdate() + .sourceId(sourceId) + .connectionConfiguration(sourcePatchRequest.configuration) + .name(sourcePatchRequest.name) + .secretId(sourcePatchRequest.secretId) + + val response = try { + configApiClient.partialUpdateSource(sourceUpdate, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for partialUpdateSource: ", e) + e.response as HttpResponse + } + + ConfigClientErrorHandler.handleError(response, sourceId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return SourceReadMapper.from(response.body()!!) + } + + /** + * Deletes a source by ID. + */ + override fun deleteSource(sourceId: UUID, userInfo: String?) { + val sourceIdRequestBody = SourceIdRequestBody().sourceId(sourceId) + val response = try { + configApiClient.deleteSource(sourceIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for source delete: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, sourceId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + } + + /** + * Gets a source by ID. + */ + override fun getSource(sourceId: UUID, userInfo: String?): SourceResponse { + val sourceIdRequestBody = SourceIdRequestBody() + sourceIdRequestBody.sourceId = sourceId + val response = try { + configApiClient.getSource(sourceIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for getSource: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, sourceId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return SourceReadMapper.from(response.body()!!) + } - fun partialUpdateSource(sourceId: UUID, sourcePatchRequest: SourcePatchRequest, userInfo: String): SourceResponse + /** + * Gets a source's schema. + */ + override fun getSourceSchema( + sourceId: UUID, + disableCache: Boolean, + userInfo: String?, + ): SourceDiscoverSchemaRead { + val sourceDiscoverSchemaRequestBody = SourceDiscoverSchemaRequestBody().sourceId(sourceId).disableCache(disableCache) - fun deleteSource(sourceId: @NotBlank UUID, userInfo: String) + val response: HttpResponse = try { + configApiClient.getSourceSchema(sourceDiscoverSchemaRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for getSourceSchema: ", e) + e.response as HttpResponse + } catch (e: ReadTimeoutException) { + log.error("Config api read timeout error for getSourceSchema: ", e) + if (disableCache) { + throw UnexpectedProblem( + "try-again", + HttpStatus.REQUEST_TIMEOUT, + "Updating cache latest source schema in progress. Please try again with cache on.", + ) + } else { + throw UnexpectedProblem(HttpStatus.REQUEST_TIMEOUT) + } + } + ConfigClientErrorHandler.handleError(response, sourceId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + if (response.body() == null || response.body()?.jobInfo?.succeeded == false) { + var errorMessage = "Something went wrong in the connector." + if (response.body() != null && response.body()?.jobInfo?.failureReason!!.internalMessage != null) { + errorMessage += " logs:" + response.body()?.jobInfo!!.failureReason!!.internalMessage + } + throw UnexpectedProblem(HttpStatus.BAD_REQUEST, errorMessage) + } + return response.body()!! + } - fun getSource(sourceId: @NotBlank UUID, userInfo: String): SourceResponse + /** + * Lists sources by workspace IDs or all sources if no workspace IDs are provided. + */ + override fun listSourcesForWorkspaces( + workspaceIds: List, + includeDeleted: Boolean, + limit: Int, + offset: Int, + userInfo: String?, + ): SourcesResponse { + val pagination: Pagination = Pagination().pageSize(limit).rowOffset(offset) + val workspaceIdsToQuery = workspaceIds.ifEmpty { userService.getAllWorkspaceIdsForUser(null, userInfo) } + val listResourcesForWorkspacesRequestBody = ListResourcesForWorkspacesRequestBody() + listResourcesForWorkspacesRequestBody.includeDeleted = includeDeleted + listResourcesForWorkspacesRequestBody.pagination = pagination + listResourcesForWorkspacesRequestBody.workspaceIds = workspaceIdsToQuery - fun getSourceSchema(sourceId: UUID, disableCache: Boolean, userInfo: String): SourceDiscoverSchemaRead + val response = try { + configApiClient.listSourcesForWorkspaces(listResourcesForWorkspacesRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for listWorkspaces: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, workspaceIds.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return SourcesResponseMapper.from( + response.body()!!, + workspaceIds, + includeDeleted, + limit, + offset, + publicApiHost!!, + ) + } } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/UserService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/UserService.kt index 5f287a0cbfb..aa8b1983608 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/UserService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/UserService.kt @@ -1,12 +1,63 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server.services +import io.airbyte.api.client.model.generated.WorkspaceReadList +import io.airbyte.api.server.constants.AIRBYTE_API_AUTH_HEADER_VALUE +import io.airbyte.api.server.constants.HTTP_RESPONSE_BODY_DEBUG_MESSAGE +import io.airbyte.api.server.errorHandlers.ConfigClientErrorHandler +import io.airbyte.api.server.forwardingClient.ConfigApiClient +import io.micronaut.context.annotation.Secondary +import io.micronaut.http.HttpResponse +import io.micronaut.http.client.exceptions.HttpClientResponseException +import jakarta.inject.Singleton +import org.slf4j.LoggerFactory import java.util.UUID interface UserService { + fun getAllWorkspaceIdsForUser( - userId: UUID, - authorization: String, + userId: UUID?, + userInfo: String?, ): List - fun getUserIdFromAuthToken(authToken: String): UUID + fun getUserIdFromUserInfoString(userInfo: String?): UUID +} + +@Singleton +@Secondary +class UserServiceImpl(private val configApiClient: ConfigApiClient) : UserService { + + companion object { + private val log = LoggerFactory.getLogger(UserServiceImpl::class.java) + } + + /** + * Hits the listAllWorkspaces endpoint since OSS has only one user. + */ + override fun getAllWorkspaceIdsForUser(userId: UUID?, userInfo: String?): List { + var response: HttpResponse + try { + response = configApiClient.listAllWorkspaces(System.getenv(AIRBYTE_API_AUTH_HEADER_VALUE)) + } catch (e: HttpClientResponseException) { + log.error("Cloud api response error for listWorkspacesByUser: ", e) + response = e.response as HttpResponse + } + + ConfigClientErrorHandler.handleError(response, "airbyte-user") + + val workspaces = response.body()?.workspaces.orEmpty() + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + + return workspaces.map { it.workspaceId } + } + + /** + * No-op for OSS. + */ + override fun getUserIdFromUserInfoString(userInfo: String?): UUID { + return UUID.fromString("00000000-0000-0000-0000-000000000000") + } } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/WorkspaceService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/WorkspaceService.kt index e106e03a135..a87e45c839c 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/WorkspaceService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/WorkspaceService.kt @@ -1,31 +1,162 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server.services import io.airbyte.airbyte_api.model.generated.WorkspaceCreateRequest import io.airbyte.airbyte_api.model.generated.WorkspaceResponse import io.airbyte.airbyte_api.model.generated.WorkspaceUpdateRequest import io.airbyte.airbyte_api.model.generated.WorkspacesResponse +import io.airbyte.api.client.model.generated.ListResourcesForWorkspacesRequestBody +import io.airbyte.api.client.model.generated.Pagination +import io.airbyte.api.client.model.generated.WorkspaceCreate +import io.airbyte.api.client.model.generated.WorkspaceIdRequestBody +import io.airbyte.api.client.model.generated.WorkspaceRead +import io.airbyte.api.client.model.generated.WorkspaceReadList +import io.airbyte.api.server.constants.AIRBYTE_API_AUTH_HEADER_VALUE +import io.airbyte.api.server.constants.HTTP_RESPONSE_BODY_DEBUG_MESSAGE +import io.airbyte.api.server.errorHandlers.ConfigClientErrorHandler +import io.airbyte.api.server.forwardingClient.ConfigApiClient +import io.airbyte.api.server.mappers.WorkspaceResponseMapper +import io.airbyte.api.server.mappers.WorkspacesResponseMapper +import io.micronaut.context.annotation.Secondary +import io.micronaut.context.annotation.Value +import io.micronaut.http.HttpResponse +import io.micronaut.http.client.exceptions.HttpClientResponseException +import jakarta.inject.Singleton +import org.slf4j.LoggerFactory +import java.util.Objects import java.util.UUID -import javax.validation.constraints.NotBlank interface WorkspaceService { - fun createCloudWorkspace(workspaceCreateRequest: @NotBlank WorkspaceCreateRequest?, authorization: String): WorkspaceResponse + fun createWorkspace(workspaceCreateRequest: WorkspaceCreateRequest, userInfo: String?): WorkspaceResponse - fun updateCloudWorkspace( + fun updateWorkspace( workspaceId: UUID, workspaceUpdateRequest: WorkspaceUpdateRequest, - authorization: String, + userInfo: String?, ): WorkspaceResponse - fun getWorkspace(workspaceId: @NotBlank UUID?, userInfo: String): WorkspaceResponse + fun getWorkspace(workspaceId: UUID, userInfo: String?): WorkspaceResponse - fun deleteWorkspace(workspaceId: @NotBlank UUID?, authorization: String, userInfo: String) + fun deleteWorkspace(workspaceId: UUID, userInfo: String?) fun listWorkspaces( - workspaceIds: List, + workspaceIds: List, + includeDeleted: Boolean = false, + limit: Int = 20, + offset: Int = 0, + + userInfo: String?, + ): WorkspacesResponse +} + +@Singleton +@Secondary +class WorkspaceServiceImpl(private val configApiClient: ConfigApiClient, private val userService: UserService) : WorkspaceService { + + @Value("\${airbyte.api.host}") + var publicApiHost: String? = null + + companion object { + private val log = LoggerFactory.getLogger(WorkspaceServiceImpl::class.java) + } + + /** + * Creates a workspace. + */ + override fun createWorkspace(workspaceCreateRequest: WorkspaceCreateRequest, userInfo: String?): WorkspaceResponse { + val workspaceCreate = WorkspaceCreate().name(workspaceCreateRequest.name) + val workspaceReadHttpResponse = try { + configApiClient.createWorkspace(workspaceCreate, System.getenv(AIRBYTE_API_AUTH_HEADER_VALUE)) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for createWorkspace: ", e) + e.response as HttpResponse + } + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + workspaceReadHttpResponse.body) + ConfigClientErrorHandler.handleError(workspaceReadHttpResponse, workspaceReadHttpResponse.body()?.workspaceId.toString()) + return WorkspaceResponseMapper.from( + Objects.requireNonNull( + workspaceReadHttpResponse.body() as WorkspaceRead, + ), + ) + } + + /** + * No-op in OSS. + */ + override fun updateWorkspace(workspaceId: UUID, workspaceUpdateRequest: WorkspaceUpdateRequest, userInfo: String?): WorkspaceResponse { + // Update workspace in the cloud version of the airbyte API currently only supports name updates, but we don't have name updates in OSS. + return WorkspaceResponse() + } + + /** + * Fetches a workspace by ID. + */ + override fun getWorkspace(workspaceId: UUID, userInfo: String?): WorkspaceResponse { + val workspaceIdRequestBody = WorkspaceIdRequestBody() + workspaceIdRequestBody.workspaceId = workspaceId + val response = try { + configApiClient.getWorkspace(workspaceIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for getWorkspace: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, workspaceId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return WorkspaceResponseMapper.from(response.body()!!) + } + + /** + * Deletes a workspace by ID. + */ + override fun deleteWorkspace(workspaceId: UUID, userInfo: String?) { + val workspaceIdRequestBody = WorkspaceIdRequestBody() + workspaceIdRequestBody.workspaceId = workspaceId + val response = try { + configApiClient.deleteWorkspace(workspaceIdRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for deleteWorkspace: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, workspaceId.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body) + } + + /** + * Lists a workspace by a set of IDs or all workspaces if no IDs are provided. + */ + override fun listWorkspaces( + workspaceIds: List, includeDeleted: Boolean, limit: Int, offset: Int, - authorization: String, - userInfo: String, - ): WorkspacesResponse + userInfo: String?, + ): WorkspacesResponse { + val pagination: Pagination = Pagination().pageSize(limit).rowOffset(offset) + + val workspaceIdsToQuery = workspaceIds.ifEmpty { userService.getAllWorkspaceIdsForUser(null, userInfo) } + log.debug("Workspaces to query: $workspaceIdsToQuery") + val listResourcesForWorkspacesRequestBody = ListResourcesForWorkspacesRequestBody() + listResourcesForWorkspacesRequestBody.includeDeleted = includeDeleted + listResourcesForWorkspacesRequestBody.pagination = pagination + listResourcesForWorkspacesRequestBody.workspaceIds = workspaceIdsToQuery + val response = try { + configApiClient.listWorkspaces(listResourcesForWorkspacesRequestBody, userInfo) + } catch (e: HttpClientResponseException) { + log.error("Config api response error for listWorkspaces: ", e) + e.response as HttpResponse + } + ConfigClientErrorHandler.handleError(response, workspaceIds.toString()) + log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) + return WorkspacesResponseMapper.from( + response.body()!!, + workspaceIds, + includeDeleted, + limit, + offset, + publicApiHost!!, + ) + } } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/ConnectionServiceImpl.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/ConnectionServiceImpl.kt deleted file mode 100644 index e1aebf7b289..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/ConnectionServiceImpl.kt +++ /dev/null @@ -1,40 +0,0 @@ -package io.airbyte.api.server.services.impls - -import io.airbyte.airbyte_api.model.generated.ConnectionResponse -import io.airbyte.airbyte_api.model.generated.ConnectionsResponse -import io.airbyte.api.model.generated.ConnectionCreate -import io.airbyte.api.model.generated.ConnectionUpdate -import io.airbyte.api.server.services.ConnectionService -import io.micronaut.context.annotation.Secondary -import jakarta.inject.Singleton -import java.util.UUID - -@Singleton -@Secondary -class ConnectionServiceImpl : ConnectionService { - override fun createConnection(connectionCreate: ConnectionCreate, endpointUserInfo: String): ConnectionResponse { - TODO("Not yet implemented") - } - - override fun deleteConnection(connectionId: UUID, endpointUserInfo: String): ConnectionResponse { - TODO("Not yet implemented") - } - - override fun getConnection(connectionId: UUID, endpointUserInfo: String): ConnectionResponse { - TODO("Not yet implemented") - } - - override fun updateConnection(connectionUpdate: ConnectionUpdate, endpointUserInfo: String): ConnectionResponse { - TODO("Not yet implemented") - } - - override fun listConnectionsForWorkspaces( - workspaceIds: MutableList, - limit: Int, - offset: Int, - includeDeleted: Boolean, - endpointUserInfo: String, - ): ConnectionsResponse { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/DestinationServiceImpl.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/DestinationServiceImpl.kt deleted file mode 100644 index 2c0e366e229..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/DestinationServiceImpl.kt +++ /dev/null @@ -1,50 +0,0 @@ -package io.airbyte.api.server.services.impls - -import io.airbyte.airbyte_api.model.generated.DestinationCreateRequest -import io.airbyte.airbyte_api.model.generated.DestinationPatchRequest -import io.airbyte.airbyte_api.model.generated.DestinationPutRequest -import io.airbyte.airbyte_api.model.generated.DestinationResponse -import io.airbyte.airbyte_api.model.generated.DestinationsResponse -import io.airbyte.api.server.services.DestinationService -import java.util.UUID - -class DestinationServiceImpl : DestinationService { - override fun createDestination( - destinationCreateRequest: DestinationCreateRequest?, - destinationDefinitionId: UUID?, - userInfo: String, - ): DestinationResponse { - TODO("Not yet implemented") - } - - override fun getDestination(destinationId: UUID?, userInfo: String): DestinationResponse { - TODO("Not yet implemented") - } - - override fun updateDestination(destinationId: UUID, destinationPutRequest: DestinationPutRequest, userInfo: String): DestinationResponse { - TODO("Not yet implemented") - } - - override fun partialUpdateDestination( - destinationId: UUID, - destinationPatchRequest: DestinationPatchRequest, - userInfo: String, - ): DestinationResponse { - TODO("Not yet implemented") - } - - override fun deleteDestination(connectionId: UUID, userInfo: String) { - TODO("Not yet implemented") - } - - override fun listDestinationsForWorkspaces( - workspaceIds: List, - includeDeleted: Boolean, - limit: Int, - offset: Int, - authorization: String, - userInfo: String, - ): DestinationsResponse? { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/JobServiceImpl.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/JobServiceImpl.kt deleted file mode 100644 index 0a0b9c90950..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/JobServiceImpl.kt +++ /dev/null @@ -1,40 +0,0 @@ -package io.airbyte.api.server.services.impls - -import io.airbyte.airbyte_api.model.generated.JobResponse -import io.airbyte.airbyte_api.model.generated.JobTypeEnum -import io.airbyte.airbyte_api.model.generated.JobsResponse -import io.airbyte.api.server.services.JobService -import java.util.UUID - -class JobServiceImpl : JobService { - override fun sync(connectionId: UUID, userInfo: String): JobResponse { - TODO("Not yet implemented") - } - - override fun reset(connectionId: UUID, userInfo: String): JobResponse { - TODO("Not yet implemented") - } - - override fun cancelJob(jobId: Long, userInfo: String): JobResponse { - TODO("Not yet implemented") - } - - override fun getJobInfoWithoutLogs(jobId: Long, userInfo: String): JobResponse { - TODO("Not yet implemented") - } - - override fun getJobList(connectionId: UUID, jobType: JobTypeEnum, limit: Int, offset: Int, userInfo: String): JobsResponse { - TODO("Not yet implemented") - } - - override fun getJobList( - workspaceIds: List, - jobType: JobTypeEnum, - limit: Int, - offset: Int, - authorization: String, - userInfo: String, - ): JobsResponse { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/OAuthServiceImpl.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/OAuthServiceImpl.kt deleted file mode 100644 index 9a3b7765af2..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/OAuthServiceImpl.kt +++ /dev/null @@ -1,41 +0,0 @@ -package io.airbyte.api.server.services.impls - -import com.fasterxml.jackson.databind.JsonNode -import io.airbyte.airbyte_api.model.generated.ActorTypeEnum -import io.airbyte.api.client.model.generated.CompleteOAuthResponse -import io.airbyte.api.client.model.generated.OAuthConsentRead -import io.airbyte.api.server.services.OAuthService -import java.util.UUID - -class OAuthServiceImpl : OAuthService { - override fun getSourceConsentUrl( - workspaceId: UUID, - definitionId: UUID, - redirectUrl: String, - oauthInputConfiguration: JsonNode, - userInfo: String, - ): OAuthConsentRead { - TODO("Not yet implemented") - } - - override fun completeSourceOAuthReturnSecret( - workspaceId: UUID, - definitionId: UUID, - redirectUrl: String, - queryParameters: MutableMap, - oauthInputConfiguration: JsonNode, - userInfo: String, - ): CompleteOAuthResponse { - TODO("Not yet implemented") - } - - override fun setWorkspaceOverrideOAuthParams( - workspaceId: UUID, - actorType: ActorTypeEnum, - definitionId: UUID, - oauthCredentialsConfiguration: JsonNode, - userInfo: String, - ) { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/SourceServiceImpl.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/SourceServiceImpl.kt deleted file mode 100644 index 86e4f68e263..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/SourceServiceImpl.kt +++ /dev/null @@ -1,39 +0,0 @@ -package io.airbyte.api.server.services.impls - -import io.airbyte.airbyte_api.model.generated.SourceCreateRequest -import io.airbyte.airbyte_api.model.generated.SourcePatchRequest -import io.airbyte.airbyte_api.model.generated.SourcePutRequest -import io.airbyte.airbyte_api.model.generated.SourceResponse -import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRead -import io.airbyte.api.server.services.SourceService -import java.util.UUID - -class SourceServiceImpl : SourceService { - override fun createSource(sourceCreateRequest: SourceCreateRequest, sourceDefinitionId: UUID, userInfo: String): SourceResponse { - TODO("Not yet implemented") - } - - override fun updateSource(sourceId: UUID, sourcePutRequest: SourcePutRequest, userInfo: String): SourceResponse { - TODO("Not yet implemented") - } - - override fun partialUpdateSource(sourceId: UUID, sourcePatchRequest: SourcePatchRequest, userInfo: String): SourceResponse { - TODO("Not yet implemented") - } - - override fun deleteSource(sourceId: UUID, userInfo: String) { - TODO("Not yet implemented") - } - - override fun getSource(sourceId: UUID, userInfo: String): SourceResponse { - TODO("Not yet implemented") - } - - override fun getSourceSchema( - sourceId: UUID, - disableCache: Boolean, - userInfo: String, - ): SourceDiscoverSchemaRead { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/UserServiceImpl.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/UserServiceImpl.kt deleted file mode 100644 index 22d58a3d119..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/UserServiceImpl.kt +++ /dev/null @@ -1,14 +0,0 @@ -package io.airbyte.api.server.services.impls - -import io.airbyte.api.server.services.UserService -import java.util.UUID - -class UserServiceImpl : UserService { - override fun getAllWorkspaceIdsForUser(userId: UUID, authorization: String): List { - TODO("Not yet implemented") - } - - override fun getUserIdFromAuthToken(authToken: String): UUID { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/WorkspaceServiceImpl.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/WorkspaceServiceImpl.kt deleted file mode 100644 index 380e74cf862..00000000000 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/impls/WorkspaceServiceImpl.kt +++ /dev/null @@ -1,37 +0,0 @@ -package io.airbyte.api.server.services.impls - -import io.airbyte.airbyte_api.model.generated.WorkspaceCreateRequest -import io.airbyte.airbyte_api.model.generated.WorkspaceResponse -import io.airbyte.airbyte_api.model.generated.WorkspaceUpdateRequest -import io.airbyte.airbyte_api.model.generated.WorkspacesResponse -import io.airbyte.api.server.services.WorkspaceService -import java.util.UUID - -class WorkspaceServiceImpl : WorkspaceService { - override fun createCloudWorkspace(workspaceCreateRequest: WorkspaceCreateRequest?, authorization: String): WorkspaceResponse { - TODO("Not yet implemented") - } - - override fun updateCloudWorkspace(workspaceId: UUID, workspaceUpdateRequest: WorkspaceUpdateRequest, authorization: String): WorkspaceResponse { - TODO("Not yet implemented") - } - - override fun getWorkspace(workspaceId: UUID?, userInfo: String): WorkspaceResponse { - TODO("Not yet implemented") - } - - override fun deleteWorkspace(workspaceId: UUID?, authorization: String, userInfo: String) { - TODO("Not yet implemented") - } - - override fun listWorkspaces( - workspaceIds: List, - includeDeleted: Boolean, - limit: Int, - offset: Int, - authorization: String, - userInfo: String, - ): WorkspacesResponse { - TODO("Not yet implemented") - } -} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/validation/AirbyteRequestBinderRegistry.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/validation/AirbyteRequestBinderRegistry.kt new file mode 100644 index 00000000000..0847c75898f --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/validation/AirbyteRequestBinderRegistry.kt @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.validation + +import io.micronaut.context.annotation.Replaces +import io.micronaut.core.convert.ConversionService +import io.micronaut.http.bind.DefaultRequestBinderRegistry +import io.micronaut.http.bind.binders.RequestArgumentBinder +import jakarta.inject.Singleton + +/** + * Required to force validation of nullable query string parameters. Taken from + * https://github.com/micronaut-projects/micronaut-core/issues/5135. Should be replaced when + * micronaut validation is improved. https://github.com/micronaut-projects/micronaut-core/pull/6808 + */ +@Singleton +@Replaces(DefaultRequestBinderRegistry::class) +class AirbyteRequestBinderRegistry(conversionService: ConversionService<*>?, binders: List?>?) : + DefaultRequestBinderRegistry(conversionService, binders) { + init { + addRequestArgumentBinder(QueryValueBinder(conversionService)) + } + } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/validation/QueryValueBinder.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/validation/QueryValueBinder.kt new file mode 100644 index 00000000000..175819d3217 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/validation/QueryValueBinder.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.server.validation + +import io.micronaut.core.bind.ArgumentBinder.BindingResult +import io.micronaut.core.convert.ArgumentConversionContext +import io.micronaut.core.convert.ConversionError +import io.micronaut.core.convert.ConversionService +import io.micronaut.http.bind.binders.QueryValueArgumentBinder +import java.util.Optional + +/** + * Required to force validation of nullable query string parameters. Taken from + * https://github.com/micronaut-projects/micronaut-core/issues/5135. Should be replaced when + * micronaut validation is improved. https://github.com/micronaut-projects/micronaut-core/pull/6808 + */ +class QueryValueBinder(conversionService: ConversionService<*>?) : QueryValueArgumentBinder(conversionService) { + override fun doConvert(value: Any?, context: ArgumentConversionContext, defaultResult: BindingResult): BindingResult { + return if (value == null && context.hasErrors()) { + object : BindingResult { + override fun getValue(): Optional? { + return null + } + + override fun isSatisfied(): Boolean { + return false + } + + override fun getConversionErrors(): List { + val errors: MutableList = ArrayList() + for (error in context) { + errors.add(error) + } + return errors + } + } + } else { + super.doConvert(value, context, defaultResult) + } + } +} diff --git a/airbyte-api-server/src/main/resources/application.yml b/airbyte-api-server/src/main/resources/application.yml index 8925e8d8b44..8bee489ce57 100644 --- a/airbyte-api-server/src/main/resources/application.yml +++ b/airbyte-api-server/src/main/resources/application.yml @@ -48,15 +48,12 @@ airbyte: min-version: ${AIRBYTE_PROTOCOL_VERSION_MIN:0.0.0} max-version: ${AIRBYTE_PROTOCOL_VERSION_MAX:0.3.0} api: - host: ${API_HOST} + host: ${AIRBYTE_API_HOST} internal: api: host: ${INTERNAL_API_HOST} documentation: host: https://reference.airbyte.com/ - cloud: - api: - host: ${CLOUD_API_HOST} endpoints: beans: diff --git a/airbyte-api-server/src/test/kotlin/io/airbyte/api/server/HealthTest.kt b/airbyte-api-server/src/test/kotlin/io/airbyte/api/server/HealthControllerTest.kt similarity index 81% rename from airbyte-api-server/src/test/kotlin/io/airbyte/api/server/HealthTest.kt rename to airbyte-api-server/src/test/kotlin/io/airbyte/api/server/HealthControllerTest.kt index 767a9b00a4c..eb7341dd933 100644 --- a/airbyte-api-server/src/test/kotlin/io/airbyte/api/server/HealthTest.kt +++ b/airbyte-api-server/src/test/kotlin/io/airbyte/api/server/HealthControllerTest.kt @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.api.server import io.micronaut.http.HttpRequest @@ -8,12 +12,12 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @MicronautTest -class HealthTest( +class HealthControllerTest( @Client("/") val client: HttpClient, ) { @Test - fun testHello() { + fun testHealthEndpoint() { val request: HttpRequest = HttpRequest.GET("/health") val body = client.toBlocking().retrieve(request) assertEquals("Successful operation", body) diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle index 44b504adaa4..dbff875c7ba 100644 --- a/airbyte-api/build.gradle +++ b/airbyte-api/build.gradle @@ -8,6 +8,7 @@ plugins { def specFile = "$projectDir/src/main/openapi/config.yaml" def airbyteApiSpecFile = "$projectDir/src/main/openapi/api.yaml" +def airbyteApiSpecTemplateDirApi = "$projectDir/src/main/resources/templates/jaxrs-spec-api" def genApiServer = tasks.register("generateApiServer", GenerateTask) { def serverOutputDir = "$buildDir/generated/api/server" @@ -131,6 +132,7 @@ def genAirbyteApiServer = tasks.register('generateAirbyteApiServer', GenerateTas generatorName = "jaxrs-spec" inputSpec = airbyteApiSpecFile outputDir = serverOutputDir + templateDir = airbyteApiSpecTemplateDirApi apiPackage = "io.airbyte.airbyte-api.generated" invokerPackage = "io.airbyte.airbyte-api.invoker.generated" diff --git a/airbyte-api/src/main/java/io/airbyte/api/common/ConfigurableActor.java b/airbyte-api/src/main/java/io/airbyte/api/common/ConfigurableActor.java new file mode 100644 index 00000000000..708dbacfe49 --- /dev/null +++ b/airbyte-api/src/main/java/io/airbyte/api/common/ConfigurableActor.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.common; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * Unified interface for actors that are configurable. + */ +public interface ConfigurableActor { + + JsonNode getConfiguration(); + +} diff --git a/airbyte-api/src/main/openapi/api.yaml b/airbyte-api/src/main/openapi/api.yaml index a6d75ae2e7f..87cbe47d8cf 100644 --- a/airbyte-api/src/main/openapi/api.yaml +++ b/airbyte-api/src/main/openapi/api.yaml @@ -79,6 +79,44 @@ paths: type: string in: query required: false + - name: status + description: The Job status you want to filter by + schema: + $ref: "#/components/schemas/JobStatusEnum" + in: query + required: false + - name: createdAtStart + description: The start date to filter by + schema: + type: string + format: date-time + in: query + required: false + example: 2023-06-22T16:15:00Z + - name: createdAtEnd + description: The end date to filter by + schema: + type: string + format: date-time + in: query + required: false + example: 2023-06-22T16:15:00Z + - name: updatedAtStart + description: The start date to filter by + schema: + type: string + format: date-time + example: 2023-06-22T16:15:00Z + in: query + required: false + - name: updatedAtEnd + description: The end date to filter by + schema: + type: string + format: date-time + in: query + required: false + example: 2023-06-22T16:15:00Z responses: "200": content: @@ -1249,6 +1287,7 @@ components: secretId: description: Optional secretID obtained through the public API OAuth redirect flow. type: string + x-implements: io.airbyte.api.common.ConfigurableActor SourcePutRequest: required: - name @@ -1259,6 +1298,7 @@ components: type: string configuration: $ref: "#/components/schemas/SourceConfiguration" + x-implements: io.airbyte.api.common.ConfigurableActor SourcePatchRequest: type: object properties: @@ -1273,6 +1313,7 @@ components: secretId: description: Optional secretID obtained through the public API OAuth redirect flow. type: string + x-implements: io.airbyte.api.common.ConfigurableActor InitiateOauthRequest: title: Root Type for initiate-oauth-post-body description: POST body for initiating OAuth via the public API @@ -1528,6 +1569,7 @@ components: type: string configuration: $ref: "#/components/schemas/DestinationConfiguration" + x-implements: io.airbyte.api.common.ConfigurableActor DestinationPatchRequest: type: object properties: @@ -1535,6 +1577,7 @@ components: type: string configuration: $ref: "#/components/schemas/DestinationConfiguration" + x-implements: io.airbyte.api.common.ConfigurableActor DestinationPutRequest: required: - name @@ -1545,6 +1588,7 @@ components: type: string configuration: $ref: "#/components/schemas/DestinationConfiguration" + x-implements: io.airbyte.api.common.ConfigurableActor WorkspaceCreateRequest: required: - name diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 8f652bff477..656882d1e12 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -127,6 +127,25 @@ paths: application/json: schema: $ref: "#/components/schemas/WorkspaceReadList" + /v1/workspaces/list_paginated: + post: + tags: + - workspace + summary: List all workspaces registered in the current Airbyte deployment, paginated + operationId: listWorkspacesPaginated + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ListResourcesForWorkspacesRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/WorkspaceReadList" /v1/workspaces/get: post: tags: @@ -869,6 +888,30 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/sources/list_paginated: + post: + tags: + - source + summary: List sources for workspace + description: List sources for workspace. Does not return deleted sources. Paginated. + operationId: listSourcesForWorkspacePaginated + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ListResourcesForWorkspacesRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/SourceReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/sources/get: post: tags: @@ -1480,6 +1523,29 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/destinations/list_paginated: + post: + tags: + - destination + summary: List configured destinations for a workspace. Pginated + operationId: listDestinationsForWorkspacesPaginated + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ListResourcesForWorkspacesRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/DestinationReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/destinations/get: post: tags: @@ -1683,6 +1749,30 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/connections/list_paginated: + post: + tags: + - connection + summary: Returns all connections for a workspace. Paginated. + description: List connections for workspace. Does not return deleted connections. Paginated. + operationId: listConnectionsForWorkspacesPaginated + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ListConnectionsForWorkspacesRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectionReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/connections/list_all: post: tags: diff --git a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/api.mustache b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/api.mustache new file mode 100644 index 00000000000..0a43722af5f --- /dev/null +++ b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/api.mustache @@ -0,0 +1,41 @@ +package {{package}}; + +{{#imports}}import {{import}}; +{{/imports}} + +import io.micronaut.http.annotation.Body; +import io.micronaut.http.annotation.RequestAttribute; +import io.micronaut.http.annotation.Body; + +import javax.ws.rs.*; +import javax.ws.rs.core.Response; +import javax.annotation.Nullable; + +{{#useSwaggerAnnotations}} +import io.swagger.annotations.*; +{{/useSwaggerAnnotations}} +{{#supportAsync}} +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CompletableFuture; +{{/supportAsync}} + +import java.io.InputStream; +import java.util.Map; +import java.util.List; +{{#useBeanValidation}}import javax.validation.constraints.*; +import javax.validation.Valid;{{/useBeanValidation}} + +@Path("{{contextPath}}{{commonPath}}"){{#useSwaggerAnnotations}} +@Api(description = "the {{{baseName}}} API"){{/useSwaggerAnnotations}}{{#hasConsumes}} +@Consumes({ {{#consumes}}"{{{mediaType}}}"{{^-last}}, {{/-last}}{{/consumes}} }){{/hasConsumes}}{{#hasProduces}} +@Produces({ {{#produces}}"{{{mediaType}}}"{{^-last}}, {{/-last}}{{/produces}} }){{/hasProduces}} +{{>generatedAnnotation}} + +public {{#interfaceOnly}}interface{{/interfaceOnly}}{{^interfaceOnly}}class{{/interfaceOnly}} {{classname}} { +{{#operations}} +{{#operation}} + +{{#interfaceOnly}}{{>apiInterface}}{{/interfaceOnly}}{{^interfaceOnly}}{{>apiMethod}}{{/interfaceOnly}} +{{/operation}} +} +{{/operations}} diff --git a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/apiInterface.mustache b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/apiInterface.mustache new file mode 100644 index 00000000000..2357ebf76fb --- /dev/null +++ b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/apiInterface.mustache @@ -0,0 +1,20 @@ + @{{httpMethod}}{{#subresourceOperation}} + @Path("{{{path}}}"){{/subresourceOperation}}{{#hasConsumes}} + @Consumes({ {{#consumes}}"{{{mediaType}}}"{{^-last}}, {{/-last}}{{/consumes}} }){{/hasConsumes}}{{#hasProduces}} + @Produces({ {{#produces}}"{{{mediaType}}}"{{^-last}}, {{/-last}}{{/produces}} }){{/hasProduces}}{{#useSwaggerAnnotations}} + @ApiOperation(value = "{{{summary}}}", notes = "{{{notes}}}"{{#hasAuthMethods}}, authorizations = { + {{#authMethods}}{{#isOAuth}}@Authorization(value = "{{name}}", scopes = { + {{#scopes}}@AuthorizationScope(scope = "{{scope}}", description = "{{description}}"){{^-last}}, + {{/-last}}{{/scopes}} }){{^-last}},{{/-last}}{{/isOAuth}} + {{^isOAuth}}@Authorization(value = "{{name}}"){{^-last}},{{/-last}} + {{/isOAuth}}{{/authMethods}} }{{/hasAuthMethods}}, tags={ {{#vendorExtensions.x-tags}}"{{tag}}"{{^-last}}, {{/-last}}{{/vendorExtensions.x-tags}} }) + {{#implicitHeadersParams.0}} + @io.swagger.annotations.ApiImplicitParams({ + {{#implicitHeadersParams}} + @io.swagger.annotations.ApiImplicitParam(name = "{{{baseName}}}", value = "{{{description}}}", {{#required}}required = true,{{/required}} dataType = "{{{dataType}}}", paramType = "header"){{^-last}},{{/-last}} + {{/implicitHeadersParams}} + }) + {{/implicitHeadersParams.0}} + @ApiResponses(value = { {{#responses}} + @ApiResponse(code = {{{code}}}, message = "{{{message}}}", response = {{{baseType}}}.class{{#returnContainer}}, responseContainer = "{{{.}}}"{{/returnContainer}}){{^-last}},{{/-last}}{{/responses}} }){{/useSwaggerAnnotations}} + {{#supportAsync}}{{>returnAsyncTypeInterface}}{{/supportAsync}}{{^supportAsync}}{{#returnResponse}}Response{{/returnResponse}}{{^returnResponse}}{{>returnTypeInterface}}{{/returnResponse}}{{/supportAsync}} {{nickname}}({{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{>formParams}}, {{/allParams}}@HeaderParam("X-Endpoint-API-UserInfo") final String userInfo); diff --git a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/beanValidation.mustache b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/beanValidation.mustache new file mode 100644 index 00000000000..b1104b4ef0d --- /dev/null +++ b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/beanValidation.mustache @@ -0,0 +1,2 @@ +{{!Copied from the base JaxRs openapi generator to override the one included in JaxRsSpec}} +{{#required}}@NotNull {{/required}}{{^isPrimitiveType}}{{^isDate}}{{^isDateTime}}{{^isString}}{{^isFile}}{{^isArray}}@Valid {{/isArray}}{{/isFile}}{{/isString}}{{/isDateTime}}{{/isDate}}{{/isPrimitiveType}}{{>beanValidationCore}} diff --git a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/beanValidationQueryParams.mustache b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/beanValidationQueryParams.mustache new file mode 100644 index 00000000000..331fa5bb8c9 --- /dev/null +++ b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/beanValidationQueryParams.mustache @@ -0,0 +1 @@ +{{#required}} @NotNull{{/required}}{{^required}} @Nullable{{/required}}{{>beanValidationCore}} \ No newline at end of file diff --git a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/bodyParams.mustache b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/bodyParams.mustache new file mode 100644 index 00000000000..47d5b52e9d1 --- /dev/null +++ b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/bodyParams.mustache @@ -0,0 +1 @@ +{{#isBodyParam}}{{#useBeanValidation}}@Valid @Body {{#required}}{{^isNullable}}@NotNull {{/isNullable}}{{/required}}{{/useBeanValidation}}{{{dataType}}} {{paramName}}{{/isBodyParam}} diff --git a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/enumOuterClass.mustache b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/enumOuterClass.mustache new file mode 100644 index 00000000000..08ed8518fdf --- /dev/null +++ b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/enumOuterClass.mustache @@ -0,0 +1,61 @@ +import java.util.Arrays; +import java.util.List; +{{#jackson}} +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +{{/jackson}} + +/** + * {{description}}{{^description}}Gets or Sets {{{name}}}{{/description}} + */ +{{>additionalEnumTypeAnnotations}}public enum {{{datatypeWithEnum}}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}} { + {{#gson}} + {{#allowableValues}}{{#enumVars}} + @SerializedName({{#isInteger}}"{{/isInteger}}{{#isDouble}}"{{/isDouble}}{{#isLong}}"{{/isLong}}{{#isFloat}}"{{/isFloat}}{{{value}}}{{#isInteger}}"{{/isInteger}}{{#isDouble}}"{{/isDouble}}{{#isLong}}"{{/isLong}}{{#isFloat}}"{{/isFloat}}) + {{{name}}}({{{value}}}){{^-last}}, + {{/-last}}{{#-last}};{{/-last}}{{/enumVars}}{{/allowableValues}} + {{/gson}} + {{^gson}} + {{#allowableValues}}{{#enumVars}} + {{{name}}}({{{value}}}){{^-last}}, + {{/-last}}{{#-last}};{{/-last}}{{/enumVars}}{{/allowableValues}} + {{/gson}} + + private {{{dataType}}} value; + + {{{datatypeWithEnum}}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}}({{{dataType}}} value) { + this.value = value; + } + + /** + * Convert a String into {{dataType}}, as specified in the + * See JAX RS 2.0 Specification, section 3.2, p. 12 + */ + public static {{#datatypeWithEnum}}{{{.}}}{{/datatypeWithEnum}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}} fromString(String s) { + for ({{#datatypeWithEnum}}{{{.}}}{{/datatypeWithEnum}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}} b : {{#datatypeWithEnum}}{{{.}}}{{/datatypeWithEnum}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}}.values()) { + // using Objects.toString() to be safe if value type non-object type + // because types like 'int' etc. will be auto-boxed + if (java.util.Objects.toString(b.value).equals(s)) { + return b; + } + } + {{#isNullable}}return null;{{/isNullable}}{{^isNullable}}throw new IllegalArgumentException("Unexpected string value '" + s + "'");{{/isNullable}} + } + + @Override + @JsonValue + public String toString() { + return String.valueOf(value); + } + + @JsonCreator + public static {{{datatypeWithEnum}}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}} fromValue({{{dataType}}} value) { + for ({{{datatypeWithEnum}}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}} b : {{{datatypeWithEnum}}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}}.values()) { + if (b.value.equals(value)) { + return b; + } + } + {{!Update here to return a human readable error message}} + {{#isNullable}}return null;{{/isNullable}}{{^isNullable}}throw new IllegalArgumentException("Unexpected value '" + value + "' for field '%s'. Allowed values are: " + Arrays.stream({{{datatypeWithEnum}}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}}.values()).map({{{datatypeWithEnum}}}{{^datatypeWithEnum}}{{{classname}}}{{/datatypeWithEnum}}::toString).toList());{{/isNullable}} + } +} diff --git a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/pojo.mustache b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/pojo.mustache new file mode 100644 index 00000000000..98a600144e1 --- /dev/null +++ b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/pojo.mustache @@ -0,0 +1,199 @@ +{{#useSwaggerAnnotations}} +import io.swagger.annotations.*; +{{/useSwaggerAnnotations}} +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.annotation.JsonTypeName; + +{{#discriminator}}{{>typeInfoAnnotation}}{{/discriminator}}{{#description}}/** + * {{.}} + **/{{/description}} +{{#useSwaggerAnnotations}}{{#description}}@ApiModel(description = "{{{.}}}"){{/description}}{{/useSwaggerAnnotations}} +@JsonTypeName("{{name}}") +{{>generatedAnnotation}}{{>additionalModelTypeAnnotations}} +{{#vendorExtensions.x-class-extra-annotation}} +{{{vendorExtensions.x-class-extra-annotation}}} +{{/vendorExtensions.x-class-extra-annotation}} +public class {{classname}} {{#parent}}extends {{{.}}}{{/parent}} {{#vendorExtensions.x-implements}}{{#-first}}implements {{{.}}}{{/-first}}{{^-first}}, {{{.}}}{{/-first}}{{/vendorExtensions.x-implements}} { + {{#vars}} + {{#isEnum}} + {{^isContainer}} + {{>enumClass}} + {{/isContainer}} + {{#isContainer}} + {{#mostInnerItems}} + {{>enumClass}} + {{/mostInnerItems}} + {{/isContainer}} + {{/isEnum}} + {{#vendorExtensions.x-field-extra-annotation}} + {{{vendorExtensions.x-field-extra-annotation}}} + {{/vendorExtensions.x-field-extra-annotation}} + {{!This is the important line, instead of just checking if useBeanValidation and then adding @Valid, we use the template itself}} + {{!This avoids us using @Valid on things like String etc.}} + private {{#useBeanValidation}}{{>beanValidation}}{{/useBeanValidation}}{{{datatypeWithEnum}}} {{name}}{{#defaultValue}} = {{{.}}}{{/defaultValue}}; + {{/vars}} + {{#generateBuilders}} + {{^additionalProperties}} + + protected {{classname}}({{classname}}Builder b) { + {{#parent}} + super(b); + {{/parent}} + {{#vars}} + this.{{name}} = b.{{name}}; + {{/vars}} + } + + public {{classname}}() { + } + {{/additionalProperties}} + {{/generateBuilders}} + + {{#vars}} + /** + {{#description}} + * {{.}} + {{/description}} + {{#minimum}} + * minimum: {{.}} + {{/minimum}} + {{#maximum}} + * maximum: {{.}} + {{/maximum}} + **/ + public {{classname}} {{name}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + return this; + } + + {{#vendorExtensions.x-extra-annotation}}{{{vendorExtensions.x-extra-annotation}}}{{/vendorExtensions.x-extra-annotation}}{{#useSwaggerAnnotations}} + @ApiModelProperty({{#example}}example = "{{{.}}}", {{/example}}{{#required}}required = {{required}}, {{/required}}value = "{{{description}}}"){{/useSwaggerAnnotations}} + @JsonProperty("{{baseName}}") +{{#useBeanValidation}}{{>beanValidation}}{{/useBeanValidation}} public {{>beanValidatedType}} {{getter}}() { + return {{name}}; + } + + @JsonProperty("{{baseName}}") + {{#vendorExtensions.x-setter-extra-annotation}}{{{vendorExtensions.x-setter-extra-annotation}}} + {{/vendorExtensions.x-setter-extra-annotation}}public void {{setter}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + } + + {{#isArray}} + public {{classname}} add{{nameInCamelCase}}Item({{{items.datatypeWithEnum}}} {{name}}Item) { + if (this.{{name}} == null) { + this.{{name}} = {{{defaultValue}}}{{^defaultValue}}new {{#uniqueItems}}LinkedHashSet{{/uniqueItems}}{{^uniqueItems}}ArrayList{{/uniqueItems}}<>(){{/defaultValue}}; + } + + this.{{name}}.add({{name}}Item); + return this; + } + + public {{classname}} remove{{nameInCamelCase}}Item({{{items.datatypeWithEnum}}} {{name}}Item) { + if ({{name}}Item != null && this.{{name}} != null) { + this.{{name}}.remove({{name}}Item); + } + + return this; + } + {{/isArray}} + {{#isMap}} + public {{classname}} put{{nameInCamelCase}}Item(String key, {{{items.datatypeWithEnum}}} {{name}}Item) { + if (this.{{name}} == null) { + this.{{name}} = {{{defaultValue}}}{{^defaultValue}}new HashMap<>(){{/defaultValue}}; + } + + this.{{name}}.put(key, {{name}}Item); + return this; + } + + public {{classname}} remove{{nameInCamelCase}}Item({{{items.datatypeWithEnum}}} {{name}}Item) { + if ({{name}}Item != null && this.{{name}} != null) { + this.{{name}}.remove({{name}}Item); + } + + return this; + } + {{/isMap}} + {{/vars}} + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + }{{#hasVars}} + {{classname}} {{classVarName}} = ({{classname}}) o; + return {{#vars}}{{#isByteArray}}Arrays{{/isByteArray}}{{^isByteArray}}Objects{{/isByteArray}}.equals(this.{{name}}, {{classVarName}}.{{name}}){{^-last}} && + {{/-last}}{{/vars}}{{#parent}} && + super.equals(o){{/parent}};{{/hasVars}}{{^hasVars}} + return {{#parent}}super.equals(o){{/parent}}{{^parent}}true{{/parent}};{{/hasVars}} + } + + @Override + public int hashCode() { + return Objects.hash({{#vars}}{{^isByteArray}}{{name}}{{/isByteArray}}{{#isByteArray}}Arrays.hashCode({{name}}){{/isByteArray}}{{^-last}}, {{/-last}}{{/vars}}{{#parent}}{{#hasVars}}, {{/hasVars}}super.hashCode(){{/parent}}); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class {{classname}} {\n"); + {{#parent}}sb.append(" ").append(toIndentedString(super.toString())).append("\n");{{/parent}} + {{#vars}}sb.append(" {{name}}: ").append(toIndentedString({{name}})).append("\n"); + {{/vars}}sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + +{{#generateBuilders}}{{^additionalProperties}} + public static {{classname}}Builder builder() { + return new {{classname}}BuilderImpl(); + } + + private static final class {{classname}}BuilderImpl extends {{classname}}Builder<{{classname}}, {{classname}}BuilderImpl> { + + @Override + protected {{classname}}BuilderImpl self() { + return this; + } + + @Override + public {{classname}} build() { + return new {{classname}}(this); + } + } + + public static abstract class {{classname}}Builder> {{#parent}}extends {{{.}}}Builder{{/parent}} { + {{#vars}} + private {{{datatypeWithEnum}}} {{name}}{{#defaultValue}} = {{{.}}}{{/defaultValue}}; + {{/vars}} + {{^parent}} + protected abstract B self(); + + public abstract C build(); + {{/parent}} + + {{#vars}} + public B {{name}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + return self(); + } + {{/vars}} + }{{/additionalProperties}}{{/generateBuilders}} +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/config/WorkerConfigsProvider.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/config/WorkerConfigsProvider.java index c21d14a5e39..dccab9a6810 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/config/WorkerConfigsProvider.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/config/WorkerConfigsProvider.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.regex.Matcher; @@ -186,7 +187,10 @@ record WorkerConfigsDefaults(WorkerEnvironment workerEnvironment, record KubeResourceKey(String variant, ResourceType type, ResourceSubType subType) {} public WorkerConfigsProvider(final List kubeResourceConfigs, final WorkerConfigsDefaults defaults) { - this.kubeResourceKeyPattern = Pattern.compile(String.format("^((?[a-z]+)-)?(?%s)(-(?%s))?$", + // In the variant name, we do not support uppercase. This is because micronaut normalizes uppercases + // with dashes (CamelCase becomes camel-case) which is confusing because the variant name no longer + // matches the config file. + this.kubeResourceKeyPattern = Pattern.compile(String.format("^((?[a-z0-9]+)-)?(?%s)(-(?%s))?$", String.join("|", Arrays.stream(ResourceType.values()).map(ResourceType::toString).toList()), String.join("|", Arrays.stream(ResourceSubType.values()).map(ResourceSubType::toString).toList())), Pattern.CASE_INSENSITIVE); @@ -221,7 +225,9 @@ public WorkerConfigs getConfig(final ResourceType name) { * @return the WorkerConfig. */ private WorkerConfigs getConfig(final KubeResourceKey key) { - final KubeResourceConfig kubeResourceConfig = getKubeResourceConfig(key).orElseThrow(); + final KubeResourceConfig kubeResourceConfig = getKubeResourceConfig(key) + .orElseThrow(() -> new NoSuchElementException(String.format("Unable to find config: {variant:%s, type:%s, subtype:%s}", + key.variant, key.type, key.subType))); final Map isolatedNodeSelectors = splitKVPairsFromEnvString(workerConfigsDefaults.isolatedNodeSelectors); validateIsolatedPoolConfigInitialization(workerConfigsDefaults.useCustomNodeSelector(), isolatedNodeSelectors); @@ -269,18 +275,51 @@ public ResourceRequirements getResourceRequirements(final ResourceRequirementsTy return getConfig(key).getResourceRequirements(); } + /** + * Look up resource configs given a key. + *

+ * We are storing configs in a tree like structure. Look up should be handled as such. Keeping in + * mind that we have defaults we want to fallback to, we should perform a complete scan of the + * configs until we find a match to make sure we do not overlook a match. + */ private Optional getKubeResourceConfig(final KubeResourceKey key) { - final Map> typeMap = getOrElseGet(kubeResourceConfigs, key.variant, DEFAULT_VARIANT); - if (typeMap == null) { + // Look up by actual variant + final var resultWithVariant = getKubeResourceConfigByType(kubeResourceConfigs.get(key.variant), key); + if (resultWithVariant.isPresent()) { + return resultWithVariant; + } + + // no match with exact variant found, try again with the default. + return getKubeResourceConfigByType(kubeResourceConfigs.get(DEFAULT_VARIANT), key); + } + + private static Optional getKubeResourceConfigByType( + final Map> configs, + final KubeResourceKey key) { + if (configs == null) { return Optional.empty(); } - final Map subTypeMap = getOrElseGet(typeMap, key.type, ResourceType.DEFAULT); - if (subTypeMap == null) { + // Look up by actual type + final var resultWithType = getKubeResourceConfigBySubType(configs.get(key.type), key); + if (resultWithType.isPresent()) { + return resultWithType; + } + + // no match with exact type found, try again with the default. + return getKubeResourceConfigBySubType(configs.get(ResourceType.DEFAULT), key); + } + + private static Optional getKubeResourceConfigBySubType(final Map configBySubType, + final KubeResourceKey key) { + if (configBySubType == null) { return Optional.empty(); } - return Optional.ofNullable(getOrElseGet(subTypeMap, key.subType, ResourceSubType.DEFAULT)); + // Lookup by actual sub type + final var config = configBySubType.get(key.subType); + // if we didn't find a match, try again with the default + return Optional.ofNullable(config != null ? config : configBySubType.get(ResourceSubType.DEFAULT)); } private void validateIsolatedPoolConfigInitialization(boolean useCustomNodeSelector, Map isolatedNodeSelectors) { @@ -338,16 +377,6 @@ private ResourceRequirements getResourceRequirementsFrom(final KubeResourceConfi .withMemoryRequest(useDefaultIfEmpty(kubeResourceConfig.getMemoryRequest(), defaultConfig.getMemoryRequest())); } - /** - * Helper function to get from a map. - *

- * Returns map.get(key) if key is present else returns map.get(fallbackKey) - */ - private static ValueT getOrElseGet(final Map map, final KeyT key, final KeyT fallbackKey) { - final ValueT lookup1 = map.get(key); - return lookup1 != null ? lookup1 : map.get(fallbackKey); - } - private static String useDefaultIfEmpty(final String value, final String defaultValue) { return (value == null || value.isBlank()) ? defaultValue : value; } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java index 6be7390355a..c44d8507422 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java @@ -5,6 +5,7 @@ package io.airbyte.workers.general; import io.airbyte.commons.concurrency.BoundedConcurrentLinkedQueue; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.converters.ThreadedTimeTracker; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.timer.Stopwatch; @@ -101,13 +102,14 @@ public BufferedReplicationWorker(final String jobId, final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper) { + final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, + final VoidCallable onReplicationRunning) { this.jobId = jobId; this.attempt = attempt; this.source = source; this.destination = destination; this.replicationWorkerHelper = new ReplicationWorkerHelper(airbyteMessageDataExtractor, fieldSelector, mapper, messageTracker, syncPersistence, - replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker()); + replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning); this.replicationFeatureFlagReader = replicationFeatureFlagReader; this.recordSchemaValidator = recordSchemaValidator; this.syncPersistence = syncPersistence; @@ -151,6 +153,8 @@ public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRo runAsync(() -> replicationWorkerHelper.startDestination(destination, syncInput, jobRoot), mdc), runAsync(() -> replicationWorkerHelper.startSource(source, syncInput, jobRoot), mdc)).join(); + replicationWorkerHelper.markReplicationRunning(); + CompletableFuture.allOf( runAsyncWithHeartbeatCheck(this::readFromSource, mdc), runAsync(this::processMessage, mdc), @@ -158,6 +162,7 @@ public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRo runAsync(this::readFromDestination, mdc)).join(); } catch (final CompletionException e) { + // Exceptions for each runnable are already handled, those exceptions are coming from the joins and // are safe to ignore at this point ApmTraceUtils.addExceptionToTrace(e); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DbtTransformationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DbtTransformationWorker.java index bca45ebc819..b42deed9775 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DbtTransformationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DbtTransformationWorker.java @@ -9,6 +9,7 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; import datadog.trace.api.Trace; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.io.LineGobbler; import io.airbyte.config.OperatorDbtInput; import io.airbyte.config.ResourceRequirements; @@ -37,15 +38,18 @@ public class DbtTransformationWorker implements Worker { private final ResourceRequirements resourceRequirements; private final AtomicBoolean cancelled; + private final VoidCallable onTransformationRunning; public DbtTransformationWorker(final String jobId, final int attempt, final ResourceRequirements resourceRequirements, - final DbtTransformationRunner dbtTransformationRunner) { + final DbtTransformationRunner dbtTransformationRunner, + final VoidCallable onTransformationRunning) { this.jobId = jobId; this.attempt = attempt; this.dbtTransformationRunner = dbtTransformationRunner; this.resourceRequirements = resourceRequirements; + this.onTransformationRunning = onTransformationRunning; this.cancelled = new AtomicBoolean(false); } @@ -60,6 +64,7 @@ public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) thr try (dbtTransformationRunner) { LOGGER.info("Running dbt transformation."); dbtTransformationRunner.start(); + onTransformationRunning.call(); final Path transformRoot = Files.createDirectories(jobRoot.resolve("transform")); if (!dbtTransformationRunner.run( jobId, diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultNormalizationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultNormalizationWorker.java index e110e58c7cd..a7e183d8a0f 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultNormalizationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultNormalizationWorker.java @@ -9,6 +9,7 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; import datadog.trace.api.Trace; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.io.LineGobbler; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.FailureReason; @@ -44,6 +45,7 @@ public class DefaultNormalizationWorker implements NormalizationWorker { private final NormalizationRunner normalizationRunner; private final WorkerEnvironment workerEnvironment; private final List traceFailureReasons = new ArrayList<>(); + private final VoidCallable onNormalizationRunning; private boolean failed = false; private final AtomicBoolean cancelled; @@ -51,11 +53,13 @@ public class DefaultNormalizationWorker implements NormalizationWorker { public DefaultNormalizationWorker(final String jobId, final int attempt, final NormalizationRunner normalizationRunner, - final WorkerEnvironment workerEnvironment) { + final WorkerEnvironment workerEnvironment, + final VoidCallable onNormalizationRunning) { this.jobId = jobId; this.attempt = attempt; this.normalizationRunner = normalizationRunner; this.workerEnvironment = workerEnvironment; + this.onNormalizationRunning = onNormalizationRunning; this.cancelled = new AtomicBoolean(false); } @@ -70,7 +74,7 @@ public NormalizationSummary run(final NormalizationInput input, final Path jobRo try (normalizationRunner) { LineGobbler.startSection("DEFAULT NORMALIZATION"); normalizationRunner.start(); - + onNormalizationRunning.call(); Path normalizationRoot = null; // There are no shared volumes on Kube; only create this for Docker. if (workerEnvironment.equals(WorkerEnvironment.DOCKER)) { diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index 349acdb8aee..f39174c02b3 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -7,6 +7,7 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; import datadog.trace.api.Trace; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.converters.ThreadedTimeTracker; import io.airbyte.commons.io.LineGobbler; import io.airbyte.config.ReplicationOutput; @@ -90,11 +91,12 @@ public DefaultReplicationWorker(final String jobId, final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper) { + final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, + final VoidCallable onReplicationRunning) { this.jobId = jobId; this.attempt = attempt; this.replicationWorkerHelper = new ReplicationWorkerHelper(airbyteMessageDataExtractor, fieldSelector, mapper, messageTracker, syncPersistence, - replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker()); + replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning); this.source = source; this.destination = destination; this.syncPersistence = syncPersistence; @@ -162,6 +164,8 @@ private void replicate(final Path jobRoot, replicationWorkerHelper.startDestination(destination, syncInput, jobRoot); replicationWorkerHelper.startSource(source, syncInput, jobRoot); + replicationWorkerHelper.markReplicationRunning(); + // note: `whenComplete` is used instead of `exceptionally` so that the original exception is still // thrown final CompletableFuture readFromDstThread = CompletableFuture.runAsync( diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java index e64f4cbcd42..693e3a5b18b 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java @@ -11,6 +11,7 @@ import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.SourceDefinitionIdRequestBody; import io.airbyte.api.client.model.generated.SourceIdRequestBody; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.SyncResourceRequirements; @@ -112,7 +113,8 @@ public ReplicationWorkerFactory( public ReplicationWorker create(final StandardSyncInput syncInput, final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, - final IntegrationLauncherConfig destinationLauncherConfig) + final IntegrationLauncherConfig destinationLauncherConfig, + final VoidCallable onReplicationRunning) throws ApiException { final UUID sourceDefinitionId = AirbyteApiClient.retryWithJitter( () -> sourceApi.getSource( @@ -158,7 +160,8 @@ public ReplicationWorker create(final StandardSyncInput syncInput, return createReplicationWorker(airbyteSource, airbyteDestination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, heartbeatTimeoutChaperone, - featureFlagClient, jobRunConfig, syncInput, airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper); + featureFlagClient, jobRunConfig, syncInput, airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, + onReplicationRunning); } /** @@ -267,7 +270,8 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou final JobRunConfig jobRunConfig, final StandardSyncInput syncInput, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper replicationEventPublishingHelper) { + final ReplicationAirbyteMessageEventPublishingHelper replicationEventPublishingHelper, + final VoidCallable onReplicationRunning) { final Context flagContext = getFeatureFlagContext(syncInput); final String workerImpl = featureFlagClient.stringVariation(ReplicationWorkerImpl.INSTANCE, flagContext); return buildReplicationWorkerInstance( @@ -284,7 +288,8 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou heartbeatTimeoutChaperone, new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, - replicationEventPublishingHelper); + replicationEventPublishingHelper, + onReplicationRunning); } private static Context getFeatureFlagContext(final StandardSyncInput syncInput) { @@ -317,19 +322,20 @@ private static ReplicationWorker buildReplicationWorkerInstance(final String wor final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper) { + final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, + final VoidCallable onReplicationRunning) { if ("buffered".equals(workerImpl)) { MetricClientFactory.getMetricClient() .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper); + messageEventPublishingHelper, onReplicationRunning); } else { MetricClientFactory.getMetricClient() .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper); + messageEventPublishingHelper, onReplicationRunning); } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java index 017771cc94c..f07fd972a59 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import io.airbyte.api.client.model.generated.StreamStatusIncompleteRunCause; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.converters.ThreadedTimeTracker; import io.airbyte.commons.io.LineGobbler; import io.airbyte.config.FailureReason; @@ -72,6 +73,7 @@ class ReplicationWorkerHelper { private final SyncPersistence syncPersistence; private final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper; private final ThreadedTimeTracker timeTracker; + private final VoidCallable onReplicationRunning; private long recordsRead; private StreamDescriptor currentDestinationStream = null; private ReplicationContext replicationContext = null; @@ -91,7 +93,8 @@ public ReplicationWorkerHelper( final MessageTracker messageTracker, final SyncPersistence syncPersistence, final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, - final ThreadedTimeTracker timeTracker) { + final ThreadedTimeTracker timeTracker, + final VoidCallable onReplicationRunning) { this.airbyteMessageDataExtractor = airbyteMessageDataExtractor; this.fieldSelector = fieldSelector; this.mapper = mapper; @@ -99,6 +102,7 @@ public ReplicationWorkerHelper( this.syncPersistence = syncPersistence; this.replicationAirbyteMessageEventPublishingHelper = replicationAirbyteMessageEventPublishingHelper; this.timeTracker = timeTracker; + this.onReplicationRunning = onReplicationRunning; this.recordsRead = 0L; } @@ -140,6 +144,11 @@ public void startSource(final AirbyteSource source, final StandardSyncInput sync } } + public void markReplicationRunning() throws Exception { + // Calls the onReplicationRunning callback, which should mark the replication as running. + onReplicationRunning.call(); + } + public void endOfReplication() { // Publish a complete status event for all streams associated with the connection. // This is to ensure that all streams end up in a terminal state and is necessary for diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/InMemoryOrchestratorHandleFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/InMemoryOrchestratorHandleFactory.java index 5c0328940a3..eb1de782b59 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/InMemoryOrchestratorHandleFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/InMemoryOrchestratorHandleFactory.java @@ -39,7 +39,11 @@ public CheckedSupplier, Exception> JobRunConfig jobRunConfig, StandardSyncInput syncInput, final Supplier activityContext) { - return () -> replicationWorkerFactory.create(syncInput, jobRunConfig, sourceLauncherConfig, destinationLauncherConfig); + return () -> replicationWorkerFactory.create(syncInput, jobRunConfig, sourceLauncherConfig, destinationLauncherConfig, /* + * this is used to track + * async state, but we + * don't do that in-memory + */() -> {}); } } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java index 9709024df91..6275401f663 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java @@ -4,22 +4,15 @@ package io.airbyte.workers.general; -import static java.lang.Thread.sleep; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.workers.internal.AirbyteSource; import io.airbyte.workers.internal.FieldSelector; -import java.util.Optional; import org.junit.jupiter.api.Test; -import org.mockito.stubbing.Answer; /** * BufferedReplicationWorkerTests. Tests in this class should be implementation specific, general @@ -43,7 +36,8 @@ ReplicationWorker getDefaultReplicationWorker(final boolean fieldSelectionEnable heartbeatTimeoutChaperone, new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, - replicationAirbyteMessageEventPublishingHelper); + replicationAirbyteMessageEventPublishingHelper, + onReplicationRunning); } // BufferedReplicationWorkerTests. @@ -102,12 +96,7 @@ void testClosurePropagationWhenCrashInReadFromDestination() throws Exception { } protected void setUpInfiniteSource() { - source = mock(AirbyteSource.class); - when(source.isFinished()).thenReturn(false); - when(source.attemptRead()).thenAnswer((Answer>) invocation -> { - sleep(100); - return Optional.of(RECORD_MESSAGE1); - }); + sourceStub.setInfiniteSourceWithMessages(RECORD_MESSAGE1); } } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultNormalizationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultNormalizationWorkerTest.java index 4ba86a4ddc5..1ad06fa8109 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultNormalizationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultNormalizationWorkerTest.java @@ -12,6 +12,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.EnvConfigs; import io.airbyte.config.FailureReason.FailureOrigin; @@ -48,6 +49,7 @@ class DefaultNormalizationWorkerTest { private Path normalizationRoot; private NormalizationInput normalizationInput; private NormalizationRunner normalizationRunner; + private VoidCallable onNormalizationRunning; @BeforeEach void setup() throws Exception { @@ -74,16 +76,19 @@ void setup() throws Exception { normalizationInput.getDestinationConfiguration(), normalizationInput.getCatalog(), workerConfigs.getResourceRequirements())) .thenReturn(true); + + onNormalizationRunning = mock(VoidCallable.class); } @Test void test() throws Exception { final DefaultNormalizationWorker normalizationWorker = - new DefaultNormalizationWorker(JOB_ID, JOB_ATTEMPT, normalizationRunner, WorkerEnvironment.DOCKER); + new DefaultNormalizationWorker(JOB_ID, JOB_ATTEMPT, normalizationRunner, WorkerEnvironment.DOCKER, onNormalizationRunning); final NormalizationSummary normalizationOutput = normalizationWorker.run(normalizationInput, jobRoot); verify(normalizationRunner).start(); + verify(onNormalizationRunning).call(); verify(normalizationRunner).normalize( JOB_ID, JOB_ATTEMPT, @@ -111,7 +116,7 @@ void testFailure() throws Exception { .thenReturn(false); final DefaultNormalizationWorker normalizationWorker = - new DefaultNormalizationWorker(JOB_ID, JOB_ATTEMPT, normalizationRunner, WorkerEnvironment.DOCKER); + new DefaultNormalizationWorker(JOB_ID, JOB_ATTEMPT, normalizationRunner, WorkerEnvironment.DOCKER, () -> {}); assertThrows(WorkerException.class, () -> normalizationWorker.run(normalizationInput, jobRoot)); @@ -135,11 +140,12 @@ void testFailureWithTraceMessage() throws Exception { when(normalizationRunner.getTraceMessages()).thenReturn(Stream.of(ERROR_TRACE_MESSAGE)); final DefaultNormalizationWorker normalizationWorker = - new DefaultNormalizationWorker(JOB_ID, JOB_ATTEMPT, normalizationRunner, WorkerEnvironment.DOCKER); + new DefaultNormalizationWorker(JOB_ID, JOB_ATTEMPT, normalizationRunner, WorkerEnvironment.DOCKER, onNormalizationRunning); final NormalizationSummary normalizationOutput = normalizationWorker.run(normalizationInput, jobRoot); verify(normalizationRunner).start(); + verify(onNormalizationRunning).call(); verify(normalizationRunner).normalize( JOB_ID, JOB_ATTEMPT, diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java index c4ba935e1eb..3423b90f2e4 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java @@ -30,7 +30,8 @@ ReplicationWorker getDefaultReplicationWorker(final boolean fieldSelectionEnable heartbeatTimeoutChaperone, new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, - replicationAirbyteMessageEventPublishingHelper); + replicationAirbyteMessageEventPublishingHelper, + onReplicationRunning); } // DefaultReplicationWorkerTests. diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java index 439458d6905..91aff94e20e 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.api.client.model.generated.StreamStatusIncompleteRunCause; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; @@ -71,6 +72,7 @@ import io.airbyte.workers.internal.HeartbeatTimeoutChaperone; import io.airbyte.workers.internal.NamespacingMapper; import io.airbyte.workers.internal.SimpleAirbyteDestination; +import io.airbyte.workers.internal.SimpleAirbyteSource; import io.airbyte.workers.internal.book_keeping.AirbyteMessageOrigin; import io.airbyte.workers.internal.book_keeping.AirbyteMessageTracker; import io.airbyte.workers.internal.book_keeping.SyncStatsTracker; @@ -106,7 +108,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; -import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -137,6 +138,7 @@ abstract class ReplicationWorkerTest { protected static final String INDUCED_EXCEPTION = "induced exception"; protected Path jobRoot; + protected SimpleAirbyteSource sourceStub; protected AirbyteSource source; protected NamespacingMapper mapper; protected AirbyteDestination destination; @@ -155,6 +157,8 @@ abstract class ReplicationWorkerTest { protected FeatureFlagClient featureFlagClient; protected AirbyteMessageDataExtractor airbyteMessageDataExtractor; + protected VoidCallable onReplicationRunning; + ReplicationWorker getDefaultReplicationWorker() { return getDefaultReplicationWorker(false); } @@ -175,7 +179,10 @@ void setup() throws Exception { sourceConfig = WorkerUtils.syncToWorkerSourceConfig(syncInput); destinationConfig = WorkerUtils.syncToWorkerDestinationConfig(syncInput); - source = mock(AirbyteSource.class); + sourceStub = new SimpleAirbyteSource(); + sourceStub.setMessages(RECORD_MESSAGE1, RECORD_MESSAGE2); + source = spy(sourceStub); + mapper = mock(NamespacingMapper.class); destination = spy(new SimpleAirbyteDestination()); messageTracker = mock(AirbyteMessageTracker.class); @@ -186,6 +193,7 @@ void setup() throws Exception { metricClient = MetricClientFactory.getMetricClient(); workerMetricReporter = new WorkerMetricReporter(metricClient, "docker_image:v1.0.0"); airbyteMessageDataExtractor = new AirbyteMessageDataExtractor(); + onReplicationRunning = mock(VoidCallable.class); final HeartbeatMonitor heartbeatMonitor = mock(HeartbeatMonitor.class); heartbeatTimeoutChaperone = new HeartbeatTimeoutChaperone(heartbeatMonitor, Duration.ofMinutes(5), null, null, null, metricClient); @@ -193,8 +201,6 @@ void setup() throws Exception { featureFlagClient = mock(TestClient.class); when(messageTracker.getSyncStatsTracker()).thenReturn(syncStatsTracker); - when(source.isFinished()).thenReturn(false, false, false, true); - when(source.attemptRead()).thenReturn(Optional.of(RECORD_MESSAGE1), Optional.empty(), Optional.of(RECORD_MESSAGE2)); when(mapper.mapCatalog(destinationConfig.getCatalog())).thenReturn(destinationConfig.getCatalog()); when(mapper.mapMessage(RECORD_MESSAGE1)).thenReturn(RECORD_MESSAGE1); @@ -217,6 +223,7 @@ void test() throws Exception { verify(source).start(sourceConfig, jobRoot); verify(destination).start(destinationConfig, jobRoot); + verify(onReplicationRunning).call(); verify(destination).accept(RECORD_MESSAGE1); verify(destination).accept(RECORD_MESSAGE2); verify(source, atLeastOnce()).close(); @@ -395,7 +402,7 @@ void testDestinationWriteExceptionWithStreamStatus(final boolean isReset) throws @Test void testInvalidSchema() throws Exception { - when(source.attemptRead()).thenReturn(Optional.of(RECORD_MESSAGE1), Optional.of(RECORD_MESSAGE2), Optional.of(RECORD_MESSAGE3)); + sourceStub.setMessages(RECORD_MESSAGE1, RECORD_MESSAGE2, RECORD_MESSAGE3); final ReplicationWorker worker = getDefaultReplicationWorker(); @@ -485,7 +492,7 @@ void testReplicationRunnableSourceFailure() throws Exception { @Test void testReplicationRunnableSourceUpdateConfig() throws Exception { - when(source.attemptRead()).thenReturn(Optional.of(RECORD_MESSAGE1), Optional.of(CONFIG_MESSAGE), Optional.empty()); + sourceStub.setMessages(RECORD_MESSAGE1, CONFIG_MESSAGE); final ReplicationWorker worker = getDefaultReplicationWorker(); @@ -499,8 +506,7 @@ CONFIG_MESSAGE, new ReplicationContext(false, syncInput.getConnectionId(), syncI @Test void testSourceConfigPersistError() throws Exception { - when(source.attemptRead()).thenReturn(Optional.of(CONFIG_MESSAGE)); - when(source.isFinished()).thenReturn(false, true); + sourceStub.setMessages(CONFIG_MESSAGE); final String persistErrorMessage = "there was a problem persisting the new config"; doThrow(new RuntimeException(persistErrorMessage)) @@ -595,9 +601,7 @@ void testOnlyStateAndRecordMessagesDeliveredToDestination() throws Exception { final AirbyteMessage traceMessage = AirbyteMessageUtils.createErrorMessage("a trace message", 123456.0); when(mapper.mapMessage(logMessage)).thenReturn(logMessage); when(mapper.mapMessage(traceMessage)).thenReturn(traceMessage); - when(source.isFinished()).thenReturn(false, false, false, false, true); - when(source.attemptRead()).thenReturn(Optional.of(RECORD_MESSAGE1), Optional.of(logMessage), Optional.of(traceMessage), - Optional.of(RECORD_MESSAGE2)); + sourceStub.setMessages(RECORD_MESSAGE1, logMessage, traceMessage, RECORD_MESSAGE2); final ReplicationWorker worker = getDefaultReplicationWorker(); @@ -617,8 +621,7 @@ void testOnlySelectedFieldsDeliveredToDestinationWithFieldSelectionEnabled() thr final AirbyteMessage recordWithExtraFields = Jsons.clone(RECORD_MESSAGE1); ((ObjectNode) recordWithExtraFields.getRecord().getData()).put("AnUnexpectedField", "SomeValue"); when(mapper.mapMessage(recordWithExtraFields)).thenReturn(recordWithExtraFields); - when(source.attemptRead()).thenReturn(Optional.of(recordWithExtraFields)); - when(source.isFinished()).thenReturn(false, true); + sourceStub.setMessages(recordWithExtraFields); // Use a real schema validator to make sure validation doesn't affect this. final String streamName = sourceConfig.getCatalog().getStreams().get(0).getStream().getName(); final String streamNamespace = sourceConfig.getCatalog().getStreams().get(0).getStream().getNamespace(); @@ -639,8 +642,7 @@ void testAllFieldsDeliveredWithFieldSelectionDisabled() throws Exception { final AirbyteMessage recordWithExtraFields = Jsons.clone(RECORD_MESSAGE1); ((ObjectNode) recordWithExtraFields.getRecord().getData()).put("AnUnexpectedField", "SomeValue"); when(mapper.mapMessage(recordWithExtraFields)).thenReturn(recordWithExtraFields); - when(source.attemptRead()).thenReturn(Optional.of(recordWithExtraFields)); - when(source.isFinished()).thenReturn(false, true); + sourceStub.setMessages(recordWithExtraFields); // Use a real schema validator to make sure validation doesn't affect this. final String streamName = sourceConfig.getCatalog().getStreams().get(0).getStream().getName(); final String streamNamespace = sourceConfig.getCatalog().getStreams().get(0).getStream().getNamespace(); @@ -740,7 +742,7 @@ void testLogMaskRegex() throws IOException { @Test void testCancellation() throws InterruptedException { final AtomicReference output = new AtomicReference<>(); - when(source.isFinished()).thenReturn(false); + sourceStub.setInfiniteSourceWithMessages(RECORD_MESSAGE1, STATE_MESSAGE); final ReplicationWorker worker = getDefaultReplicationWorker(); @@ -931,12 +933,7 @@ void testSourceFailingTimeout() throws Exception { heartbeatTimeoutChaperone = new HeartbeatTimeoutChaperone(heartbeatMonitor, Duration.ofMillis(1), new TestClient(Map.of("heartbeat.failSync", true)), UUID.randomUUID(), connectionId, mMetricClient); - source = mock(AirbyteSource.class); - when(source.isFinished()).thenReturn(false); - when(source.attemptRead()).thenAnswer((Answer>) invocation -> { - sleep(100); - return Optional.of(RECORD_MESSAGE1); - }); + sourceStub.setInfiniteSourceWithMessages(RECORD_MESSAGE1); final ReplicationWorker worker = getDefaultReplicationWorker(); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java index d3c421747ca..a4b96f1db53 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java @@ -40,7 +40,7 @@ public ReplicationWorker getReplicationWorker(final String jobId, final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper) { return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper); + messageEventPublishingHelper, () -> {}); } public static void main(final String[] args) throws IOException, InterruptedException { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java index d7b5dfaad1e..4ba578546b5 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java @@ -40,7 +40,7 @@ public ReplicationWorker getReplicationWorker(final String jobId, final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper) { return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper); + messageEventPublishingHelper, /* we don't care about the onReplicationRunning callback here */ () -> {}); } public static void main(final String[] args) throws IOException, InterruptedException { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteDestination.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteDestination.java index 41c5e9dbbac..3f18e9e7bdd 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteDestination.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteDestination.java @@ -12,7 +12,7 @@ import java.util.concurrent.LinkedBlockingQueue; /** - * Simple in memory implemenation of an AirbyteDestination for testing purpose. + * Simple in memory implementation of an AirbyteDestination for testing purpose. */ public class SimpleAirbyteDestination implements AirbyteDestination { @@ -36,7 +36,7 @@ public void notifyEndOfInput() throws Exception { @Override public boolean isFinished() { - return isFinished; + return isFinished && messages.isEmpty(); } @Override diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteDestinationTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteDestinationTest.java new file mode 100644 index 00000000000..d375edf9648 --- /dev/null +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteDestinationTest.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.workers.test_utils.AirbyteMessageUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class SimpleAirbyteDestinationTest { + + protected static final String STREAM_NAME = "stream1"; + protected static final String FIELD_NAME = "field1"; + + protected static final AirbyteMessage RECORD_MESSAGE1 = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "m1"); + protected static final AirbyteMessage RECORD_MESSAGE2 = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "m2"); + protected static final AirbyteMessage STATE_MESSAGE1 = AirbyteMessageUtils.createStateMessage(STREAM_NAME, "checkpoint", "1"); + protected static final AirbyteMessage STATE_MESSAGE2 = AirbyteMessageUtils.createStateMessage(STREAM_NAME, "checkpoint", "2"); + + private SimpleAirbyteDestination destination; + + @BeforeEach + void beforeEach() { + destination = new SimpleAirbyteDestination(); + } + + @Test + void testNotifyEndOfInputTerminatesTheDestination() throws Exception { + assertFalse(destination.isFinished()); + destination.notifyEndOfInput(); + assertTrue(destination.isFinished()); + } + + @Test + void testDestinationEchoesStateMessages() throws Exception { + destination.accept(RECORD_MESSAGE1); + destination.accept(RECORD_MESSAGE1); + destination.accept(STATE_MESSAGE1); + destination.accept(RECORD_MESSAGE2); + destination.accept(STATE_MESSAGE2); + + assertEquals(STATE_MESSAGE1, destination.attemptRead().get()); + assertEquals(STATE_MESSAGE2, destination.attemptRead().get()); + } + + @Test + void testDestinationWillReturnAllStateMessagesBeforeClosing() throws Exception { + destination.accept(STATE_MESSAGE2); + destination.accept(STATE_MESSAGE1); + destination.notifyEndOfInput(); + + assertFalse(destination.isFinished()); + assertEquals(STATE_MESSAGE2, destination.attemptRead().get()); + assertEquals(STATE_MESSAGE1, destination.attemptRead().get()); + assertTrue(destination.isFinished()); + } + +} diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteSource.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteSource.java new file mode 100644 index 00000000000..28d0de70c28 --- /dev/null +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteSource.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import io.airbyte.config.WorkerSourceConfig; +import io.airbyte.protocol.models.AirbyteMessage; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Queue; + +/** + * Simple in memory implementation of an AirbyteSource for testing purpose. + */ +public class SimpleAirbyteSource implements AirbyteSource { + + private final Queue messages = new LinkedList<>(); + private final List infiniteMessages = new ArrayList<>(); + + /** + * Configure the source to loop infinitely on the messages. + */ + public void setInfiniteSourceWithMessages(final AirbyteMessage... messages) { + this.infiniteMessages.clear(); + this.messages.clear(); + this.infiniteMessages.addAll(Arrays.stream(messages).toList()); + } + + /** + * Configure the source to return all the messages then terminate. + */ + public void setMessages(final AirbyteMessage... messages) { + this.infiniteMessages.clear(); + this.messages.clear(); + this.messages.addAll(Arrays.stream(messages).toList()); + } + + @Override + public void start(WorkerSourceConfig sourceConfig, Path jobRoot) throws Exception { + + } + + @Override + public boolean isFinished() { + return messages.isEmpty() && infiniteMessages.isEmpty(); + } + + @Override + public int getExitValue() { + return 0; + } + + @Override + public Optional attemptRead() { + if (messages.isEmpty() && !infiniteMessages.isEmpty()) { + this.messages.addAll(infiniteMessages); + } + + if (!messages.isEmpty()) { + return Optional.of(messages.poll()); + } + return Optional.empty(); + } + + @Override + public void close() throws Exception { + + } + + @Override + public void cancel() throws Exception { + + } + +} diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteSourceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteSourceTest.java new file mode 100644 index 00000000000..e1e7e23a494 --- /dev/null +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/SimpleAirbyteSourceTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.workers.test_utils.AirbyteMessageUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class SimpleAirbyteSourceTest { + + protected static final String STREAM_NAME = "stream1"; + protected static final String FIELD_NAME = "field1"; + + protected static final AirbyteMessage RECORD_MESSAGE1 = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "m1"); + protected static final AirbyteMessage RECORD_MESSAGE2 = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "m2"); + protected static final AirbyteMessage STATE_MESSAGE = AirbyteMessageUtils.createStateMessage(STREAM_NAME, "checkpoint", "1"); + + private SimpleAirbyteSource source; + + @BeforeEach + void beforeEach() { + source = new SimpleAirbyteSource(); + } + + @Test + void testMessages() { + source.setMessages(RECORD_MESSAGE1, RECORD_MESSAGE2, STATE_MESSAGE); + + // Reading all the messages from the source + final List messagesRead = new ArrayList<>(); + while (!source.isFinished()) { + messagesRead.add(source.attemptRead().get()); + } + + // Once the source is finished, subsequent attemptRead should return emtpy + assertEquals(Optional.empty(), source.attemptRead()); + + assertEquals(List.of(RECORD_MESSAGE1, RECORD_MESSAGE2, STATE_MESSAGE), messagesRead); + } + + @Test + void testInfiniteMessages() { + source.setInfiniteSourceWithMessages(RECORD_MESSAGE1, RECORD_MESSAGE2, STATE_MESSAGE); + + // Reading 10 the messages from the source + final List messagesRead = new ArrayList<>(); + for (int i = 0; i < 10; ++i) { + assertFalse(source.isFinished()); + messagesRead.add(source.attemptRead().get()); + } + + // source should be looping on the 3 messages set in the init call + assertEquals(List.of( + RECORD_MESSAGE1, RECORD_MESSAGE2, STATE_MESSAGE, + RECORD_MESSAGE1, RECORD_MESSAGE2, STATE_MESSAGE, + RECORD_MESSAGE1, RECORD_MESSAGE2, STATE_MESSAGE, + RECORD_MESSAGE1), messagesRead); + assertFalse(source.isFinished()); + } + +} diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/Application.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/Application.java index af03f490203..7fd7e8cf2f3 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/Application.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/Application.java @@ -75,7 +75,6 @@ int run() { .build()) { asyncStateManager.write(AsyncKubePodStatus.INITIALIZING); - asyncStateManager.write(AsyncKubePodStatus.RUNNING); asyncStateManager.write(AsyncKubePodStatus.SUCCEEDED, jobOrchestrator.runJob().orElse("")); } catch (final Throwable t) { log.error("Killing orchestrator because of an Exception", t); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java index 9078235262a..78d1a9d1cc7 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java @@ -8,6 +8,7 @@ import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.temporal.sync.OrchestratorConstants; import io.airbyte.config.EnvConfigs; +import io.airbyte.container_orchestrator.AsyncStateManager; import io.airbyte.container_orchestrator.orchestrator.DbtJobOrchestrator; import io.airbyte.container_orchestrator.orchestrator.JobOrchestrator; import io.airbyte.container_orchestrator.orchestrator.NoOpOrchestrator; @@ -101,12 +102,13 @@ JobOrchestrator jobOrchestrator( final ProcessFactory processFactory, final WorkerConfigsProvider workerConfigsProvider, final JobRunConfig jobRunConfig, - final ReplicationWorkerFactory replicationWorkerFactory) { + final ReplicationWorkerFactory replicationWorkerFactory, + final AsyncStateManager asyncStateManager) { return switch (application) { case ReplicationLauncherWorker.REPLICATION -> new ReplicationJobOrchestrator(envConfigs, jobRunConfig, - replicationWorkerFactory); - case NormalizationLauncherWorker.NORMALIZATION -> new NormalizationJobOrchestrator(envConfigs, processFactory, jobRunConfig); - case DbtLauncherWorker.DBT -> new DbtJobOrchestrator(envConfigs, workerConfigsProvider, processFactory, jobRunConfig); + replicationWorkerFactory, asyncStateManager); + case NormalizationLauncherWorker.NORMALIZATION -> new NormalizationJobOrchestrator(envConfigs, processFactory, jobRunConfig, asyncStateManager); + case DbtLauncherWorker.DBT -> new DbtJobOrchestrator(envConfigs, workerConfigsProvider, processFactory, jobRunConfig, asyncStateManager); case AsyncOrchestratorPodProcess.NO_OP -> new NoOpOrchestrator(); default -> throw new IllegalStateException("Could not find job orchestrator for application: " + application); }; diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/DbtJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/DbtJobOrchestrator.java index dbdeaaf763c..1478e105740 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/DbtJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/DbtJobOrchestrator.java @@ -12,6 +12,7 @@ import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.config.Configs; import io.airbyte.config.OperatorDbtInput; +import io.airbyte.container_orchestrator.AsyncStateManager; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; @@ -21,6 +22,7 @@ import io.airbyte.workers.general.DbtTransformationRunner; import io.airbyte.workers.general.DbtTransformationWorker; import io.airbyte.workers.normalization.DefaultNormalizationRunner; +import io.airbyte.workers.process.AsyncKubePodStatus; import io.airbyte.workers.process.KubePodProcess; import io.airbyte.workers.process.ProcessFactory; import io.airbyte.workers.sync.ReplicationLauncherWorker; @@ -41,15 +43,19 @@ public class DbtJobOrchestrator implements JobOrchestrator { private final WorkerConfigsProvider workerConfigsProvider; private final ProcessFactory processFactory; private final JobRunConfig jobRunConfig; + // Used by the orchestrator to mark the job RUNNING once the relevant pods are spun up. + private final AsyncStateManager asyncStateManager; public DbtJobOrchestrator(final Configs configs, final WorkerConfigsProvider workerConfigsProvider, final ProcessFactory processFactory, - final JobRunConfig jobRunConfig) { + final JobRunConfig jobRunConfig, + final AsyncStateManager asyncStateManager) { this.configs = configs; this.workerConfigsProvider = workerConfigsProvider; this.processFactory = processFactory; this.jobRunConfig = jobRunConfig; + this.asyncStateManager = asyncStateManager; } @Override @@ -86,7 +92,8 @@ public Optional runJob() throws Exception { processFactory, new DefaultNormalizationRunner( processFactory, destinationLauncherConfig.getNormalizationDockerImage(), - destinationLauncherConfig.getNormalizationIntegrationType()))); + destinationLauncherConfig.getNormalizationIntegrationType())), + this::markJobRunning); log.info("Running dbt worker..."); final Path jobRoot = TemporalUtils.getJobRoot(configs.getWorkspaceRoot(), @@ -96,4 +103,8 @@ processFactory, new DefaultNormalizationRunner( return Optional.empty(); } + private void markJobRunning() { + asyncStateManager.write(AsyncKubePodStatus.RUNNING); + } + } diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java index fa50c99e55a..5908feaff4b 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java @@ -14,12 +14,14 @@ import io.airbyte.config.Configs; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; +import io.airbyte.container_orchestrator.AsyncStateManager; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.general.DefaultNormalizationWorker; import io.airbyte.workers.normalization.DefaultNormalizationRunner; import io.airbyte.workers.normalization.NormalizationWorker; +import io.airbyte.workers.process.AsyncKubePodStatus; import io.airbyte.workers.process.KubePodProcess; import io.airbyte.workers.process.ProcessFactory; import io.airbyte.workers.sync.ReplicationLauncherWorker; @@ -37,11 +39,17 @@ public class NormalizationJobOrchestrator implements JobOrchestrator runJob() throws Exception { processFactory, destinationLauncherConfig.getNormalizationDockerImage(), destinationLauncherConfig.getNormalizationIntegrationType()), - configs.getWorkerEnvironment()); + configs.getWorkerEnvironment(), + this::markJobRunning); log.info("Running normalization worker..."); final Path jobRoot = TemporalUtils.getJobRoot(configs.getWorkspaceRoot(), @@ -88,4 +97,8 @@ public Optional runJob() throws Exception { return Optional.of(Jsons.serialize(normalizationSummary)); } + private void markJobRunning() { + asyncStateManager.write(AsyncKubePodStatus.RUNNING); + } + } diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java index cbc7b595552..cc38987ff00 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java @@ -15,11 +15,13 @@ import io.airbyte.config.Configs; import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSyncInput; +import io.airbyte.container_orchestrator.AsyncStateManager; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.general.ReplicationWorker; import io.airbyte.workers.general.ReplicationWorkerFactory; +import io.airbyte.workers.process.AsyncKubePodStatus; import io.airbyte.workers.process.KubePodProcess; import io.airbyte.workers.sync.ReplicationLauncherWorker; import java.lang.invoke.MethodHandles; @@ -38,13 +40,17 @@ public class ReplicationJobOrchestrator implements JobOrchestrator runJob() throws Exception { SOURCE_DOCKER_IMAGE_KEY, sourceLauncherConfig.getDockerImage())); final ReplicationWorker replicationWorker = - replicationWorkerFactory.create(syncInput, jobRunConfig, sourceLauncherConfig, destinationLauncherConfig); + replicationWorkerFactory.create(syncInput, jobRunConfig, sourceLauncherConfig, destinationLauncherConfig, this::markJobRunning); log.info("Running replication worker..."); final var jobRoot = TemporalUtils.getJobRoot(configs.getWorkspaceRoot(), @@ -88,4 +94,8 @@ public Optional runJob() throws Exception { return Optional.of(Jsons.serialize(replicationOutput)); } + private void markJobRunning() { + asyncStateManager.write(AsyncKubePodStatus.RUNNING); + } + } diff --git a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/ApplicationTest.java b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/ApplicationTest.java index 552f766f8a8..937028e68f0 100644 --- a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/ApplicationTest.java +++ b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/ApplicationTest.java @@ -38,7 +38,7 @@ void testHappyPath() throws Exception { assertEquals(0, code); verify(jobOrchestrator).runJob(); verify(asyncStateManager).write(AsyncKubePodStatus.INITIALIZING); - verify(asyncStateManager).write(AsyncKubePodStatus.RUNNING); + // NOTE: we don't expect it to write RUNNING, because the job orchestrator is responsible for that. verify(asyncStateManager).write(AsyncKubePodStatus.SUCCEEDED, output); } @@ -51,7 +51,7 @@ void testJobFailedWritesFailedStatus() throws Exception { assertEquals(1, code); verify(jobOrchestrator).runJob(); verify(asyncStateManager).write(AsyncKubePodStatus.INITIALIZING); - verify(asyncStateManager).write(AsyncKubePodStatus.RUNNING); + // NOTE: we don't expect it to write RUNNING, because the job orchestrator is responsible for that. verify(asyncStateManager).write(AsyncKubePodStatus.FAILED); } diff --git a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactoryTest.java b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactoryTest.java index 5e0ae5acc61..63234a06106 100644 --- a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactoryTest.java +++ b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactoryTest.java @@ -8,9 +8,11 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.config.EnvConfigs; +import io.airbyte.container_orchestrator.AsyncStateManager; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.TestClient; import io.airbyte.persistence.job.models.JobRunConfig; @@ -30,6 +32,7 @@ import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; import java.util.Map; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; // tests may be running on a real k8s environment, override the environment to something else for @@ -59,11 +62,18 @@ class ContainerOrchestratorFactoryTest { @Inject ReplicationWorkerFactory replicationWorkerFactory; + AsyncStateManager asyncStateManager; + // Tests will fail if this is uncommented, due to how the implementation of the DocumentStoreClient // is being created // @Inject // DocumentStoreClient documentStoreClient; + @BeforeEach + void beforeEach() { + asyncStateManager = mock(AsyncStateManager.class); + } + @Test void featureFlags() { assertNotNull(featureFlags); @@ -97,25 +107,29 @@ void jobOrchestrator() { final var factory = new ContainerOrchestratorFactory(); final var repl = factory.jobOrchestrator( - ReplicationLauncherWorker.REPLICATION, envConfigs, processFactory, workerConfigsProvider, jobRunConfig, replicationWorkerFactory); + ReplicationLauncherWorker.REPLICATION, envConfigs, processFactory, workerConfigsProvider, jobRunConfig, replicationWorkerFactory, + asyncStateManager); assertEquals("Replication", repl.getOrchestratorName()); final var norm = factory.jobOrchestrator( - NormalizationLauncherWorker.NORMALIZATION, envConfigs, processFactory, workerConfigsProvider, jobRunConfig, replicationWorkerFactory); + NormalizationLauncherWorker.NORMALIZATION, envConfigs, processFactory, workerConfigsProvider, jobRunConfig, replicationWorkerFactory, + asyncStateManager); assertEquals("Normalization", norm.getOrchestratorName()); final var dbt = factory.jobOrchestrator( DbtLauncherWorker.DBT, envConfigs, processFactory, workerConfigsProvider, jobRunConfig, - replicationWorkerFactory); + replicationWorkerFactory, asyncStateManager); assertEquals("DBT Transformation", dbt.getOrchestratorName()); final var noop = factory.jobOrchestrator( - AsyncOrchestratorPodProcess.NO_OP, envConfigs, processFactory, workerConfigsProvider, jobRunConfig, replicationWorkerFactory); + AsyncOrchestratorPodProcess.NO_OP, envConfigs, processFactory, workerConfigsProvider, jobRunConfig, replicationWorkerFactory, + asyncStateManager); assertEquals("NO_OP", noop.getOrchestratorName()); var caught = false; try { - factory.jobOrchestrator("does not exist", envConfigs, processFactory, workerConfigsProvider, jobRunConfig, replicationWorkerFactory); + factory.jobOrchestrator("does not exist", envConfigs, processFactory, workerConfigsProvider, jobRunConfig, replicationWorkerFactory, + asyncStateManager); } catch (final Exception e) { caught = true; } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index b626c85c239..7d24fbbff1c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -19,6 +19,7 @@ import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.JobInfoRead; +import io.airbyte.api.model.generated.ListConnectionsForWorkspacesRequestBody; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.auth.SecuredWorkspace; import io.airbyte.commons.server.handlers.ConnectionsHandler; @@ -89,6 +90,17 @@ public ConnectionReadList listConnectionsForWorkspace(@Body final WorkspaceIdReq return ApiHelper.execute(() -> connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody)); } + @SuppressWarnings("LineLength") + @Post(uri = "/list_paginated") + @Secured({READER}) + @SecuredWorkspace + @ExecuteOn(AirbyteTaskExecutors.IO) + @Override + public ConnectionReadList listConnectionsForWorkspacesPaginated( + @Body final ListConnectionsForWorkspacesRequestBody listConnectionsForWorkspacesRequestBody) { + return ApiHelper.execute(() -> connectionsHandler.listConnectionsForWorkspaces(listConnectionsForWorkspacesRequestBody)); + } + @Override @Post(uri = "/list_all") @Secured({READER}) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java index 3ed83d5f90d..c330629e127 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java @@ -16,6 +16,7 @@ import io.airbyte.api.model.generated.DestinationReadList; import io.airbyte.api.model.generated.DestinationSearch; import io.airbyte.api.model.generated.DestinationUpdate; +import io.airbyte.api.model.generated.ListResourcesForWorkspacesRequestBody; import io.airbyte.api.model.generated.PartialDestinationUpdate; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.auth.SecuredWorkspace; @@ -109,6 +110,16 @@ public DestinationReadList listDestinationsForWorkspace(@Body final WorkspaceIdR return ApiHelper.execute(() -> destinationHandler.listDestinationsForWorkspace(workspaceIdRequestBody)); } + @SuppressWarnings("LineLength") + @Post(uri = "/list_paginated") + @Secured({READER}) + @SecuredWorkspace + @ExecuteOn(AirbyteTaskExecutors.IO) + @Override + public DestinationReadList listDestinationsForWorkspacesPaginated(@Body final ListResourcesForWorkspacesRequestBody listResourcesForWorkspacesRequestBody) { + return ApiHelper.execute(() -> destinationHandler.listDestinationsForWorkspaces(listResourcesForWorkspacesRequestBody)); + } + @Post(uri = "/search") @ExecuteOn(AirbyteTaskExecutors.IO) @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java index c8678a068d9..7ef90ae9941 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceApiController.java @@ -12,6 +12,7 @@ import io.airbyte.api.model.generated.ActorCatalogWithUpdatedAt; import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.DiscoverCatalogResult; +import io.airbyte.api.model.generated.ListResourcesForWorkspacesRequestBody; import io.airbyte.api.model.generated.PartialSourceUpdate; import io.airbyte.api.model.generated.SourceAutoPropagateChange; import io.airbyte.api.model.generated.SourceCloneRequestBody; @@ -30,6 +31,7 @@ import io.airbyte.commons.server.handlers.SourceHandler; import io.airbyte.commons.server.scheduling.AirbyteTaskExecutors; import io.micronaut.http.HttpStatus; +import io.micronaut.http.annotation.Body; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; import io.micronaut.http.annotation.Status; @@ -145,6 +147,15 @@ public SourceReadList listSourcesForWorkspace(final WorkspaceIdRequestBody works return ApiHelper.execute(() -> sourceHandler.listSourcesForWorkspace(workspaceIdRequestBody)); } + @Post(uri = "/list_paginated") + @Secured({READER}) + @SecuredWorkspace + @ExecuteOn(AirbyteTaskExecutors.IO) + @Override + public SourceReadList listSourcesForWorkspacePaginated(@Body final ListResourcesForWorkspacesRequestBody listResourcesForWorkspacesRequestBody) { + return ApiHelper.execute(() -> sourceHandler.listSourcesForWorkspaces(listResourcesForWorkspacesRequestBody)); + } + @Post("/search") @Override public SourceReadList searchSources(final SourceSearch sourceSearch) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java index c1df18f06e3..33a592b1a15 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java @@ -10,6 +10,7 @@ import io.airbyte.api.generated.WorkspaceApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; +import io.airbyte.api.model.generated.ListResourcesForWorkspacesRequestBody; import io.airbyte.api.model.generated.SlugRequestBody; import io.airbyte.api.model.generated.WorkspaceCreate; import io.airbyte.api.model.generated.WorkspaceGiveFeedback; @@ -86,6 +87,15 @@ public WorkspaceReadList listWorkspaces() { return ApiHelper.execute(workspacesHandler::listWorkspaces); } + @Post(uri = "/list_paginated") + @Secured({READER}) + @SecuredWorkspace + @ExecuteOn(AirbyteTaskExecutors.IO) + @Override + public WorkspaceReadList listWorkspacesPaginated(@Body final ListResourcesForWorkspacesRequestBody listResourcesForWorkspacesRequestBody) { + return ApiHelper.execute(() -> workspacesHandler.listWorkspacesPaginated(listResourcesForWorkspacesRequestBody)); + } + @Post("/update") @Secured({EDITOR}) @SecuredWorkspace diff --git a/airbyte-webapp/nginx/cloud.conf.template b/airbyte-webapp/nginx/cloud.conf.template index 67988ff65b0..6de816c0042 100644 --- a/airbyte-webapp/nginx/cloud.conf.template +++ b/airbyte-webapp/nginx/cloud.conf.template @@ -11,6 +11,11 @@ server { location / { root /usr/share/nginx/html; + + location ~ ^/docs/.* { + try_files $uri $uri/ =404; + } + location ~ ^/(?!(assets/.*)) { try_files $uri $uri/ /index.html; } diff --git a/airbyte-webapp/nginx/default.conf.template b/airbyte-webapp/nginx/default.conf.template index 05662a9be7c..aa9c77ed860 100644 --- a/airbyte-webapp/nginx/default.conf.template +++ b/airbyte-webapp/nginx/default.conf.template @@ -19,6 +19,11 @@ server { location / { root /usr/share/nginx/html; + + location ~ ^/docs/.* { + try_files $uri $uri/ =404; + } + location ~ ^/(?!(assets/.*)) { try_files $uri $uri/ /index.html; } diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.module.scss b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.module.scss index a8dd2d449d0..d620c469fa4 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.module.scss +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.module.scss @@ -1,11 +1,7 @@ @use "scss/colors"; @use "scss/variables"; -.container { - margin-bottom: variables.$spacing-xl; -} - .list { - background-color: colors.$grey-50; border-radius: variables.$border-radius-xs; + overflow: hidden; } diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx index 97cc2587e3b..dafd2e8da89 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx @@ -1,6 +1,8 @@ import React from "react"; import { FormattedMessage } from "react-intl"; +import { Box } from "components/ui/Box"; +import { FlexContainer } from "components/ui/Flex"; import { Modal, ModalProps } from "components/ui/Modal"; import { ConnectionFormMode } from "hooks/services/ConnectionForm/ConnectionFormService"; @@ -65,7 +67,7 @@ export const ArrayOfObjectsEditor = ({ return ( <> -

+ ({ disabled={disabled} /> {items.length ? ( -
+ {items.map((item, index) => ( ({ disabled={disabled} /> ))} -
+ ) : null} -
+ {mode !== "readonly" && isEditable && renderEditModal()} ); diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsHookFormEditor.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsHookFormEditor.tsx new file mode 100644 index 00000000000..c6834cc6267 --- /dev/null +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsHookFormEditor.tsx @@ -0,0 +1,69 @@ +import React from "react"; +import { FieldArrayWithId } from "react-hook-form"; + +import { Box } from "components/ui/Box"; +import { FlexContainer } from "components/ui/Flex"; + +import styles from "./ArrayOfObjectsEditor.module.scss"; +import { EditorHeader } from "./components/EditorHeader"; +import { EditorRow } from "./components/EditorRow"; + +export interface ArrayOfObjectsHookFormEditorProps { + fields: T[]; + mainTitle?: React.ReactNode; + addButtonText?: React.ReactNode; + renderItemName?: (item: T, index: number) => React.ReactNode | undefined; + renderItemDescription?: (item: T, index: number) => React.ReactNode | undefined; + onAddItem: () => void; + onStartEdit: (n: number) => void; + onRemove: (index: number) => void; +} + +/** + * The component is used to render a list of react-hook-form FieldArray items with the ability to add, edit and remove items. + * It's a react-hook-form version of the ArrayOfObjectsEditor component and will replace it in the future. + * @see ArrayOfObjectsEditor + * @param fields + * @param mainTitle + * @param addButtonText + * @param onAddItem + * @param renderItemName + * @param renderItemDescription + * @param onStartEdit + * @param onRemove + * @param mode + * @constructor + */ +export const ArrayOfObjectsHookFormEditor = ({ + fields, + mainTitle, + addButtonText, + onAddItem, + renderItemName, + renderItemDescription, + onStartEdit, + onRemove, +}: ArrayOfObjectsHookFormEditorProps) => ( + + + {fields.length ? ( + + {fields.map((field, index) => ( + + ))} + + ) : null} + +); diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.module.scss b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.module.scss deleted file mode 100644 index 885e0e96ce5..00000000000 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.module.scss +++ /dev/null @@ -1,14 +0,0 @@ -@use "scss/colors"; -@use "scss/variables"; - -.editorHeader { - display: flex; - justify-content: space-between; - align-items: center; - flex-direction: row; - color: colors.$dark-blue-900; - font-weight: 500; - font-size: variables.$font-size-lg; - line-height: 1.2; - margin: 5px 0 10px; -} diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx index f6990cd4570..3979b916e85 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx @@ -1,22 +1,27 @@ import React from "react"; import { FormattedMessage } from "react-intl"; +import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; +import { Text } from "components/ui/Text"; import { ConnectionFormMode } from "hooks/services/ConnectionForm/ConnectionFormService"; -import styles from "./EditorHeader.module.scss"; - interface EditorHeaderProps { mainTitle?: React.ReactNode; addButtonText?: React.ReactNode; itemsCount: number; onAddItem: () => void; + /** + * seems like "mode" and "disabled" props can be removed since we can control fields enable/disable states on higher levels + * TODO: remove during ArrayOfObjectsEditor refactoring and CreateConnectionForm migration + */ mode?: ConnectionFormMode; disabled?: boolean; } -const EditorHeader: React.FC = ({ +export const EditorHeader: React.FC = ({ itemsCount, onAddItem, mainTitle, @@ -25,15 +30,17 @@ const EditorHeader: React.FC = ({ disabled, }) => { return ( -
- {mainTitle || } - {mode !== "readonly" && ( - - )} -
+ + + + {mainTitle || } + + {mode !== "readonly" && ( + + )} + + ); }; - -export { EditorHeader }; diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.module.scss b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.module.scss index e68c468febe..d195b4a6291 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.module.scss +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.module.scss @@ -1,21 +1,14 @@ @use "scss/colors"; @use "scss/variables"; -.container + .container { +.container { border-top: 1px solid colors.$foreground; + background-color: colors.$grey-50; } .body { - display: flex; - justify-content: space-between; - align-items: center; - flex-direction: row; color: colors.$dark-blue; - font-weight: 400; - font-size: variables.$font-size-sm; - line-height: 1.4; - padding: variables.$spacing-xs variables.$spacing-xs variables.$spacing-xs variables.$spacing-md; - gap: variables.$spacing-xs; + padding: variables.$spacing-xs variables.$spacing-md; } .name { @@ -23,8 +16,3 @@ text-overflow: ellipsis; white-space: nowrap; } - -.actions { - display: flex; - flex-direction: row; -} diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx index 2184263b338..b0624e097a3 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx @@ -1,9 +1,10 @@ import React from "react"; import { useIntl } from "react-intl"; -import { CrossIcon } from "components/icons/CrossIcon"; -import { PencilIcon } from "components/icons/PencilIcon"; import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; +import { Text } from "components/ui/Text"; import { Tooltip } from "components/ui/Tooltip"; import styles from "./EditorRow.module.scss"; @@ -21,9 +22,11 @@ export const EditorRow: React.FC = ({ name, id, description, onE const { formatMessage } = useIntl(); const body = ( -
-
{name || id}
-
+ + + {name || id} + +
-
+ + ); return ( diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx index b4a645f0160..d8b44d1689c 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx @@ -1,4 +1,2 @@ -import { ArrayOfObjectsEditor } from "./ArrayOfObjectsEditor"; - -export default ArrayOfObjectsEditor; -export { ArrayOfObjectsEditor }; +export { ArrayOfObjectsEditor } from "./ArrayOfObjectsEditor"; +export { ArrayOfObjectsHookFormEditor } from "./ArrayOfObjectsHookFormEditor"; diff --git a/airbyte-webapp/src/components/EntityTable/ConnectionTable.module.scss b/airbyte-webapp/src/components/EntityTable/ConnectionTable.module.scss index 8c71a8012af..c8516f7af29 100644 --- a/airbyte-webapp/src/components/EntityTable/ConnectionTable.module.scss +++ b/airbyte-webapp/src/components/EntityTable/ConnectionTable.module.scss @@ -4,8 +4,8 @@ th.width30 { width: 30%; } -th.width10 { - width: 10%; +th.width20 { + width: 20%; } .thEnabled { diff --git a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx index 5b20e4be42d..6032f6dc8fe 100644 --- a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx +++ b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx @@ -173,7 +173,7 @@ const ConnectionTable: React.FC = ({ data, entity, onClick ), cell: (props) => , meta: { - thClassName: styles.width10, + thClassName: styles.width20, }, }), columnHelper.accessor("enabled", { diff --git a/airbyte-webapp/src/components/EntityTable/components/ConnectEntitiesCell.module.scss b/airbyte-webapp/src/components/EntityTable/components/ConnectEntitiesCell.module.scss index 6c6cb5a2e43..371bbb36551 100644 --- a/airbyte-webapp/src/components/EntityTable/components/ConnectEntitiesCell.module.scss +++ b/airbyte-webapp/src/components/EntityTable/components/ConnectEntitiesCell.module.scss @@ -11,6 +11,7 @@ .connectors { text-overflow: ellipsis; + white-space: nowrap; overflow: hidden; max-width: 150px; } diff --git a/airbyte-webapp/src/components/EntityTable/components/LastSyncCell.module.scss b/airbyte-webapp/src/components/EntityTable/components/LastSyncCell.module.scss index d46751de192..8b96c5b6bf4 100644 --- a/airbyte-webapp/src/components/EntityTable/components/LastSyncCell.module.scss +++ b/airbyte-webapp/src/components/EntityTable/components/LastSyncCell.module.scss @@ -3,6 +3,7 @@ .text { color: colors.$grey; + white-space: nowrap; &.enabled { color: colors.$dark-blue; diff --git a/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.module.scss b/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.module.scss index 18276c775fb..d53e2dfc323 100644 --- a/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.module.scss +++ b/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.module.scss @@ -2,28 +2,27 @@ @use "scss/variables"; .container { - margin-bottom: 6px; -} + margin-bottom: variables.$spacing-sm; -.label { - padding-left: variables.$spacing-md; - font-size: variables.$font-size-lg; - color: colors.$dark-blue; - cursor: pointer; -} + .label { + font-size: variables.$font-size-lg; + color: colors.$dark-blue; + cursor: pointer; -.disabled { - color: colors.$grey-300; - cursor: auto; -} + &--disabled { + color: colors.$grey; + cursor: auto; + } + } -.message { - padding-left: variables.$spacing-sm; - color: colors.$grey-300; - font-size: variables.$font-size-md; + .message { + padding-left: variables.$spacing-sm; + color: colors.$grey; + font-size: variables.$font-size-md; - & a { - text-decoration: underline; - color: colors.$blue; + & a { + text-decoration: underline; + color: colors.$blue; + } } } diff --git a/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.tsx b/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.tsx index 605b4e5e680..f7f4fc35701 100644 --- a/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.tsx +++ b/airbyte-webapp/src/components/LabeledRadioButton/LabeledRadioButton.tsx @@ -5,23 +5,23 @@ import { FlexContainer } from "components/ui/Flex"; import { RadioButton } from "components/ui/RadioButton"; import styles from "./LabeledRadioButton.module.scss"; -type IProps = { + +export interface LabeledRadioButtonProps extends React.InputHTMLAttributes { message?: React.ReactNode; label?: React.ReactNode; - className?: string; -} & React.InputHTMLAttributes; +} -const LabeledRadioButton: React.FC = (props) => ( - +export const LabeledRadioButton = React.forwardRef((props, ref) => ( + -); +)); -export default LabeledRadioButton; +LabeledRadioButton.displayName = "LabeledRadioButton"; diff --git a/airbyte-webapp/src/components/LabeledRadioButton/index.tsx b/airbyte-webapp/src/components/LabeledRadioButton/index.tsx index 85f792c69ca..e7619250d92 100644 --- a/airbyte-webapp/src/components/LabeledRadioButton/index.tsx +++ b/airbyte-webapp/src/components/LabeledRadioButton/index.tsx @@ -1,4 +1,2 @@ -import LabeledRadioButton from "./LabeledRadioButton"; - -export default LabeledRadioButton; -export { LabeledRadioButton }; +export { LabeledRadioButton } from "./LabeledRadioButton"; +export type { LabeledRadioButtonProps } from "./LabeledRadioButton"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/LabeledRadioButtonFormControl.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/LabeledRadioButtonFormControl.tsx new file mode 100644 index 00000000000..a240040cee0 --- /dev/null +++ b/airbyte-webapp/src/components/connection/ConnectionForm/LabeledRadioButtonFormControl.tsx @@ -0,0 +1,39 @@ +import React from "react"; +import { Controller, useFormContext } from "react-hook-form"; + +import { LabeledRadioButton, LabeledRadioButtonProps } from "components"; + +interface LabeledRadioButtonFormControlProps extends LabeledRadioButtonProps { + controlId: string; + name: string; // redeclare name to make it required +} + +export const LabeledRadioButtonFormControl: React.FC = ({ + controlId, + name, + label, + value, + message, + ...restProps +}) => { + const { control } = useFormContext(); + + return ( + ( + + )} + /> + ); +}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationHookFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationHookFormField.tsx new file mode 100644 index 00000000000..4738efb4e5c --- /dev/null +++ b/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationHookFormField.tsx @@ -0,0 +1,52 @@ +import React from "react"; +import { FormattedMessage, useIntl } from "react-intl"; + +import { Box } from "components/ui/Box"; +import { ExternalLink } from "components/ui/Link"; + +import { NormalizationType } from "core/domain/connection"; +import { links } from "core/utils/links"; +import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; + +import { LabeledRadioButtonFormControl } from "./LabeledRadioButtonFormControl"; + +/** + * react-hook-form field for normalization operation + * ready for migration to + * @see CreateConnectionForm + * old formik form field component: + * @see NormalizationField + */ +export const NormalizationHookFormField: React.FC = () => { + const { formatMessage } = useIntl(); + const { mode } = useConnectionFormService(); + + return ( + + + {lnk}, + }} + /> + ) + } + /> + + ); +}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/TransformationField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/TransformationField.tsx index 759b5475dba..83de45885c0 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/TransformationField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/TransformationField.tsx @@ -2,7 +2,7 @@ import { ArrayHelpers, FormikProps } from "formik"; import React, { useState } from "react"; import { FormattedMessage } from "react-intl"; -import ArrayOfObjectsEditor from "components/ArrayOfObjectsEditor"; +import { ArrayOfObjectsEditor } from "components/ArrayOfObjectsEditor"; import TransformationForm from "components/connection/TransformationForm"; import { OperationRead } from "core/request/AirbyteClient"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/TransformationFieldHookForm.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/TransformationFieldHookForm.tsx new file mode 100644 index 00000000000..60edfb038f7 --- /dev/null +++ b/airbyte-webapp/src/components/connection/ConnectionForm/TransformationFieldHookForm.tsx @@ -0,0 +1,60 @@ +import React from "react"; +import { useFieldArray } from "react-hook-form"; +import { FormattedMessage } from "react-intl"; + +import { ArrayOfObjectsHookFormEditor } from "components/ArrayOfObjectsEditor"; + +import { isDefined } from "core/utils/common"; +import { useModalService } from "hooks/services/Modal"; +import { CustomTransformationsFormValues } from "pages/connections/ConnectionTransformationPage/CustomTransformationsCard"; + +import { useDefaultTransformation } from "./formConfig"; +import { DbtOperationReadOrCreate, TransformationHookForm } from "../TransformationHookForm"; + +/** + * Custom transformations field for react-hook-form + * will replace TransformationField in the future + * @see TransformationField + * @constructor + */ +export const TransformationFieldHookForm: React.FC = () => { + const { fields, append, remove, update } = useFieldArray({ + name: "transformations", + }); + const { openModal, closeModal } = useModalService(); + const defaultTransformation = useDefaultTransformation(); + + const openEditModal = (transformationItemIndex?: number) => + openModal({ + size: "xl", + title: , + content: () => ( + { + isDefined(transformationItemIndex) + ? update(transformationItemIndex, transformation) + : append(transformation); + closeModal(); + }} + onCancel={closeModal} + /> + ), + }); + + return ( + } + addButtonText={} + renderItemName={(item) => item.name} + onAddItem={() => openEditModal()} + onStartEdit={openEditModal} + onRemove={remove} + /> + ); +}; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap index 60d49b1ead1..47727090e36 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap @@ -1338,18 +1338,27 @@ exports[`CreateConnectionForm should render 1`] = `
- No custom transformation - + No custom transformation +

+ +
diff --git a/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts b/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts index 58765983570..b5e9c654b26 100644 --- a/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts +++ b/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts @@ -2,6 +2,19 @@ import { AttemptRead, AttemptStats, AttemptStatus, AttemptStreamStats } from "co import { progressBarCalculations } from "./utils"; +// used for tests which rely on Date.now(), to account for if the test takes slightly longer to run sometimes +expect.extend({ + toBeWithinTolerance(received, center, tolerance) { + const floor = center - tolerance; + const ceiling = center + tolerance; + const pass = received >= floor && received <= ceiling; + return { + message: () => `expected ${received} to be within tolerance of ${tolerance} around ${center}`, + pass, + }; + }, +}); + describe("#progressBarCalculations", () => { it("for an attempt with no throughput information", () => { const attempt = makeAttempt(); @@ -18,8 +31,8 @@ describe("#progressBarCalculations", () => { expect(displayProgressBar).toEqual(true); expect(totalPercentRecords).toEqual(0.01); - expect(elapsedTimeMS).toEqual(10 * 1000); - expect(timeRemaining).toEqual(990 * 1000); + expect(elapsedTimeMS).toBeWithinTolerance(10 * 1000, 2); + expect(timeRemaining).toBeWithinTolerance(990 * 1000, 2); }); it("for an attempt with per-stream stats", () => { @@ -42,8 +55,8 @@ describe("#progressBarCalculations", () => { expect(displayProgressBar).toEqual(true); expect(totalPercentRecords).toEqual(0.01); - expect(elapsedTimeMS).toEqual(10 * 1000); - expect(timeRemaining).toEqual(990 * 1000); + expect(elapsedTimeMS).toBeWithinTolerance(10 * 1000, 2); + expect(timeRemaining).toBeWithinTolerance(990 * 1000, 2); }); }); diff --git a/airbyte-webapp/src/components/connection/TransformationHookForm/TransformationHookForm.tsx b/airbyte-webapp/src/components/connection/TransformationHookForm/TransformationHookForm.tsx new file mode 100644 index 00000000000..d010b9d1f39 --- /dev/null +++ b/airbyte-webapp/src/components/connection/TransformationHookForm/TransformationHookForm.tsx @@ -0,0 +1,104 @@ +import React from "react"; +import { useIntl } from "react-intl"; + +import { Form, FormControl } from "components/forms"; +import { ModalFormSubmissionButtons } from "components/forms/ModalFormSubmissionButtons"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { ModalBody, ModalFooter } from "components/ui/Modal"; + +import { useOperationsCheck } from "core/api"; +import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; + +import { dbtOperationReadOrCreateSchema } from "./schema"; +import { DbtOperationReadOrCreate } from "./types"; + +interface TransformationHookFormProps { + transformation: DbtOperationReadOrCreate; + onDone: (tr: DbtOperationReadOrCreate) => void; + onCancel: () => void; +} + +/** + * react-hook-form Form for create/update transformation + * old version of TransformationField + * @see TransformationForm + * @param transformation + * @param onDone + * @param onCancel + * @constructor + */ +export const TransformationHookForm: React.FC = ({ transformation, onDone, onCancel }) => { + const { formatMessage } = useIntl(); + const operationCheck = useOperationsCheck(); + const { clearFormChange } = useFormChangeTrackerService(); + const formId = useUniqueFormId(); + + const onSubmit = async (values: DbtOperationReadOrCreate) => { + await operationCheck(values); + clearFormChange(formId); + onDone(values); + }; + + const onFormCancel = () => { + clearFormChange(formId); + onCancel(); + }; + + return ( + + onSubmit={onSubmit} + schema={dbtOperationReadOrCreateSchema} + defaultValues={transformation} + // TODO: uncomment when trackDirtyChanges will be fixed + // trackDirtyChanges + > + + + + + + `<${node}>` } + )} + /> + + + + + + + + + + + + ); +}; diff --git a/airbyte-webapp/src/components/connection/TransformationHookForm/index.tsx b/airbyte-webapp/src/components/connection/TransformationHookForm/index.tsx new file mode 100644 index 00000000000..95f78547439 --- /dev/null +++ b/airbyte-webapp/src/components/connection/TransformationHookForm/index.tsx @@ -0,0 +1,3 @@ +export { TransformationHookForm } from "./TransformationHookForm"; +export { dbtOperationReadOrCreateSchema } from "./schema"; +export { type DbtOperationRead, type DbtOperationReadOrCreate } from "./types"; diff --git a/airbyte-webapp/src/components/connection/TransformationHookForm/schema.test.ts b/airbyte-webapp/src/components/connection/TransformationHookForm/schema.test.ts new file mode 100644 index 00000000000..7272471de60 --- /dev/null +++ b/airbyte-webapp/src/components/connection/TransformationHookForm/schema.test.ts @@ -0,0 +1,51 @@ +import merge from "lodash/merge"; +import { InferType, ValidationError } from "yup"; + +import { dbtOperationReadOrCreateSchema } from "./schema"; + +describe(" - validationSchema", () => { + const customTransformationFields: InferType = { + name: "test name", + workspaceId: "test workspace id", + operationId: undefined, + operatorConfiguration: { + operatorType: "dbt", + dbt: { + gitRepoUrl: "https://github.com/username/example.git", + dockerImage: undefined, + dbtArguments: undefined, + gitRepoBranch: "", + }, + }, + }; + + it("should successfully validate the schema", async () => { + await expect(dbtOperationReadOrCreateSchema.validate(customTransformationFields)).resolves.toBeTruthy(); + }); + + it("should fail if 'name' is empty", async () => { + await expect(async () => { + await dbtOperationReadOrCreateSchema.validateAt("name", { ...customTransformationFields, name: "" }); + }).rejects.toThrow(ValidationError); + }); + + it("should fail if 'gitRepoUrl' is invalid", async () => { + await expect(async () => { + await dbtOperationReadOrCreateSchema.validateAt( + "operatorConfiguration.dbt.gitRepoUrl", + merge(customTransformationFields, { + operatorConfiguration: { dbt: { gitRepoUrl: "" } }, + }) + ); + }).rejects.toThrow(ValidationError); + + await expect(async () => { + await dbtOperationReadOrCreateSchema.validateAt( + "operatorConfiguration.dbt.gitRepoUrl", + merge(customTransformationFields, { + operatorConfiguration: { dbt: { gitRepoUrl: "https://github.com/username/example.git/" } }, + }) + ); + }).rejects.toThrow(ValidationError); + }); +}); diff --git a/airbyte-webapp/src/components/connection/TransformationHookForm/schema.ts b/airbyte-webapp/src/components/connection/TransformationHookForm/schema.ts new file mode 100644 index 00000000000..c625f9a8375 --- /dev/null +++ b/airbyte-webapp/src/components/connection/TransformationHookForm/schema.ts @@ -0,0 +1,26 @@ +import { SchemaOf } from "yup"; +import * as yup from "yup"; + +import { DbtOperationReadOrCreate } from "./types"; + +export const dbtOperationReadOrCreateSchema: SchemaOf = yup.object().shape({ + workspaceId: yup.string().required("form.empty.error"), + operationId: yup.string().optional(), // during creation, this is not required + name: yup.string().required("form.empty.error"), + operatorConfiguration: yup + .object() + .shape({ + operatorType: yup.mixed().oneOf(["dbt"]).default("dbt"), + dbt: yup.object({ + gitRepoUrl: yup + .string() + .trim() + .matches(/((http(s)?)|(git@[\w.]+))(:(\/\/)?)([\w.@:/\-~]+)(\.git)$/, "form.repositoryUrl.invalidUrl") + .required("form.empty.error"), + gitRepoBranch: yup.string().optional(), + dockerImage: yup.string().optional(), + dbtArguments: yup.string().optional(), + }), + }) + .required(), +}); diff --git a/airbyte-webapp/src/components/connection/TransformationHookForm/types.ts b/airbyte-webapp/src/components/connection/TransformationHookForm/types.ts new file mode 100644 index 00000000000..d1691f346df --- /dev/null +++ b/airbyte-webapp/src/components/connection/TransformationHookForm/types.ts @@ -0,0 +1,15 @@ +import { OperationId, OperatorDbt } from "core/request/AirbyteClient"; + +export interface DbtOperationRead { + name: string; + workspaceId: string; + operationId: OperationId; + operatorConfiguration: { + operatorType: "dbt"; + dbt: OperatorDbt; + }; +} + +export interface DbtOperationReadOrCreate extends Omit { + operationId?: OperationId; +} diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 47915c3812a..c2778117ef6 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -559,7 +559,11 @@ "connection.replicationFrequency": "Replication frequency*", "connection.replicationFrequency.subtitle": "Set how often data should sync to the destination", "connection.normalization": "Normalization", + "connection.normalization.successMessage": "Normalization settings were updated successfully!", + "connection.normalization.errorMessage": "There was an error during updating your normalization settings", "connection.customTransformations": "Custom Transformations", + "connection.customTransformations.successMessage": "Custom transformation settings were updated successfully!", + "connection.customTransformations.errorMessage": "There was an error during updating your custom transformation settings", "tables.name": "Name", "tables.connector": "Connector", diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx index 88a58210773..ecbb97b3880 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsCard.tsx @@ -6,11 +6,17 @@ import { useToggle } from "react-use"; import { ConnectionEditFormCard } from "components/connection/ConnectionEditFormCard"; import { getInitialTransformations } from "components/connection/ConnectionForm/formConfig"; import { TransformationField } from "components/connection/ConnectionForm/TransformationField"; +import { DbtOperationReadOrCreate } from "components/connection/TransformationHookForm"; import { OperationCreate, OperationRead } from "core/request/AirbyteClient"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { FormikOnSubmit } from "types/formik"; +// will be used in 2nd part of migration, TransformationFieldHookForm refers to this interface +export interface CustomTransformationsFormValues { + transformations: DbtOperationReadOrCreate[]; +} + export const CustomTransformationsCard: React.FC<{ operations?: OperationCreate[]; onSubmit: FormikOnSubmit<{ transformations?: OperationRead[] }>; diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.module.scss b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.module.scss index 2713a5f8d9c..5961dfbcff0 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.module.scss +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.module.scss @@ -28,7 +28,11 @@ } .statusHeader { - width: 0; // makes the column width dynamic, matching the widest content in this column + width: 0; + } + + .statusCell { + white-space: nowrap; } .actionsHeader { diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.tsx b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.tsx index 0ee312ecd65..3a6a37f2b81 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.tsx +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.tsx @@ -67,7 +67,7 @@ export const StreamsList = () => { id: "statusIcon", header: () => , cell: (props) => ( - + diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index dc1acc3f73a..bc86a638715 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -158,7 +158,8 @@ private CheckedSupplier, Exception> getLegacyWork processFactory, new DefaultNormalizationRunner( processFactory, destinationLauncherConfig.getNormalizationDockerImage(), - destinationLauncherConfig.getNormalizationIntegrationType()))); + destinationLauncherConfig.getNormalizationIntegrationType())), + () -> {}); } @SuppressWarnings("LineLength") diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index f921a5af374..dd36276dde2 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -251,7 +251,7 @@ private CheckedSupplier, Except processFactory, destinationLauncherConfig.getNormalizationDockerImage(), destinationLauncherConfig.getNormalizationIntegrationType()), - workerEnvironment); + workerEnvironment, () -> {}); } @SuppressWarnings("LineLength") diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java index 5d0efa8d379..a27b3a59477 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java @@ -17,6 +17,6 @@ public interface RefreshSchemaActivity { @ActivityMethod boolean shouldRefreshSchema(UUID sourceCatalogId); - public void refreshSchema(UUID sourceCatalogId, UUID connectionId); + public void refreshSchema(UUID sourceCatalogId, UUID connectionId) throws Exception; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index 4e6dd224b83..838e3477065 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -73,13 +73,13 @@ public boolean shouldRefreshSchema(final UUID sourceCatalogId) { @Override @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) - public void refreshSchema(final UUID sourceId, final UUID connectionId) { + public void refreshSchema(final UUID sourceId, final UUID connectionId) throws Exception { if (!envVariableFeatureFlags.autoDetectSchema()) { return; } final UUID sourceDefinitionId = - AirbyteApiClient.retryWithJitter(() -> sourceApi.getSource(new SourceIdRequestBody().sourceId(sourceId)).getSourceDefinitionId(), + AirbyteApiClient.retryWithJitterThrows(() -> sourceApi.getSource(new SourceIdRequestBody().sourceId(sourceId)).getSourceDefinitionId(), "Get the source definition id by source id"); final List featureFlagContexts = List.of(new SourceDefinition(sourceDefinitionId), new Connection(connectionId)); @@ -93,31 +93,13 @@ public void refreshSchema(final UUID sourceId, final UUID connectionId) { final SourceDiscoverSchemaRequestBody requestBody = new SourceDiscoverSchemaRequestBody().sourceId(sourceId).disableCache(true).connectionId(connectionId).notifySchemaChange(true); - final SourceDiscoverSchemaRead sourceDiscoverSchemaRead; + final SourceDiscoverSchemaRead sourceDiscoverSchemaRead = AirbyteApiClient.retryWithJitterThrows( + () -> sourceApi.discoverSchemaForSource(requestBody), + "Trigger discover schema"); - try { - sourceDiscoverSchemaRead = AirbyteApiClient.retryWithJitter( - () -> sourceApi.discoverSchemaForSource(requestBody), - "Trigger discover schema"); - } catch (final Exception e) { - ApmTraceUtils.addExceptionToTrace(e); - // catching this exception because we don't want to block replication due to a failed schema refresh - log.error("Attempted schema refresh, but failed with error: ", e); - return; - } - - final UUID workspaceId; - - try { - workspaceId = AirbyteApiClient.retryWithJitter( - () -> workspaceApi.getWorkspaceByConnectionId(new ConnectionIdRequestBody().connectionId(connectionId)).getWorkspaceId(), - "Get the workspace by connection Id"); - } catch (final Exception e) { - ApmTraceUtils.addExceptionToTrace(e); - // catching this exception because we don't want to block replication due to a failed schema refresh - log.error("Attempted fetching workspace by connection id, but failed with error: ", e); - return; - } + final UUID workspaceId = AirbyteApiClient.retryWithJitterThrows( + () -> workspaceApi.getWorkspaceByConnectionId(new ConnectionIdRequestBody().connectionId(connectionId)).getWorkspaceId(), + "Get the workspace by connection Id"); final boolean autoPropagationIsEnabledForWorkspace = featureFlagClient.boolVariation(AutoPropagateSchema.INSTANCE, new Workspace(workspaceId)); @@ -128,18 +110,12 @@ public void refreshSchema(final UUID sourceId, final UUID connectionId) { .workspaceId(workspaceId) .catalogId(sourceDiscoverSchemaRead.getCatalogId()); - try { - AirbyteApiClient.retryWithJitter( - () -> { - sourceApi.applySchemaChangeForSource(sourceAutoPropagateChange); - return null; - }, - "Auto propagate the schema change"); - } catch (final Exception e) { - ApmTraceUtils.addExceptionToTrace(e); - // catching this exception because we don't want to block replication due to a failed schema refresh - log.error("Attempted schema propagation, but failed with error: ", e); - } + AirbyteApiClient.retryWithJitterThrows( + () -> { + sourceApi.applySchemaChangeForSource(sourceAutoPropagateChange); + return null; + }, + "Auto propagate the schema change"); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 9e6e1ebb97e..e221cd95cb2 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -100,6 +100,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, try { refreshSchemaActivity.refreshSchema(sourceId.get(), connectionId); } catch (final Exception e) { + ApmTraceUtils.addExceptionToTrace(e); return SyncOutputProvider.getRefreshSchemaFailure(e); } } diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index 89d5835fc55..87735d63b45 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -156,38 +156,165 @@ airbyte: cpu-request: ${REPLICATION_ORCHESTRATOR_CPU_REQUEST:} memory-limit: ${REPLICATION_ORCHESTRATOR_MEMORY_LIMIT:} memory-request: ${REPLICATION_ORCHESTRATOR_MEMORY_REQUEST:} + + # Generally low resource containers + destination-stderr: + cpu-limit: 0.5 + cpu-request: 0.1 + memory-limit: 50Mi + memory-request: 25Mi + destination-stdout: + cpu-limit: 0.5 + cpu-request: 0.1 + memory-limit: 50Mi + memory-request: 25Mi + heartbeat: + cpu-limit: 0.2 + cpu-request: 0.05 + memory-limit: 50Mi + memory-request: 25Mi + source-stderr: + cpu-limit: 0.5 + cpu-request: 0.1 + memory-limit: 50Mi + memory-request: 25Mi + # Default base sync values + destination: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 1Gi + destination-stdin: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + orchestrator: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 2Gi source: - cpu-limit: ${SOURCE_CONTAINER_CPU_LIMIT:} - cpu-request: ${SOURCE_CONTAINER_CPU_REQUEST:0.5} - memory-limit: ${SOURCE_CONTAINER_MEMORY_LIMIT:} - memory-request: ${SOURCE_CONTAINER_MEMORY_REQUEST:} + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 1Gi + source-stdout: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + # Database syncs resource override source-database: - cpu-limit: ${SOURCE_DATABASE_CONTAINER_CPU_LIMIT:} - cpu-request: ${SOURCE_DATABASE_CONTAINER_CPU_REQUEST:1} - memory-limit: ${SOURCE_DATABASE_CONTAINER_MEMORY_LIMIT:} - memory-request: ${SOURCE_DATABASE_CONTAINER_MEMORY_REQUEST:} - lowrss--heartbeat: + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 1Gi + source-stdout-database: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + orchestrator-database: + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 2Gi + destination-database: + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 1Gi + destination-stdin-database: + cpu-limit: 2 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + + # base is the proposed default syncs values + # Generally low resource containers + base--destination-stderr: cpu-limit: 0.5 - cpu-request: 0.05 + cpu-request: 0.1 memory-limit: 50Mi memory-request: 25Mi - lowrss--destination-stderr: + base--destination-stdout: cpu-limit: 0.5 cpu-request: 0.1 memory-limit: 50Mi memory-request: 25Mi - lowrss--source-stderr: + base--heartbeat: + cpu-limit: 0.2 + cpu-request: 0.05 + memory-limit: 50Mi + memory-request: 25Mi + base--source-stderr: cpu-limit: 0.5 cpu-request: 0.1 memory-limit: 50Mi memory-request: 25Mi + # Default base sync values + base--destination: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 1Gi + base--destination-stdin: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + base--orchestrator: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 2Gi + base--source: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 1Gi + base--source-stdout: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + + # Database syncs resource override + base--source-database: + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 1Gi + base--source-stdout-database: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + base--orchestrator-database: + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 2Gi + base--destination-database: + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 1Gi + base--destination-stdin-database: + cpu-limit: 2 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + + # Currently in testing for APIs lowrss--source-api: - cpu-limit: 0.5 + cpu-limit: 1 cpu-request: 0.2 memory-limit: 2Gi memory-request: 1Gi lowrss--source-stdout-api: - cpu-limit: 0.5 + cpu-limit: 1 cpu-request: 0.2 memory-limit: 50Mi memory-request: 25Mi @@ -202,13 +329,35 @@ airbyte: memory-limit: 2Gi memory-request: 1Gi lowrss--destination-stdin-api: + cpu-limit: 1 + cpu-request: 0.2 + memory-limit: 50Mi + memory-request: 25Mi + + # Previous version of lowrss + lowrss2--source-api: + cpu-limit: 0.5 + cpu-request: 0.2 + memory-limit: 2Gi + memory-request: 1Gi + lowrss2--source-stdout-api: cpu-limit: 0.5 cpu-request: 0.2 memory-limit: 50Mi memory-request: 25Mi - lowrss--destination-stdout-api: + lowrss2--orchestrator-api: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 2Gi + lowrss2--destination-api: + cpu-limit: 1 + cpu-request: 0.2 + memory-limit: 2Gi + memory-request: 1Gi + lowrss2-destination-stdin-api: cpu-limit: 0.5 - cpu-request: 0.1 + cpu-request: 0.2 memory-limit: 50Mi memory-request: 25Mi verylowrss--source: diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/config/WorkerConfigProviderMicronautTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/config/WorkerConfigProviderMicronautTest.java index a9b2ef95214..558a918b4e2 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/config/WorkerConfigProviderMicronautTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/config/WorkerConfigProviderMicronautTest.java @@ -133,6 +133,14 @@ void testUnknownVariantFallsBackToDefaultVariant() { final ResourceRequirements sourceDatabase = workerConfigsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE, Optional.of("database")); assertEquals(sourceDatabase, unknownVariantSourceDatabase); + + // This is a corner case where the variant exists but not the type. We want to make sure + // it falls back to the default + final ResourceRequirements defaultOrchestratorApiReq = + workerConfigsProvider.getResourceRequirements(ResourceRequirementsType.ORCHESTRATOR, Optional.of("api")); + final ResourceRequirements unknownTypeInVariant = + workerConfigsProvider.getResourceRequirements(ResourceRequirementsType.ORCHESTRATOR, Optional.of("api"), "incompletevariant"); + assertEquals(defaultOrchestratorApiReq, unknownTypeInVariant); } @Test diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index a2297f6fd00..0d91a4886fb 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -115,7 +115,7 @@ void testShouldRefreshSchemaRecentRefreshLessThanValueFromFF() throws ApiExcepti } @Test - void testRefreshSchema() throws ApiException { + void testRefreshSchema() throws Exception { final UUID sourceId = UUID.randomUUID(); final UUID connectionId = UUID.randomUUID(); final UUID catalogId = UUID.randomUUID(); @@ -160,7 +160,7 @@ void testRefreshSchema() throws ApiException { } @Test - void testRefreshSchemaWithRefreshSchemaFeatureFlagAsFalse() throws ApiException { + void testRefreshSchemaWithRefreshSchemaFeatureFlagAsFalse() throws Exception { final UUID sourceId = UUID.randomUUID(); final UUID connectionId = UUID.randomUUID(); final UUID sourceDefinitionId = UUID.randomUUID(); @@ -176,7 +176,7 @@ void testRefreshSchemaWithRefreshSchemaFeatureFlagAsFalse() throws ApiException } @Test - void testRefreshSchemaWithAutoPropagateFeatureFlagAsFalse() throws ApiException { + void testRefreshSchemaWithAutoPropagateFeatureFlagAsFalse() throws Exception { final UUID sourceId = UUID.randomUUID(); final UUID connectionId = UUID.randomUUID(); final UUID workspaceId = UUID.randomUUID(); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 6da04e833c6..8e16fa5c89a 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -225,7 +225,7 @@ private StandardSyncOutput execute() { } @Test - void testSuccess() { + void testSuccess() throws Exception { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -251,7 +251,7 @@ void testSuccess() { } @Test - void testReplicationFailure() { + void testReplicationFailure() throws Exception { doThrow(new IllegalArgumentException("induced exception")).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -268,7 +268,7 @@ void testReplicationFailure() { } @Test - void testReplicationFailedGracefully() { + void testReplicationFailedGracefully() throws Exception { doReturn(replicationFailOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -294,7 +294,7 @@ void testReplicationFailedGracefully() { } @Test - void testNormalizationFailure() { + void testNormalizationFailure() throws Exception { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -316,7 +316,7 @@ void testNormalizationFailure() { } @Test - void testCancelDuringReplication() { + void testCancelDuringReplication() throws Exception { doAnswer(ignored -> { cancelWorkflow(); return replicationSuccessOutput; @@ -336,7 +336,7 @@ void testCancelDuringReplication() { } @Test - void testCancelDuringNormalization() { + void testCancelDuringNormalization() throws Exception { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -362,7 +362,7 @@ void testCancelDuringNormalization() { @Test @Disabled("This behavior has been disabled temporarily (OC Issue #741)") - void testSkipNormalization() { + void testSkipNormalization() throws Exception { final SyncStats syncStats = new SyncStats().withRecordsCommitted(0L); final StandardSyncSummary standardSyncSummary = new StandardSyncSummary().withTotalStats(syncStats); final StandardSyncOutput replicationSuccessOutputNoRecordsCommitted = @@ -405,7 +405,7 @@ void testWebhookOperation() { } @Test - void testSkipReplicationAfterRefreshSchema() { + void testSkipReplicationAfterRefreshSchema() throws Exception { when(configFetchActivity.getStatus(any())).thenReturn(Optional.of(ConnectionStatus.INACTIVE)); final StandardSyncOutput output = execute(); verifyShouldRefreshSchema(refreshSchemaActivity); @@ -416,7 +416,7 @@ void testSkipReplicationAfterRefreshSchema() { } @Test - void testGetProperFailureIfRefreshFails() { + void testGetProperFailureIfRefreshFails() throws Exception { when(refreshSchemaActivity.shouldRefreshSchema(any())).thenReturn(true); doThrow(new RuntimeException()) .when(refreshSchemaActivity).refreshSchema(any(), any()); @@ -473,7 +473,7 @@ private static void verifyShouldRefreshSchema(final RefreshSchemaActivity refres verify(refreshSchemaActivity).shouldRefreshSchema(SOURCE_ID); } - private static void verifyRefreshSchema(final RefreshSchemaActivity refreshSchemaActivity, final StandardSync sync) { + private static void verifyRefreshSchema(final RefreshSchemaActivity refreshSchemaActivity, final StandardSync sync) throws Exception { verify(refreshSchemaActivity).refreshSchema(SOURCE_ID, sync.getConnectionId()); } diff --git a/airbyte-workers/src/test/resources/application-config-test.yaml b/airbyte-workers/src/test/resources/application-config-test.yaml index 5531e78da4d..1e09bf89046 100644 --- a/airbyte-workers/src/test/resources/application-config-test.yaml +++ b/airbyte-workers/src/test/resources/application-config-test.yaml @@ -20,12 +20,18 @@ airbyte: memory-request: ${SOMETHING_NOT_THERE:} source: cpu-request: 0.5 + cpu-limit: "" # This unsets the value from the inherited test file to test defaults source-database: cpu-request: 1 + cpu-limit: "" # This unsets the value from the inherited test file to test defaults orchestrator-api: cpu-request: 11 destination-api: cpu-request: 12 + variantwithnumb3rz-source: # Making sure we support numbers + cpu-request: 13 + incompletevariant-source-api: + cpu-request: 42 micronauttest-source: cpu-limit: 5 micronauttest-source-database: diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 6f04e3e32ea..e80fb243b5c 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-api-server/templates/deployment.yaml b/charts/airbyte-api-server/templates/deployment.yaml index a4b81e7960f..862a664a36d 100644 --- a/charts/airbyte-api-server/templates/deployment.yaml +++ b/charts/airbyte-api-server/templates/deployment.yaml @@ -55,6 +55,16 @@ spec: image: {{ printf "%s:%s" .Values.image.repository (include "airbyte-api-server.imageTag" .) }} imagePullPolicy: "{{ .Values.image.pullPolicy }}" env: + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INTERNAL_API_HOST + - name: AIRBYTE_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_API_HOST {{- if .Values.debug.enabled }} - name: JAVA_TOOL_OPTIONS value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n" diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 337b9572992..71fd0ea47c3 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 4e74afff2a6..5c928f6a419 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index cbcbe6e62d3..97a919b5144 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 4d784a39133..445c9aa5458 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 09f64450a40..b296c97d54f 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 0e7a778836e..bb255f0989c 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 353a98f76c8..48f23f54d71 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index fad8ceae237..0861b5e3fbb 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index b3c963ef532..0016528779d 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 9c8ee5b80b2..2bc3bacfdb9 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 27bc28f9d29..c5d08664319 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 48c5323f5a3..a3302dcfe29 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 -digest: sha256:920cc0bfc8bbb0232d8bc013c958749dae9df30d34dd17a903bcb633444b5d0e -generated: "2023-07-24T21:10:46.154024665Z" + version: 0.47.15 +digest: sha256:395bce5e05fd1f7e4a6c656cb405893f01e7d04b1126335a52cb65b957d9659b +generated: "2023-08-01T20:18:02.868952081Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 3c7e17012eb..9f5d6495ece 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.47.13 +version: 0.47.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 + version: 0.47.15 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.47.13 \ No newline at end of file + version: 0.47.15 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 3e42809ac08..93e7c39798c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -261,6 +261,8 @@ services: environment: - AIRBYTE_VERSION=${VERSION} - DEPLOYMENT_MODE=${DEPLOYMENT_MODE} + - INTERNAL_API_HOST=${INTERNAL_API_HOST} + - AIRBYTE_API_HOST=${AIRBYTE_API_HOST} networks: - airbyte_internal depends_on: diff --git a/tools/bin/install_airbyte_pro_on_helm.sh b/tools/bin/install_airbyte_pro_on_helm.sh index 1a0f6c91f34..ef61a74125d 100755 --- a/tools/bin/install_airbyte_pro_on_helm.sh +++ b/tools/bin/install_airbyte_pro_on_helm.sh @@ -8,7 +8,7 @@ airbyte_yml_file_path="$script_dir/../../airbyte.yml" airbyte_pro_values_yml_file_path="$script_dir/../../charts/airbyte/airbyte-pro-values.yaml" # Define the helm release name for this installation of Airbyte Pro. -if [ ! -z RELEASE_NAME ]; then +if [ ! -z "$RELEASE_NAME" ]; then airbyte_pro_release_name="$RELEASE_NAME" else # Default release name. Change this to your liking.