Merge branch 'airbytehq:main' into main

airbytehq · Jul 11, 2024 · a87835c · a87835c
2 parents dd5286e + 0da1d9c
commit a87835c
Show file tree

Hide file tree

Showing 553 changed files with 10,172 additions and 14,793 deletions.
diff --git a/.env b/.env
@@ -88,11 +88,6 @@ JOB_MAIN_CONTAINER_CPU_LIMIT=
 JOB_MAIN_CONTAINER_MEMORY_REQUEST=
 JOB_MAIN_CONTAINER_MEMORY_LIMIT=
 
-NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=
-NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=
-NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=
-NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=
-
 ### LOGGING/MONITORING/TRACKING ###
 TRACKING_STRATEGY=segment
 SEGMENT_WRITE_KEY=7UDdp5K55CyiGgsauOr2pNNujGvmhaeu

diff --git a/airbyte-api/build.gradle.kts b/airbyte-api/build.gradle.kts
@@ -467,6 +467,7 @@ val genConnectorBuilderServerApiClient = tasks.register<GenerateTask>("genConnec
       "enumPropertyNaming"  to "UPPERCASE",
       "generatePom"         to "false",
       "interfaceOnly"       to "true",
+      "serializationLibrary" to "jackson",
     )
 
     doLast {

diff --git a/airbyte-api/src/main/kotlin/config/ClientSupportFactory.kt b/airbyte-api/src/main/kotlin/config/ClientSupportFactory.kt
@@ -129,20 +129,20 @@ class ClientSupportFactory {
           r.counter(
             "$metricPrefix.abort",
             *metricTags,
-            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.result.request.method),
-            *getUrlTags(l.result.request.url),
+            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.result?.request?.method ?: UNKNOWN),
+            *getUrlTags(l.result?.request?.url),
           ).increment()
         }
       }
       .onFailure { l ->
-        logger.error(l.exception) { "Failed to call ${l.result.request.url}.  Last response: ${l.result}" }
+        logger.error(l.exception) { "Failed to call ${l.result?.request?.url ?: UNKNOWN}.  Last response: ${l.result}" }
         meterRegistry.ifPresent {
             r ->
           r.counter(
             "$metricPrefix.failure",
             *metricTags,
-            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.result.request.method),
-            *getUrlTags(l.result.request.url),
+            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.result?.request?.method ?: UNKNOWN),
+            *getUrlTags(l.result?.request?.url),
           ).increment()
         }
       }
@@ -153,8 +153,8 @@ class ClientSupportFactory {
           r.counter(
             "$metricPrefix.retry",
             *metricTags,
-            *arrayOf("retry-attempt", l.attemptCount.toString(), "url", "method", l.lastResult.request.method),
-            *getUrlTags(l.lastResult.request.url),
+            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.lastResult?.request?.method ?: UNKNOWN),
+            *getUrlTags(l.lastResult?.request?.url),
           ).increment()
         }
       }
@@ -165,8 +165,8 @@ class ClientSupportFactory {
           r.counter(
             "$metricPrefix.retries_exceeded",
             *metricTags,
-            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.result.request.method),
-            *getUrlTags(l.result.request.url),
+            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.result?.request?.method ?: UNKNOWN),
+            *getUrlTags(l.result?.request?.url),
           ).increment()
         }
       }
@@ -177,8 +177,8 @@ class ClientSupportFactory {
           r.counter(
             "$metricPrefix.success",
             *metricTags,
-            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.result.request.method),
-            *getUrlTags(l.result.request.url),
+            *arrayOf("retry-attempt", l.attemptCount.toString(), "method", l.result?.request?.method ?: UNKNOWN),
+            *getUrlTags(l.result?.request?.url),
           ).increment()
         }
       }
@@ -188,12 +188,18 @@ class ClientSupportFactory {
       .build()
   }
 
-  private fun getUrlTags(httpUrl: HttpUrl): Array<String> {
-    val last = httpUrl.pathSegments.last()
-    if (last.contains("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}".toRegex())) {
-      return arrayOf("url", httpUrl.toString().removeSuffix(last), "workload-id", last)
-    } else {
-      return arrayOf("url", httpUrl.toString())
-    }
+  private fun getUrlTags(httpUrl: HttpUrl?): Array<String> {
+    return httpUrl?.let {
+      val last = httpUrl.pathSegments.last()
+      if (last.contains("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}".toRegex())) {
+        return arrayOf("url", httpUrl.toString().removeSuffix(last), "workload-id", last)
+      } else {
+        return arrayOf("url", httpUrl.toString())
+      }
+    } ?: emptyArray()
+  }
+
+  companion object {
+    private const val UNKNOWN = "unknown"
   }
 }
diff --git a/airbyte-api/src/main/kotlin/config/InternalApiAuthenticationFactory.kt b/airbyte-api/src/main/kotlin/config/InternalApiAuthenticationFactory.kt
@@ -8,6 +8,7 @@ import com.auth0.jwt.JWT
 import com.auth0.jwt.JWTCreator
 import com.google.auth.oauth2.ServiceAccountCredentials
 import io.github.oshai.kotlinlogging.KotlinLogging
+import io.micrometer.core.instrument.MeterRegistry
 import io.micronaut.context.annotation.Factory
 import io.micronaut.context.annotation.Primary
 import io.micronaut.context.annotation.Prototype
@@ -18,6 +19,7 @@ import jakarta.inject.Singleton
 import java.io.FileInputStream
 import java.security.interfaces.RSAPrivateKey
 import java.util.Date
+import java.util.Optional
 import java.util.concurrent.TimeUnit
 
 private val logger = KotlinLogging.logger {}
@@ -61,6 +63,7 @@ class InternalApiAuthenticationFactory {
     @Value("\${airbyte.control.plane.auth-endpoint}") controlPlaneAuthEndpoint: String,
     @Value("\${airbyte.data.plane.service-account.email}") dataPlaneServiceAccountEmail: String,
     @Value("\${airbyte.data.plane.service-account.credentials-path}") dataPlaneServiceAccountCredentialsPath: String,
+    meterRegistry: Optional<MeterRegistry>,
   ): String {
     return try {
       val now = Date()
@@ -82,9 +85,12 @@ class InternalApiAuthenticationFactory {
       val cred = ServiceAccountCredentials.fromStream(stream)
       val key = cred.privateKey as RSAPrivateKey
       val algorithm: com.auth0.jwt.algorithms.Algorithm = com.auth0.jwt.algorithms.Algorithm.RSA256(null, key)
-      "Bearer " + token.sign(algorithm)
+      val signedToken = token.sign(algorithm)
+      meterRegistry.ifPresent { registry -> registry.counter("airbyte-api-client.auth-token.success").increment() }
+      return "Bearer $signedToken"
     } catch (e: Exception) {
-      logger.error(e) { "An issue occurred while generating a data plane auth token. Defaulting to empty string. Error Message: {}" }
+      meterRegistry.ifPresent { registry -> registry.counter("airbyte-api-client.auth-token.failure").increment() }
+      logger.error(e) { "An issue occurred while generating a data plane auth token. Defaulting to empty string." }
       ""
     }
   }

diff --git a/airbyte-api/src/main/openapi/cloud-config.yaml b/airbyte-api/src/main/openapi/cloud-config.yaml
@@ -1622,9 +1622,11 @@ components:
       properties:
         accountId:
           type: integer
+          format: int64
           description: The account id associated with the job
         jobId:
           type: integer
+          format: int64
           description: The the specific job id returned by the dbt Cloud API
         jobName:
           type: string