CDK s3-destinations: fixes for s3 connector compilation (#36868)

airbyte-cdk/java/airbyte-cdk/README.md

@@ -144,6 +144,7 @@ Maven and Gradle will automatically reference the correct (pinned) version of th
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.29.8  | 2024-04-08 | [\#36868](https://github.com/airbytehq/airbyte/pull/36868) | Destinations: s3-destinations Compilation fixes for connector                                                                                                  |
 | 0.29.7  | 2024-04-08 | [\#36768](https://github.com/airbytehq/airbyte/pull/36768) | Destinations: Make destination state fetch/commit logic more resilient to errors                                                                               |
 | 0.29.6  | 2024-04-05 | [\#36577](https://github.com/airbytehq/airbyte/pull/36577) | Do not send system_error trace message for config exceptions.                                                                                                  |
 | 0.29.5  | 2024-04-05 | [\#36620](https://github.com/airbytehq/airbyte/pull/36620) | Missed changes - open for extension for destination-postgres                                                                                                   |

airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.kt

@@ -20,12 +20,13 @@ object AdaptiveDestinationRunner {
     private const val DEPLOYMENT_MODE_KEY = EnvVariableFeatureFlags.DEPLOYMENT_MODE
     private const val CLOUD_MODE = "CLOUD"
 
+    @JvmStatic
     fun baseOnEnv(): OssDestinationBuilder {
         val mode = System.getenv(DEPLOYMENT_MODE_KEY)
         return OssDestinationBuilder(mode)
     }
 
-    class OssDestinationBuilder(private val deploymentMode: String) {
+    class OssDestinationBuilder(private val deploymentMode: String?) {
         fun <OT : Destination> withOssDestination(
             ossDestinationSupplier: Supplier<OT>
         ): CloudDestinationBuilder<OT> {
@@ -34,7 +35,7 @@ object AdaptiveDestinationRunner {
     }
 
     class CloudDestinationBuilder<OT : Destination>(
-        private val deploymentMode: String,
+        private val deploymentMode: String?,
         private val ossDestinationSupplier: Supplier<OT>
     ) {
         fun <CT : Destination> withCloudDestination(

airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

@@ -1 +1 @@
-version=0.29.7
+version=0.29.8

airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.kt

@@ -38,10 +38,9 @@ object GeneralStagingFunctions {
         typerDeduper: TyperDeduper
     ): OnStartFunction {
         return OnStartFunction {
-            log.info(
-                "Preparing raw tables in destination started for {} streams",
-                writeConfigs.size
-            )
+            log.info {
+                "Preparing raw tables in destination started for ${writeConfigs.size} streams"
+            }
             typerDeduper.prepareSchemasAndRunMigrations()
 
             // Create raw tables
@@ -53,20 +52,16 @@ object GeneralStagingFunctions {
                 val stageName = stagingOperations.getStageName(schema, dstTableName)
                 val stagingPath =
                     stagingOperations.getStagingPath(
-                        SerialStagingConsumerFactory.Companion.RANDOM_CONNECTION_ID,
+                        RANDOM_CONNECTION_ID,
                         schema,
                         stream,
                         writeConfig.outputTableName,
                         writeConfig.writeDatetime
                     )
 
-                log.info(
-                    "Preparing staging area in destination started for schema {} stream {}: target table: {}, stage: {}",
-                    schema,
-                    stream,
-                    dstTableName,
-                    stagingPath
-                )
+                log.info {
+                    "Preparing staging area in destination started for schema $schema stream $stream: target table: $dstTableName, stage: $stagingPath"
+                }
 
                 stagingOperations.createSchemaIfNotExists(database, schema)
                 stagingOperations.createTableIfNotExists(database, schema, dstTableName)
@@ -84,16 +79,14 @@ object GeneralStagingFunctions {
                             "Unrecognized sync mode: " + writeConfig.syncMode
                         )
                 }
-                log.info(
-                    "Preparing staging area in destination completed for schema {} stream {}",
-                    schema,
-                    stream
-                )
+                log.info {
+                    "Preparing staging area in destination completed for schema $schema stream $stream"
+                }
             }
 
             typerDeduper.prepareFinalTables()
 
-            log.info("Executing finalization of tables.")
+            log.info { "Executing finalization of tables." }
             stagingOperations.executeTransaction(database, queryList)
         }
     }
@@ -167,7 +160,7 @@ object GeneralStagingFunctions {
             // After moving data from staging area to the target table (airybte_raw) clean up the
             // staging
             // area (if user configured)
-            log.info("Cleaning up destination started for {} streams", writeConfigs.size)
+            log.info { "Cleaning up destination started for ${writeConfigs.size} streams" }
             typerDeduper.typeAndDedupe(streamSyncSummaries)
             for (writeConfig in writeConfigs) {
                 val schemaName = writeConfig.outputSchemaName
@@ -182,12 +175,9 @@ object GeneralStagingFunctions {
                             writeConfig.outputTableName,
                             writeConfig.writeDatetime
                         )
-                    log.info(
-                        "Cleaning stage in destination started for stream {}. schema {}, stage: {}",
-                        writeConfig.streamName,
-                        schemaName,
-                        stagePath
-                    )
+                    log.info {
+                        "Cleaning stage in destination started for stream ${writeConfig.streamName}. schema $schemaName, stage: $stagePath"
+                    }
                     // TODO: This is another weird manifestation of Redshift vs Snowflake using
                     // either or variables from
                     // stageName/StagingPath.
@@ -196,7 +186,7 @@ object GeneralStagingFunctions {
             }
             typerDeduper.commitFinalTables()
             typerDeduper.cleanup()
-            log.info("Cleaning up destination completed.")
+            log.info { "Cleaning up destination completed." }
         }
     }
 }

airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt

@@ -429,6 +429,7 @@ abstract class DestinationAcceptanceTest {
         val configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog)
         val messages: List<io.airbyte.protocol.models.v0.AirbyteMessage> =
             MoreResources.readResource(messagesFilename)
+                .trim()
                 .lines()
                 .map {
                     Jsons.deserialize(it, io.airbyte.protocol.models.v0.AirbyteMessage::class.java)
@@ -458,6 +459,7 @@ abstract class DestinationAcceptanceTest {
         val configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog)
         val messages: List<io.airbyte.protocol.models.v0.AirbyteMessage> =
             MoreResources.readResource(messagesFilename)
+                .trim()
                 .lines()
                 .map {
                     Jsons.deserialize(it, io.airbyte.protocol.models.v0.AirbyteMessage::class.java)
@@ -515,6 +517,7 @@ abstract class DestinationAcceptanceTest {
                         getProtocolVersion()
                     )
                 )
+                .trim()
                 .lines()
                 .map {
                     Jsons.deserialize<io.airbyte.protocol.models.v0.AirbyteMessage>(
@@ -712,6 +715,7 @@ abstract class DestinationAcceptanceTest {
                         getProtocolVersion()
                     )
                 )
+                .trim()
                 .lines()
                 .map { Jsons.deserialize(it, AirbyteMessage::class.java) }
                 .toList()
@@ -1406,6 +1410,7 @@ abstract class DestinationAcceptanceTest {
                         getProtocolVersion()
                     )
                 )
+                .trim()
                 .lines()
                 .map { Jsons.deserialize(it, AirbyteMessage::class.java) }
         val config = getConfig()
@@ -2328,7 +2333,7 @@ abstract class DestinationAcceptanceTest {
         private fun readMessagesFromFile(
             messagesFilename: String
         ): List<io.airbyte.protocol.models.v0.AirbyteMessage> {
-            return MoreResources.readResource(messagesFilename).lines().map {
+            return MoreResources.readResource(messagesFilename).trim().lines().map {
                 Jsons.deserialize(it, AirbyteMessage::class.java)
             }
         }

airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/argproviders/DataArgumentsProvider.kt

@@ -43,6 +43,7 @@ class DataArgumentsProvider : ArgumentsProvider {
     }
 
     companion object {
+        @JvmField
         val EXCHANGE_RATE_CONFIG: CatalogMessageTestConfigPair =
             CatalogMessageTestConfigPair("exchange_rate_catalog.json", "exchange_rate_messages.txt")
         val EDGE_CASE_CONFIG: CatalogMessageTestConfigPair =

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/BaseS3Destination.kt

@@ -29,7 +29,7 @@ protected constructor(
     override fun check(config: JsonNode): AirbyteConnectionStatus? {
         try {
             val destinationConfig = configFactory.getS3DestinationConfig(config, storageProvider())
-            val s3Client = destinationConfig!!.getS3Client()
+            val s3Client = destinationConfig.getS3Client()
 
             S3BaseChecks.testIAMUserHasListObjectPermission(s3Client, destinationConfig.bucketName)
             S3BaseChecks.testSingleUpload(
@@ -64,7 +64,7 @@ protected constructor(
         return S3ConsumerFactory()
             .create(
                 outputRecordCollector,
-                S3StorageOperations(nameTransformer, s3Config!!.getS3Client(), s3Config),
+                S3StorageOperations(nameTransformer, s3Config.getS3Client(), s3Config),
                 nameTransformer,
                 getCreateFunction(
                     s3Config,

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/BlobStorageOperations.kt

@@ -12,7 +12,7 @@ abstract class BlobStorageOperations protected constructor() {
     protected val blobDecorators: MutableList<BlobDecorator> = ArrayList()
 
     abstract fun getBucketObjectPath(
-        namespace: String,
+        namespace: String?,
         streamName: String,
         writeDatetime: DateTime,
         customFormat: String
@@ -29,7 +29,7 @@ abstract class BlobStorageOperations protected constructor() {
     @Throws(Exception::class)
     abstract fun uploadRecordsToBucket(
         recordsData: SerializableBuffer,
-        namespace: String,
+        namespace: String?,
         objectPath: String
     ): String?
 
@@ -46,7 +46,7 @@ abstract class BlobStorageOperations protected constructor() {
      * @param pathFormat formatted string for the path
      */
     abstract fun cleanUpBucketObject(
-        namespace: String,
+        namespace: String?,
         streamName: String,
         objectPath: String,
         pathFormat: String

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseChecks.kt

@@ -91,6 +91,7 @@ object S3BaseChecks {
      *
      * @param endpoint URL string representing an accessible S3 bucket
      */
+    @JvmStatic
     fun testCustomEndpointSecured(endpoint: String?): Boolean {
         // if user does not use a custom endpoint, do not fail
         return if (endpoint == null || endpoint.length == 0) {

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/S3ConsumerFactory.kt

@@ -69,7 +69,7 @@ class S3ConsumerFactory {
                         pathFormat
                     )
                     storageOperations.cleanUpBucketObject(
-                        namespace!!,
+                        namespace,
                         stream,
                         outputBucketPath,
                         pathFormat
@@ -124,7 +124,7 @@ class S3ConsumerFactory {
                     writeConfig!!.addStoredFile(
                         storageOperations.uploadRecordsToBucket(
                             writer,
-                            writeConfig.namespace!!,
+                            writeConfig.namespace,
                             writeConfig.fullOutputPath
                         )!!
                     )
@@ -183,7 +183,7 @@ class S3ConsumerFactory {
                     "Undefined destination sync mode"
                 )
                 val abStream = stream.stream
-                val namespace = abStream.namespace
+                val namespace: String? = abStream.namespace
                 val streamName = abStream.name
                 val bucketPath = s3Config.bucketPath
                 val customOutputFormat = java.lang.String.join("/", bucketPath, s3Config.pathFormat)

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfigFactory.kt

@@ -6,8 +6,8 @@ package io.airbyte.cdk.integrations.destination.s3
 import com.fasterxml.jackson.databind.JsonNode
 import javax.annotation.Nonnull
 
-class S3DestinationConfigFactory {
-    fun getS3DestinationConfig(
+open class S3DestinationConfigFactory {
+    open fun getS3DestinationConfig(
         config: JsonNode,
         @Nonnull storageProvider: StorageProvider
     ): S3DestinationConfig {

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/S3StorageOperations.kt

@@ -30,8 +30,6 @@ import java.util.concurrent.ConcurrentMap
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.regex.Pattern
 import org.apache.commons.io.FilenameUtils
-import org.apache.commons.lang3.StringUtils
-import org.apache.logging.log4j.util.Strings
 import org.joda.time.DateTime
 
 private val logger = KotlinLogging.logger {}
@@ -46,13 +44,13 @@ open class S3StorageOperations(
     private val partCounts: ConcurrentMap<String, AtomicInteger> = ConcurrentHashMap()
 
     override fun getBucketObjectPath(
-        namespace: String,
+        namespace: String?,
         streamName: String,
         writeDatetime: DateTime,
         customFormat: String
     ): String {
         val namespaceStr: String =
-            nameTransformer.getNamespace(if (Strings.isNotBlank(namespace)) namespace else "")
+            nameTransformer.getNamespace(if (!namespace.isNullOrBlank()) namespace else "")
         val streamNameStr: String = nameTransformer.getIdentifier(streamName)
         return nameTransformer.applyDefaultCase(
             customFormat
@@ -114,7 +112,7 @@ open class S3StorageOperations(
 
     override fun uploadRecordsToBucket(
         recordsData: SerializableBuffer,
-        namespace: String,
+        namespace: String?,
         objectPath: String
     ): String {
         val exceptionsThrown: MutableList<Exception> = ArrayList()
@@ -174,7 +172,7 @@ open class S3StorageOperations(
         val partId: String = getPartId(objectPath)
         val fileExtension: String = getExtension(recordsData.filename)
         val fullObjectKey: String =
-            if (StringUtils.isNotBlank(s3Config.fileNamePattern)) {
+            if (!s3Config.fileNamePattern.isNullOrBlank()) {
                 s3FilenameTemplateManager.applyPatternToFilename(
                     S3FilenameTemplateParameterObject.builder()
                         .partId(partId)
@@ -291,7 +289,7 @@ open class S3StorageOperations(
     }
 
     override fun cleanUpBucketObject(
-        namespace: String,
+        namespace: String?,
         streamName: String,
         objectPath: String,
         pathFormat: String

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.kt

@@ -108,6 +108,7 @@ abstract class S3AvroParquetDestinationAcceptanceTest protected constructor(s3Fo
     @Throws(IOException::class)
     private fun readMessagesFromFile(messagesFilename: String): List<AirbyteMessage> {
         return MoreResources.readResource(messagesFilename)
+            .trim()
             .lines()
             .map { record -> Jsons.deserialize(record, AirbyteMessage::class.java) }
             .toList()

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.kt

@@ -45,12 +45,15 @@ abstract class S3BaseCsvDestinationAcceptanceTest : S3DestinationAcceptanceTest(
         val fieldTypes = getFieldTypes(streamSchema)
         val jsonRecords: MutableList<JsonNode> = LinkedList()
 
-        for (objectSummary in objectSummaries!!) {
-            s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key).use { `object` ->
+        for (objectSummary in objectSummaries) {
+            s3Client!!.getObject(objectSummary.bucketName, objectSummary.key).use { `object` ->
                 getReader(`object`).use { `in` ->
                     val records: Iterable<CSVRecord> =
-                        CSVFormat.DEFAULT.withQuoteMode(QuoteMode.NON_NUMERIC)
-                            .withFirstRecordAsHeader()
+                        CSVFormat.Builder.create()
+                            .setHeader()
+                            .setSkipHeaderRecord(true)
+                            .setQuoteMode(QuoteMode.NON_NUMERIC)
+                            .build()
                             .parse(`in`)
                     StreamSupport.stream(records.spliterator(), false).forEach { r: CSVRecord ->
                         jsonRecords.add(getJsonNode(r.toMap(), fieldTypes))
@@ -87,7 +90,7 @@ abstract class S3BaseCsvDestinationAcceptanceTest : S3DestinationAcceptanceTest(
             input: Map<String, String>,
             fieldTypes: Map<String, String>
         ): JsonNode {
-            val json: ObjectNode = S3DestinationAcceptanceTest.Companion.MAPPER.createObjectNode()
+            val json: ObjectNode = MAPPER.createObjectNode()
 
             if (input.containsKey(JavaBaseConstants.COLUMN_NAME_DATA)) {
                 return Jsons.deserialize(input[JavaBaseConstants.COLUMN_NAME_DATA])
@@ -100,7 +103,7 @@ abstract class S3BaseCsvDestinationAcceptanceTest : S3DestinationAcceptanceTest(
                 ) {
                     continue
                 }
-                if (value == null || value == "") {
+                if (value == "") {
                     continue
                 }
                 val type = fieldTypes[key]

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvGzipDestinationAcceptanceTest.kt

@@ -11,14 +11,13 @@ import java.io.IOException
 import java.io.InputStreamReader
 import java.io.Reader
 import java.nio.charset.StandardCharsets
-import java.util.Map
 import java.util.zip.GZIPInputStream
 
 abstract class S3BaseCsvGzipDestinationAcceptanceTest : S3BaseCsvDestinationAcceptanceTest() {
     override val formatConfig: JsonNode?
         get() = // config without compression defaults to GZIP
         Jsons.jsonNode(
-                Map.of("format_type", outputFormat, "flattening", Flattening.ROOT_LEVEL.value)
+                mapOf("format_type" to outputFormat, "flattening" to Flattening.ROOT_LEVEL.value)
             )
 
     @Throws(IOException::class)