diff --git a/airbyte-cdk/bulk/core/base/build.gradle b/airbyte-cdk/bulk/core/base/build.gradle index 0b753f4c9aea..5ecf5624885e 100644 --- a/airbyte-cdk/bulk/core/base/build.gradle +++ b/airbyte-cdk/bulk/core/base/build.gradle @@ -13,29 +13,30 @@ dependencies { exclude group: 'org.apache.commons' exclude group: 'commons-io' } - api 'io.github.oshai:kotlin-logging-jvm:5.1.0' + api 'io.github.oshai:kotlin-logging-jvm:7.0.0' api 'io.micronaut:micronaut-runtime' - api 'org.apache.sshd:sshd-mina:2.12.1' + api 'org.apache.sshd:sshd-mina:2.13.2' api 'org.jetbrains.kotlinx:kotlinx-coroutines-core' - implementation 'com.datadoghq:dd-trace-api:1.28.0' - implementation 'com.datadoghq:dd-trace-ot:1.28.0' + implementation 'com.datadoghq:dd-trace-api:1.39.0' + implementation 'com.datadoghq:dd-trace-ot:1.39.0' implementation 'com.fasterxml.jackson.module:jackson-module-kotlin' implementation 'com.fasterxml.jackson.module:jackson-module-afterburner' implementation 'info.picocli:picocli:4.7.6' implementation 'io.micronaut.picocli:micronaut-picocli:5.5.0' - implementation 'jakarta.validation:jakarta.validation-api:3.0.2' - implementation 'org.apache.commons:commons-lang3:3.14.0' + implementation 'jakarta.validation:jakarta.validation-api:3.1.0' + implementation 'org.apache.commons:commons-lang3:3.17.0' implementation 'org.apache.logging.log4j:log4j-api' implementation 'org.apache.logging.log4j:log4j-core' implementation 'org.apache.logging.log4j:log4j-slf4j-impl' implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' - implementation 'org.apache.logging.log4j:log4j-layout-template-json:2.17.2' - implementation 'org.bouncycastle:bcprov-jdk18on:1.77' + implementation 'org.apache.logging.log4j:log4j-layout-template-json:2.24.0' + implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1' implementation 'org.openapi4j:openapi-schema-validator:1.0.7' - runtimeOnly 'com.google.guava:guava:33.2.0-jre' - runtimeOnly 'org.apache.commons:commons-compress:1.26.1' + // this is only needed because of exclusions in airbyte-protocol + runtimeOnly 'com.google.guava:guava:33.3.0-jre' + runtimeOnly 'org.apache.commons:commons-compress:1.27.1' testFixturesApi 'org.jetbrains.kotlin:kotlin-test-junit' testFixturesApi 'org.jetbrains.kotlin:kotlin-reflect' @@ -47,5 +48,5 @@ dependencies { } testFixturesApi 'io.micronaut.test:micronaut-test-core:4.5.0' testFixturesApi 'io.micronaut.test:micronaut-test-junit5:4.5.0' - testFixturesApi 'io.github.deblockt:json-diff:1.0.1' + testFixturesApi 'io.github.deblockt:json-diff:1.1.0' } diff --git a/airbyte-cdk/bulk/core/extract/build.gradle b/airbyte-cdk/bulk/core/extract/build.gradle index bc631cc3e37b..491109498d15 100644 --- a/airbyte-cdk/bulk/core/extract/build.gradle +++ b/airbyte-cdk/bulk/core/extract/build.gradle @@ -1,6 +1,6 @@ dependencies { implementation project(':airbyte-cdk:bulk:core:bulk-cdk-core-base') - implementation 'org.apache.commons:commons-lang3:3.14.0' + implementation 'org.apache.commons:commons-lang3:3.17.0' implementation 'hu.webarticum:tree-printer:3.2.1' testFixturesApi testFixtures(project(':airbyte-cdk:bulk:core:bulk-cdk-core-base')) diff --git a/airbyte-cdk/bulk/core/load/build.gradle b/airbyte-cdk/bulk/core/load/build.gradle index 6200c67acc44..e670f0cbe328 100644 --- a/airbyte-cdk/bulk/core/load/build.gradle +++ b/airbyte-cdk/bulk/core/load/build.gradle @@ -1,12 +1,12 @@ dependencies { implementation project(':airbyte-cdk:bulk:core:bulk-cdk-core-base') - implementation 'org.apache.commons:commons-lang3:3.14.0' + implementation 'org.apache.commons:commons-lang3:3.17.0' // For ranges and rangesets implementation("com.google.guava:guava:33.3.0-jre") testFixturesApi testFixtures(project(':airbyte-cdk:bulk:core:bulk-cdk-core-base')) - testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.1") - implementation "org.jetbrains.kotlin:kotlin-reflect:2.0.0" + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.1") + implementation "org.jetbrains.kotlin:kotlin-reflect:2.0.20" } diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 383120e84190..b423bfe4cacb 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -174,6 +174,7 @@ corresponds to that version. | Version | Date | Pull Request | Subject | |:-----------|:-----------|:------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.46.0 | 2024-09-18 | [\#45432](https://github.com/airbytehq/airbyte/pull/45432) | upgrade all libraries to latest version | | 0.45.1 | 2024-09-17 | [\#45638](https://github.com/airbytehq/airbyte/pull/45638) | upgrade apache mina sshd to 2.13.2 to handle openssh tcpkeepalive requests | | 0.45.0 | 2024-09-16 | [\#45469](https://github.com/airbytehq/airbyte/pull/45469) | Fix some race conditions, improve thread filtering, improve test logging | | 0.44.22 | 2024-09-10 | [\#45368](https://github.com/airbytehq/airbyte/pull/45368) | Remove excessive debezium logging | diff --git a/airbyte-cdk/java/airbyte-cdk/azure-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/azure-destinations/build.gradle index d0f2cc46ce1e..8485eae7cea9 100644 --- a/airbyte-cdk/java/airbyte-cdk/azure-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/azure-destinations/build.gradle @@ -2,7 +2,7 @@ dependencies { implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core') - implementation 'com.azure:azure-storage-blob:12.12.0' + implementation 'com.azure:azure-storage-blob:12.27.1' testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core')) } diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle index f5816a036486..24c6248a07f5 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -29,28 +29,28 @@ compileTestKotlin { dependencies { - api 'com.datadoghq:dd-trace-api:1.28.0' - api 'com.datadoghq:dd-trace-ot:1.28.0' + api 'com.datadoghq:dd-trace-api:1.39.0' + api 'com.datadoghq:dd-trace-ot:1.39.0' api 'com.zaxxer:HikariCP:5.1.0' - api 'org.jooq:jooq:3.16.23' - api 'org.apache.commons:commons-csv:1.10.0' + api 'org.jooq:jooq:3.19.11' + api 'org.apache.commons:commons-csv:1.11.0' implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') - implementation 'commons-cli:commons-cli:1.4' + implementation 'commons-cli:commons-cli:1.9.0' implementation 'io.aesy:datasize:1.0.0' implementation 'net.i2p.crypto:eddsa:0.3.0' implementation 'org.apache.httpcomponents:httpcore:4.4.16' - implementation 'org.apache.logging.log4j:log4j-layout-template-json:2.17.2' + implementation 'org.apache.logging.log4j:log4j-layout-template-json:2.24.0' implementation 'org.apache.sshd:sshd-mina:2.13.2' // bouncycastle is pinned to version-match the transitive dependency from kubernetes client-java // because a version conflict causes "parameter object not a ECParameterSpec" on ssh tunnel initiation - implementation 'org.bouncycastle:bcpkix-jdk15on:1.66' - implementation 'org.bouncycastle:bcprov-jdk15on:1.66' - implementation 'org.bouncycastle:bctls-jdk15on:1.66' + implementation 'org.bouncycastle:bcpkix-jdk18on:1.78.1' + implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1' + implementation 'org.bouncycastle:bctls-jdk18on:1.78.1' - testFixturesApi 'org.testcontainers:testcontainers:1.19.0' - testFixturesApi 'org.testcontainers:jdbc:1.19.0' + testFixturesApi 'org.testcontainers:testcontainers:1.20.1' + testFixturesApi 'org.testcontainers:jdbc:1.20.1' testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-db-sources')) @@ -58,8 +58,8 @@ dependencies { testImplementation 'mysql:mysql-connector-java:8.0.33' testImplementation 'org.postgresql:postgresql:42.7.3' - testImplementation 'org.testcontainers:mysql:1.19.0' - testImplementation 'org.testcontainers:postgresql:1.19.0' + testImplementation 'org.testcontainers:mysql:1.20.1' + testImplementation 'org.testcontainers:postgresql:1.20.1' testImplementation 'org.xbib.elasticsearch:joptsimple:6.3.2.1' - testImplementation 'org.mockito.kotlin:mockito-kotlin:5.2.1' + testImplementation 'org.mockito.kotlin:mockito-kotlin:5.4.0' } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index e673f85be9ea..1a5da107f67a 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.45.1 +version=0.46.0 diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle index 6e0be863c86e..4a0aff6e8a52 100644 --- a/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core') - api 'com.google.cloud:google-cloud-bigquery:2.37.0' + api 'com.google.cloud:google-cloud-bigquery:2.42.2' testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core')) } diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/kotlin/io/airbyte/cdk/db/bigquery/BigQueryDatabase.kt b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/kotlin/io/airbyte/cdk/db/bigquery/BigQueryDatabase.kt index a85b1698adcb..bdeae98fffa4 100644 --- a/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/kotlin/io/airbyte/cdk/db/bigquery/BigQueryDatabase.kt +++ b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/kotlin/io/airbyte/cdk/db/bigquery/BigQueryDatabase.kt @@ -115,7 +115,7 @@ constructor( val result = executeQuery(bigQuery, getQueryConfig(sql, params)) if (result.getLeft() != null) { - val fieldList = result.getLeft()!!.getQueryResults().schema.fields + val fieldList = result.getLeft()!!.getQueryResults().schema!!.fields return Streams.stream(result.getLeft()!!.getQueryResults().iterateAll()).map { fieldValues: FieldValueList -> sourceOperations!!.rowToJson(BigQueryResultSet(fieldValues, fieldList)) diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle index e135929d9120..a325af57c376 100644 --- a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle @@ -28,5 +28,5 @@ dependencies { testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core')) - testFixturesApi 'org.testcontainers:mongodb:1.19.0' + testFixturesApi 'org.testcontainers:mongodb:1.20.1' } diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle index 77db6b416e8f..44d9e2f062c1 100644 --- a/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle @@ -2,9 +2,9 @@ dependencies { implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core') - api 'org.postgresql:postgresql:42.6.2' + api 'org.postgresql:postgresql:42.7.4' testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core')) - testFixturesApi 'org.testcontainers:postgresql:1.19.0' + testFixturesApi 'org.testcontainers:postgresql:1.20.1' } diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/kotlin/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.kt b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/kotlin/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.kt index da46c5686252..522367e276b1 100644 --- a/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/kotlin/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.kt +++ b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/kotlin/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.kt @@ -62,7 +62,7 @@ object PostgresSslConnectionUtils { if (encryption.has(PARAM_CLIENT_KEY_PASSWORD)) encryption[PARAM_CLIENT_KEY_PASSWORD].asText() else "" - var keyStorePassword = RandomStringUtils.randomAlphanumeric(10) + var keyStorePassword = RandomStringUtils.insecure().nextAlphanumeric(10) if (sslPassword.isEmpty()) { val file = File(ENCRYPT_FILE_NAME) if (file.exists()) { diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle index dae007abfb24..5d6b3818d492 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -2,7 +2,7 @@ compileTestFixturesKotlin.compilerOptions.allWarningsAsErrors = false compileTestKotlin.compilerOptions.allWarningsAsErrors = false dependencies { - api 'org.apache.commons:commons-csv:1.10.0' + api 'org.apache.commons:commons-csv:1.11.0' implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core') @@ -10,8 +10,8 @@ dependencies { implementation 'io.aesy:datasize:1.0.0' - testFixturesCompileOnly 'org.projectlombok:lombok:1.18.30' - testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.30' + testFixturesCompileOnly 'org.projectlombok:lombok:1.18.34' + testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.34' testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies')) testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core') @@ -20,6 +20,6 @@ dependencies { testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-typing-deduping')) testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-typing-deduping') - testImplementation 'org.mockito.kotlin:mockito-kotlin:5.2.1' + testImplementation 'org.mockito.kotlin:mockito-kotlin:5.4.0' } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.kt index 37b2212bd0a5..310a0da43b40 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.kt @@ -17,17 +17,7 @@ import java.time.Instant import java.util.Locale import java.util.Optional import kotlin.Int -import org.jooq.Condition -import org.jooq.CreateTableColumnStep -import org.jooq.DSLContext -import org.jooq.DataType -import org.jooq.Field -import org.jooq.InsertValuesStepN -import org.jooq.Name -import org.jooq.Record -import org.jooq.SQLDialect -import org.jooq.SelectConditionStep -import org.jooq.SelectFieldOrAsterisk +import org.jooq.* import org.jooq.conf.ParamType import org.jooq.impl.DSL import org.jooq.impl.SQLDataType @@ -331,7 +321,7 @@ constructor( ): String { val hasGenerationId = columns == DestinationColumns.V2_WITH_GENERATION - val createTable: CreateTableColumnStep = + val createTable: CreateTableElementListStep = dslContext .createTable(rawTableName) .column( diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle index 5d668111d8c3..10b6eb335c8a 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle @@ -34,7 +34,7 @@ dependencies { implementation 'io.debezium:debezium-api:2.4.0.Final' implementation 'io.debezium:debezium-embedded:2.4.0.Final' - implementation 'org.codehaus.plexus:plexus-utils:4.0.0' + implementation 'org.codehaus.plexus:plexus-utils:4.0.1' testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies')) @@ -42,13 +42,13 @@ dependencies { testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core')) testFixturesImplementation 'net.sourceforge.argparse4j:argparse4j:0.9.0' - testFixturesImplementation 'io.swagger:swagger-annotations:1.6.13' + testFixturesImplementation 'io.swagger:swagger-annotations:1.6.14' testFixturesImplementation 'org.hamcrest:hamcrest-all:1.3' - testFixturesImplementation 'org.junit.platform:junit-platform-launcher:1.10.1' + testFixturesImplementation 'org.junit.platform:junit-platform-launcher:1.11.0' testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-datastore-postgres') testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-datastore-postgres')) - testImplementation 'uk.org.webcompere:system-stubs-jupiter:2.0.1' - testImplementation 'org.mockito.kotlin:mockito-kotlin:5.2.1' + testImplementation 'uk.org.webcompere:system-stubs-jupiter:2.1.6' + testImplementation 'org.mockito.kotlin:mockito-kotlin:5.4.0' } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/jdbc/JdbcSSLConnectionUtils.kt b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/jdbc/JdbcSSLConnectionUtils.kt index e399be12b3f0..2a300aef1049 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/jdbc/JdbcSSLConnectionUtils.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/jdbc/JdbcSSLConnectionUtils.kt @@ -195,7 +195,7 @@ class JdbcSSLConnectionUtils { ) { sslModeConfig[PARAM_CLIENT_KEY_PASSWORD].asText() } else { - RandomStringUtils.randomAlphanumeric(10) + RandomStringUtils.insecure().nextAlphanumeric(10) } return clientKeyPassword } diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle b/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle index 428d19774c9f..176513321ce4 100644 --- a/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle @@ -10,48 +10,48 @@ compileTestKotlin.compilerOptions.allWarningsAsErrors = false def generate = tasks.register('generate') dependencies { - api platform('com.fasterxml.jackson:jackson-bom:2.15.2') + api platform('com.fasterxml.jackson:jackson-bom:2.17.2') api 'com.fasterxml.jackson.core:jackson-annotations' api 'com.fasterxml.jackson.core:jackson-databind' api 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' api 'com.fasterxml.jackson.module:jackson-module-kotlin' - api 'com.google.guava:guava:33.0.0-jre' - api 'commons-io:commons-io:2.15.1' + api 'com.google.guava:guava:33.3.0-jre' + api 'commons-io:commons-io:2.16.1' api ('io.airbyte.airbyte-protocol:protocol-models:0.11.0') { exclude group: 'com.google.api-client', module: 'google-api-client' } api 'javax.annotation:javax.annotation-api:1.3.2' - api 'org.apache.commons:commons-compress:1.25.0' - api 'org.apache.commons:commons-lang3:3.14.0' - api 'org.apache.logging.log4j:log4j-api:2.21.1' - api 'org.apache.logging.log4j:log4j-core:2.21.1' - api 'org.apache.logging.log4j:log4j-slf4j-impl:2.21.1' - api 'org.apache.logging.log4j:log4j-slf4j2-impl:2.21.1' - api 'org.slf4j:log4j-over-slf4j:2.0.11' - api 'org.slf4j:slf4j-api:2.0.11' - api 'io.github.oshai:kotlin-logging-jvm:6.0.9' - api 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0' - - - implementation 'com.jayway.jsonpath:json-path:2.7.0' - implementation 'com.networknt:json-schema-validator:1.0.72' - implementation 'commons-cli:commons-cli:1.4' - implementation 'io.swagger:swagger-annotations:1.6.2' + api 'org.apache.commons:commons-compress:1.27.1' + api 'org.apache.commons:commons-lang3:3.17.0' + api 'org.apache.logging.log4j:log4j-api:2.24.0' + api 'org.apache.logging.log4j:log4j-core:2.24.0' + api 'org.apache.logging.log4j:log4j-slf4j-impl:2.24.0' + api 'org.apache.logging.log4j:log4j-slf4j2-impl:2.24.0' + api 'org.slf4j:log4j-over-slf4j:2.0.16' + api 'org.slf4j:slf4j-api:2.0.16' + api 'io.github.oshai:kotlin-logging-jvm:7.0.0' + api 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.1' + + + implementation 'com.jayway.jsonpath:json-path:2.9.0' + implementation 'com.networknt:json-schema-validator:1.5.1' + implementation 'commons-cli:commons-cli:1.9.0' + implementation 'io.swagger:swagger-annotations:1.6.14' implementation 'javax.validation:validation-api:2.0.1.Final' implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' // needed so that we can follow $ref when parsing json - implementation 'org.openapitools:jackson-databind-nullable:0.2.1' - implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0' + implementation 'org.openapitools:jackson-databind-nullable:0.2.6' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.1' testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core')) testFixturesApi ('io.airbyte:airbyte-api:0.55.2') { transitive = false } testFixturesApi 'org.jetbrains.kotlin:kotlin-test' - testFixturesImplementation 'io.swagger:swagger-annotations:1.6.2' - testFixturesImplementation 'org.apache.ant:ant:1.10.11' + testFixturesImplementation 'io.swagger:swagger-annotations:1.6.14' + testFixturesImplementation 'org.apache.ant:ant:1.10.15' - testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' - testImplementation 'org.mockito.kotlin:mockito-kotlin:5.2.1' + testImplementation 'com.squareup.okhttp3:mockwebserver:4.12.0' + testImplementation 'org.mockito.kotlin:mockito-kotlin:5.4.0' } diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/commons/string/Strings.kt b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/commons/string/Strings.kt index b5f53c462570..c80dd9593a6f 100644 --- a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/commons/string/Strings.kt +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/commons/string/Strings.kt @@ -16,7 +16,9 @@ object Strings { fun addRandomSuffix(base: String, separator: String, suffixLength: Int): String { return base + separator + - RandomStringUtils.randomAlphabetic(suffixLength).lowercase(Locale.getDefault()) + RandomStringUtils.insecure() + .nextAlphanumeric(suffixLength) + .lowercase(Locale.getDefault()) } @JvmStatic diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/validation/json/JsonSchemaValidator.kt b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/validation/json/JsonSchemaValidator.kt index 24fc828cc4b0..9c91d6aa80eb 100644 --- a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/validation/json/JsonSchemaValidator.kt +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/validation/json/JsonSchemaValidator.kt @@ -10,20 +10,20 @@ import com.networknt.schema.* import io.github.oshai.kotlinlogging.KotlinLogging import java.io.File import java.io.IOException -import java.net.URI -import java.net.URISyntaxException import me.andrz.jackson.JsonContext import me.andrz.jackson.JsonReferenceException import me.andrz.jackson.JsonReferenceProcessor private val LOGGER = KotlinLogging.logger {} -class JsonSchemaValidator @VisibleForTesting constructor(private val baseUri: URI?) { - private val jsonSchemaFactory: JsonSchemaFactory = +class JsonSchemaValidator +@VisibleForTesting +constructor(private val schemaLocation: SchemaLocation) { + public val jsonSchemaFactory: JsonSchemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7) private val schemaToValidators: MutableMap = HashMap() - constructor() : this(DEFAULT_BASE_URI) + constructor() : this(DEFAULT_BASE_SCHEMA_LOCATION) /** * Create and cache a schema validator for a particular schema. This validator is used when @@ -92,14 +92,16 @@ class JsonSchemaValidator @VisibleForTesting constructor(private val baseUri: UR .toSet() } - fun getValidationMessageArgs(schemaJson: JsonNode, objectJson: JsonNode): List> { + fun getValidationMessageArgs(schemaJson: JsonNode, objectJson: JsonNode): List> { return validateInternal(schemaJson, objectJson).map { obj: ValidationMessage -> obj.arguments } } - fun getValidationMessagePaths(schemaJson: JsonNode, objectJson: JsonNode): List { - return validateInternal(schemaJson, objectJson).map { obj: ValidationMessage -> obj.path } + fun getValidationMessagePaths(schemaJson: JsonNode, objectJson: JsonNode): List { + return validateInternal(schemaJson, objectJson).map { obj: ValidationMessage -> + obj.evaluationPath + } } @Throws(JsonValidationException::class) @@ -163,15 +165,16 @@ class JsonSchemaValidator @VisibleForTesting constructor(private val baseUri: UR } } - val context = - ValidationContext( - jsonSchemaFactory.uriFactory, - null, - metaschema, - jsonSchemaFactory, + val context = ValidationContext(metaschema, jsonSchemaFactory, null) + + val schema = + jsonSchemaFactory.create( + context, + schemaLocation, + JsonNodePath(context.config.pathType), + schemaJson, null ) - val schema = JsonSchema(context, baseUri, schemaJson) return schema } @@ -180,15 +183,8 @@ class JsonSchemaValidator @VisibleForTesting constructor(private val baseUri: UR // This URI just needs to point at any path in the same directory as // /app/WellKnownTypes.json // It's required for the JsonSchema#validate method to resolve $ref correctly. - private var DEFAULT_BASE_URI: URI? = null - - init { - try { - DEFAULT_BASE_URI = URI("file:///app/nonexistent_file.json") - } catch (e: URISyntaxException) { - throw RuntimeException(e) - } - } + private var DEFAULT_BASE_SCHEMA_LOCATION: SchemaLocation = + SchemaLocation.of("file:///app/nonexistent_file.json") /** * Get JsonNode for an object defined as the main object in a JsonSchema file. Able to diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/kotlin/io/airbyte/commons/json/JsonsTest.kt b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/kotlin/io/airbyte/commons/json/JsonsTest.kt index add78013093b..8d728ec0afdc 100644 --- a/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/kotlin/io/airbyte/commons/json/JsonsTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/kotlin/io/airbyte/commons/json/JsonsTest.kt @@ -236,7 +236,7 @@ internal class JsonsTest { fun testToPrettyString() { val jsonNode = Jsons.jsonNode(ImmutableMap.of(TEST, ABC)) val expectedOutput = """{ - "test": "abc" + "test" : "abc" } """ Assertions.assertEquals(expectedOutput, Jsons.toPrettyString(jsonNode)) diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/kotlin/io/airbyte/validation/json/JsonSchemaValidatorTest.kt b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/kotlin/io/airbyte/validation/json/JsonSchemaValidatorTest.kt index 8ddb8b6a6a23..618b34b2deed 100644 --- a/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/kotlin/io/airbyte/validation/json/JsonSchemaValidatorTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/kotlin/io/airbyte/validation/json/JsonSchemaValidatorTest.kt @@ -4,10 +4,10 @@ package io.airbyte.validation.json import com.fasterxml.jackson.databind.JsonNode +import com.networknt.schema.SchemaLocation import io.airbyte.commons.io.IOs import io.airbyte.commons.json.Jsons import java.io.IOException -import java.net.URI import java.net.URISyntaxException import java.nio.file.Files import org.junit.jupiter.api.Assertions @@ -110,7 +110,9 @@ internal class JsonSchemaValidatorTest { ) .toFile() val jsonSchemaValidator = - JsonSchemaValidator(URI("file://" + schemaFile.parentFile.absolutePath + "/foo.json")) + JsonSchemaValidator( + SchemaLocation.of("file://" + schemaFile.parentFile.absolutePath + "/foo.json") + ) val validationResult = jsonSchemaValidator.validate( diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle index 3005bcbef169..b40ea5e63b36 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle @@ -9,7 +9,7 @@ dependencies { implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core') api project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-s3-destinations') - api 'com.google.cloud:google-cloud-storage:2.32.1' + api 'com.google.cloud:google-cloud-storage:2.42.0' testFixturesApi testFixtures(project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core')) testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-s3-destinations') diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.kt b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.kt index a65b28c6147f..d19bbf484eba 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.kt +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.kt @@ -62,7 +62,9 @@ constructor( false, true ) - LOGGER.info { "Avro schema for stream ${stream.name}: ${schema!!.toString(false)}" } + LOGGER.info { + "Avro schema for stream ${stream.name}: ${@Suppress("DEPRECATION")schema!!.toString(false)}" + } val outputFilename: String = getOutputFilename(uploadTimestamp, FileUploadFormat.AVRO) outputPath = java.lang.String.join("/", outputPrefix, outputFilename) diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.kt b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.kt index 8c2ad5ae911c..d04fc2e6a4fb 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.kt +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.kt @@ -15,7 +15,7 @@ class GcsS3FileSystem : S3AFileSystem() { */ @Retries.RetryTranslated @Throws(IOException::class) - override fun verifyBucketExistsV2() { + /*override*/ fun verifyBucketExistsV2() { super.verifyBucketExists() } } diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index 823172c7d6b8..f6bdeb53af69 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -20,23 +20,23 @@ dependencies { implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-db-destinations') // Re-export dependencies for gcs-destinations. - api 'com.amazonaws:aws-java-sdk-s3:1.12.647' - api 'com.amazonaws:aws-java-sdk-sts:1.12.647' + api 'com.amazonaws:aws-java-sdk-s3:1.12.772' + api 'com.amazonaws:aws-java-sdk-sts:1.12.772' api ('com.github.airbytehq:json-avro-converter:1.1.3') { exclude group: 'ch.qos.logback', module: 'logback-classic'} api 'com.github.alexmojaki:s3-stream-upload:2.2.4' - api 'org.apache.avro:avro:1.11.3' - api 'org.apache.commons:commons-csv:1.10.0' - api 'org.apache.commons:commons-text:1.11.0' - api ('org.apache.hadoop:hadoop-aws:3.3.6') { exclude group: 'com.amazonaws', module: 'aws-java-sdk-bundle' } - api ('org.apache.hadoop:hadoop-common:3.3.6') { + api 'org.apache.avro:avro:1.12.0' + api 'org.apache.commons:commons-csv:1.11.0' + api 'org.apache.commons:commons-text:1.12.0' + api ('org.apache.hadoop:hadoop-aws:3.4.0') { exclude group: 'com.amazonaws', module: 'aws-java-sdk-bundle' } + api ('org.apache.hadoop:hadoop-common:3.4.0') { exclude group: 'org.apache.zookeeper' exclude group: 'org.apache.hadoop', module: 'hadoop-yarn-common' } - api ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6') { + api ('org.apache.hadoop:hadoop-mapreduce-client-core:3.4.0') { exclude group: 'org.apache.zookeeper' exclude group: 'org.apache.hadoop', module: 'hadoop-yarn-common' } - api 'org.apache.parquet:parquet-avro:1.13.1' + api 'org.apache.parquet:parquet-avro:1.14.2' runtimeOnly 'com.hadoop.gplcompression:hadoop-lzo:0.4.20' testFixturesApi project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/writer/ProductionWriterFactory.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/writer/ProductionWriterFactory.kt index 60aa8c2f6b8b..e5d0cca4b332 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/writer/ProductionWriterFactory.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/writer/ProductionWriterFactory.kt @@ -36,7 +36,9 @@ class ProductionWriterFactory : S3WriterFactory { val avroSchema = schemaConverter.getAvroSchema(stream.jsonSchema, stream.name, stream.namespace) - LOGGER.info { "Avro schema for stream ${stream.name}: ${avroSchema.toString(false)}" } + LOGGER.info { + "Avro schema for stream ${stream.name}: ${@Suppress("DEPRECATION")avroSchema.toString(false)}" + } return if (format == FileUploadFormat.AVRO) { S3AvroWriter( diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.kt index a387caf4a7fb..2a22da6791db 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.kt @@ -151,7 +151,7 @@ protected constructor( String.format( "%s_test_%s", outputFormat.name.lowercase(), - RandomStringUtils.randomAlphanumeric(5), + RandomStringUtils.insecure().nextAlphanumeric(5), ) (configJson as ObjectNode) .put("s3_bucket_path", testBucketPath)