From b3839e79856a6bdedc012d1f749c6dfa91252d0f Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Sat, 15 Jun 2024 03:33:01 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9BDestination=20Iceberg:=20Bump=20Ice?= =?UTF-8?q?berg=20to=201.5.2=20and=20Spark=20to=203.5.1=20(#38283)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marcos Marx --- .../destination-iceberg/build.gradle | 30 +++++++++++-------- .../destination-iceberg/metadata.yaml | 2 +- docs/integrations/destinations/iceberg.md | 3 +- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/destination-iceberg/build.gradle b/airbyte-integrations/connectors/destination-iceberg/build.gradle index a8672abce6e6..628a123b7aed 100644 --- a/airbyte-integrations/connectors/destination-iceberg/build.gradle +++ b/airbyte-integrations/connectors/destination-iceberg/build.gradle @@ -24,24 +24,24 @@ application { dependencies { - implementation('org.apache.spark:spark-sql_2.13:3.3.2') { + implementation('org.apache.spark:spark-sql_2.13:3.5.1') { exclude(group: 'org.apache.hadoop', module: 'hadoop-common') } - implementation('org.apache.spark:spark-hive_2.13:3.3.2') { + implementation('org.apache.spark:spark-hive_2.13:3.5.1') { exclude(group: 'org.apache.hadoop', module: 'hadoop-common') } - implementation 'org.apache.iceberg:iceberg-spark-runtime-3.3_2.13:1.3.0' + implementation 'org.apache.iceberg:iceberg-spark-runtime-3.5_2.13:1.5.2' // force awssdk version required by Iceberg - implementation "software.amazon.awssdk:utils:2.20.18" - implementation "software.amazon.awssdk:url-connection-client:2.20.18" - implementation "software.amazon.awssdk:s3:2.20.18" - implementation "software.amazon.awssdk:glue:2.20.18" - implementation "software.amazon.awssdk:dynamodb:2.20.18" - implementation "software.amazon.awssdk:kms:2.20.18" - implementation "software.amazon.awssdk:sts:2.20.18" - implementation "software.amazon.awssdk:sdk-core:2.20.18" - implementation "software.amazon.awssdk:aws-core:2.20.18" + implementation "software.amazon.awssdk:utils:2.24.5" + implementation "software.amazon.awssdk:url-connection-client:2.24.5" + implementation "software.amazon.awssdk:s3:2.24.5" + implementation "software.amazon.awssdk:glue:2.24.5" + implementation "software.amazon.awssdk:dynamodb:2.24.5" + implementation "software.amazon.awssdk:kms:2.24.5" + implementation "software.amazon.awssdk:sts:2.24.5" + implementation "software.amazon.awssdk:sdk-core:2.24.5" + implementation "software.amazon.awssdk:aws-core:2.24.5" implementation "org.apache.hadoop:hadoop-aws:3.3.2" implementation "org.apache.hadoop:hadoop-client-api:3.3.2" @@ -49,6 +49,12 @@ dependencies { implementation "org.postgresql:postgresql:42.5.0" implementation "commons-collections:commons-collections:3.2.2" + implementation('org.antlr:antlr4-runtime') { + version { + strictly('4.9.3') + } + } + testImplementation libs.testcontainers.postgresql integrationTestJavaImplementation libs.testcontainers.postgresql diff --git a/airbyte-integrations/connectors/destination-iceberg/metadata.yaml b/airbyte-integrations/connectors/destination-iceberg/metadata.yaml index c222f75b02c1..43ef3592f01f 100644 --- a/airbyte-integrations/connectors/destination-iceberg/metadata.yaml +++ b/airbyte-integrations/connectors/destination-iceberg/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: df65a8f3-9908-451b-aa9b-445462803560 - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 dockerRepository: airbyte/destination-iceberg githubIssueLabel: destination-iceberg license: MIT diff --git a/docs/integrations/destinations/iceberg.md b/docs/integrations/destinations/iceberg.md index b9dab9c4a04b..e8dbb6afe746 100644 --- a/docs/integrations/destinations/iceberg.md +++ b/docs/integrations/destinations/iceberg.md @@ -63,7 +63,8 @@ specify the target size of compacted Iceberg data file. Expand to review | Version | Date | Pull Request | Subject | -| :------ | :--------- | :-------------------------------------------------------- | :--------------------------------------------------------- | +|:--------|:-----------| :-------------------------------------------------------- |:-----------------------------------------------------------| +| 0.1.7 | 2024-05-17 | [38283](https://github.com/airbytehq/airbyte/pull/38283) | Bump Iceberg library to 1.5.2 and Spark to 3.5.1 | | 0.1.6 | 2024-04-04 | [#36846](https://github.com/airbytehq/airbyte/pull/36846) | Remove duplicate S3 Region | | 0.1.5 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | | 0.1.4 | 2023-07-20 | [28506](https://github.com/airbytehq/airbyte/pull/28506) | Support server-managed storage config |