From b19dbc0be3ba81bae4bb073d7c8b725444f0758d Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Sat, 29 Jul 2023 18:22:01 -0400 Subject: [PATCH] Bump hbase client to 2.x (#27746) --- CHANGES.md | 5 +++-- sdks/java/io/hbase/build.gradle | 14 ++------------ .../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 11 +++++++---- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a26de7dd9772..c6e756ee9646 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -63,7 +63,8 @@ * Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). * Python GCSIO is now implemented with GCP GCS Client instead of apitools ([#25676](https://github.com/apache/beam/issues/25676)) * Java KafkaIO now supports picking up topics via topicPattern ([#26948](https://github.com/apache/beam/pull/26948)) -* Support for read from Cosmos DB Core SQL API [#23604](https://github.com/apache/beam/issues/23604) +* Support for read from Cosmos DB Core SQL API ([#23604](https://github.com/apache/beam/issues/23604)) +* Upgraded to HBase 2.5.5 for HBaseIO. (Java) ([#27711](https://github.com/apache/beam/issues/19554)) ## New Features / Improvements @@ -89,7 +90,7 @@ * ([#X](https://github.com/apache/beam/issues/X)). -# [2.49.0] - Unreleased +# [2.49.0] - 2023-07-17 ## I/Os diff --git a/sdks/java/io/hbase/build.gradle b/sdks/java/io/hbase/build.gradle index 8224c804867e..8debaeaba00a 100644 --- a/sdks/java/io/hbase/build.gradle +++ b/sdks/java/io/hbase/build.gradle @@ -34,7 +34,7 @@ test { jvmArgs "-Dtest.build.data.basedirectory=build/test-data" } -def hbase_version = "1.2.6" +def hbase_version = "2.5.5" dependencies { implementation library.java.vendored_guava_26_0_jre @@ -49,17 +49,7 @@ dependencies { testImplementation library.java.hadoop_minicluster testImplementation library.java.hadoop_hdfs testImplementation library.java.hadoop_common - testImplementation "org.apache.hbase:hbase-shaded-server:$hbase_version" - testImplementation("org.apache.hbase:hbase-server:$hbase_version:tests") { - // We prevent bringing in unshaded dependencies to not conflict - // with hbase-shaded-server and hbase-shaded-client - transitive = false - } - testImplementation("org.apache.hbase:hbase-common:$hbase_version:tests") { - // We prevent bringing in unshaded dependencies to not conflict - // with hbase-shaded-server and hbase-shaded-client - transitive = false - } + testImplementation("org.apache.hbase:hbase-shaded-testing-util:$hbase_version") testImplementation "org.apache.hbase:hbase-hadoop-compat:$hbase_version:tests" testImplementation "org.apache.hbase:hbase-hadoop2-compat:$hbase_version:tests" testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java index 6b096492437a..b4e0de58cf37 100644 --- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java +++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java @@ -51,11 +51,12 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.StartMiniClusterOption; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -85,7 +86,7 @@ public class HBaseIOTest { @Rule public TemporaryHBaseTable tmpTable = new TemporaryHBaseTable(); private static HBaseTestingUtility htu; - private static HBaseAdmin admin; + private static Admin admin; private static final Configuration conf = HBaseConfiguration.create(); private static final byte[] COLUMN_FAMILY = Bytes.toBytes("info"); @@ -103,10 +104,12 @@ public static void beforeClass() throws Exception { // We don't use the full htu.startMiniCluster() to avoid starting unneeded HDFS/MR daemons htu.startMiniZKCluster(); - MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 4); + StartMiniClusterOption option = + StartMiniClusterOption.builder().numMasters(1).numRegionServers(4).build(); + MiniHBaseCluster hbm = htu.startMiniHBaseCluster(option); hbm.waitForActiveAndReadyMaster(); - admin = htu.getHBaseAdmin(); + admin = htu.getAdmin(); } @AfterClass