Skip to content

Commit

Permalink
[apache#1974] fix(hive): Hive table listPartitions reduces HMS RPC ca…
Browse files Browse the repository at this point in the history
…lls (apache#1975)

### What changes were proposed in this pull request?
Use HMS's `listPartitions` RPC to replace the two RPC calls of
`listPartitionNames` and `getPartitionsByNames`.

### Why are the changes needed?

Fix: apache#1974

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
exist UT
  • Loading branch information
cxzl25 authored Feb 3, 2024
1 parent 7e20101 commit 39226df
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ private Map<String, String> buildTableParameters() {
return parameters;
}

private List<FieldSchema> buildPartitionKeys() {
public List<FieldSchema> buildPartitionKeys() {
return Arrays.stream(partitioning)
.map(p -> getPartitionKey(((Transforms.IdentityTransform) p).fieldName()))
.collect(Collectors.toList());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
Expand Down Expand Up @@ -58,30 +60,23 @@ public String[] listPartitionNames() {

@Override
public Partition[] listPartitions() {
List<String> partitionNames;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions;
try {
partitionNames =
table
.clientPool()
.run(c -> c.listPartitionNames(table.schemaName(), table.name(), (short) -1));
partitions =
table
.clientPool()
.run(c -> c.getPartitionsByNames(table.schemaName(), table.name(), partitionNames));
.run(c -> c.listPartitions(table.schemaName(), table.name(), (short) -1));
} catch (TException | InterruptedException e) {
throw new RuntimeException(e);
}

// should never happen
Preconditions.checkArgument(
partitionNames.size() == partitions.size(),
"oops?! partition names and partitions size are not equal: %s vs %s",
partitionNames.size(),
partitions.size());

return IntStream.range(0, partitionNames.size())
.mapToObj(i -> fromHivePartition(partitionNames.get(i), partitions.get(i)))
List<String> partCols =
table.buildPartitionKeys().stream().map(FieldSchema::getName).collect(Collectors.toList());

return partitions.stream()
.map(
partition ->
fromHivePartition(
FileUtils.makePartName(partCols, partition.getValues()), partition))
.toArray(Partition[]::new);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public class TestHiveTableOperations extends MiniHiveMetastoreService {
private static Partition existingPartition;

@BeforeAll
private static void setup() {
public static void setup() {
hiveCatalog = initHiveCatalog();
hiveSchema = initHiveSchema(hiveCatalog);
hiveTable = createPartitionedTable();
Expand Down

0 comments on commit 39226df

Please sign in to comment.