Skip to content

Commit

Permalink
lint
Browse files Browse the repository at this point in the history
  • Loading branch information
the-other-tim-brown committed Aug 2, 2024
1 parent a1f0bcd commit 437cc15
Show file tree
Hide file tree
Showing 13 changed files with 31 additions and 48 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,7 @@ public class ConversionConfig {

@Builder
ConversionConfig(
@NonNull SourceTable sourceTable,
List<TargetTable> targetTables,
SyncMode syncMode) {
@NonNull SourceTable sourceTable, List<TargetTable> targetTables, SyncMode syncMode) {
this.sourceTable = sourceTable;
this.targetTables = targetTables;
Preconditions.checkArgument(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package org.apache.xtable.conversion;

import java.util.Properties;

import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NonNull;
Expand All @@ -26,38 +28,22 @@

import com.google.common.base.Preconditions;

import java.util.Properties;

/**
* Defines a reference to a table in a particular format.
*/
/** Defines a reference to a table in a particular format. */
@Getter
@EqualsAndHashCode
class ExternalTable {
/**
* The name of the table.
*/
/** The name of the table. */
protected final @NonNull String name;
/**
* The format of the table (e.g. DELTA, ICEBERG, HUDI)
*/
/** The format of the table (e.g. DELTA, ICEBERG, HUDI) */
protected final @NonNull String formatName;
/**
* The path to the root of the table or the metadata directory depending on the format
*/
/** The path to the root of the table or the metadata directory depending on the format */
protected final @NonNull String basePath;
/**
* Optional namespace for the table
*/
/** Optional namespace for the table */
protected final String[] namespace;
/**
* The configuration for interacting with the catalog that manages this table
*/
/** The configuration for interacting with the catalog that manages this table */
protected final CatalogConfig catalogConfig;

/**
* Optional, additional properties that can be used to define interactions with the table
*/
/** Optional, additional properties that can be used to define interactions with the table */
protected final Properties additionalProperties;

ExternalTable(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,17 @@

package org.apache.xtable.conversion;

import java.util.Properties;

import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NonNull;

import java.util.Properties;

@EqualsAndHashCode(callSuper = true)
@Getter
public class SourceTable extends ExternalTable {
/**
* The path to the data files, defaults to the metadataPath
*/
/** The path to the data files, defaults to the metadataPath */
@NonNull private final String dataPath;

@Builder(toBuilder = true)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ void sanitizePath() {
@Test
void errorIfRequiredArgsNotSet() {
assertThrows(
NullPointerException.class, () -> new ExternalTable("name", "hudi", null, null, null, null));
NullPointerException.class,
() -> new ExternalTable("name", "hudi", null, null, null, null));

assertThrows(
NullPointerException.class,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@

package org.apache.xtable.delta;

import java.util.Map;

import org.apache.spark.sql.SparkSession;

import io.delta.tables.DeltaTable;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@
public class HudiConversionSourceProvider extends ConversionSourceProvider<HoodieInstant> {

@Override
public HudiConversionSource getConversionSourceInstance(
SourceTable sourceTable) {
public HudiConversionSource getConversionSourceInstance(SourceTable sourceTable) {
HoodieTableMetaClient metaClient =
HoodieTableMetaClient.builder()
.setConf(hadoopConf)
Expand All @@ -45,7 +44,8 @@ public HudiConversionSource getConversionSourceInstance(
}

final HudiSourcePartitionSpecExtractor sourcePartitionSpecExtractor =
HudiSourceConfig.fromProperties(sourceTable.getAdditionalProperties()).loadSourcePartitionSpecExtractor();
HudiSourceConfig.fromProperties(sourceTable.getAdditionalProperties())
.loadSourcePartitionSpecExtractor();

return new HudiConversionSource(metaClient, sourcePartitionSpecExtractor);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,7 @@ public HudiConversionTarget() {}
(int) targetTable.getMetadataRetention().toHours(),
maxNumDeltaCommitsBeforeCompaction,
BaseFileUpdatesExtractor.of(
new HoodieJavaEngineContext(configuration),
new CachingPath(targetTable.getBasePath())),
new HoodieJavaEngineContext(configuration), new CachingPath(targetTable.getBasePath())),
AvroSchemaConverter.getInstance(),
HudiTableManager.of(configuration),
CommitState::new);
Expand Down Expand Up @@ -169,8 +168,7 @@ public void init(TargetTable targetTable, Configuration configuration) {
(int) targetTable.getMetadataRetention().toHours(),
HoodieMetadataConfig.COMPACT_NUM_DELTA_COMMITS.defaultValue(),
BaseFileUpdatesExtractor.of(
new HoodieJavaEngineContext(configuration),
new CachingPath(targetTable.getBasePath())),
new HoodieJavaEngineContext(configuration), new CachingPath(targetTable.getBasePath())),
AvroSchemaConverter.getInstance(),
HudiTableManager.of(configuration),
CommitState::new);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;

import lombok.Value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.xtable.conversion.ConversionConfig;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
Expand All @@ -85,6 +84,7 @@

import com.google.common.collect.ImmutableList;

import org.apache.xtable.conversion.ConversionConfig;
import org.apache.xtable.conversion.ConversionController;
import org.apache.xtable.conversion.ConversionSourceProvider;
import org.apache.xtable.conversion.SourceTable;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -415,8 +415,7 @@ public void testsShowingVacuumHasNoEffectOnIncrementalSync() {
InstantsForIncrementalSync.builder()
.lastSyncInstant(Instant.ofEpochMilli(timestamp1))
.build();
conversionSource =
conversionSourceProvider.getConversionSourceInstance(tableConfig);
conversionSource = conversionSourceProvider.getConversionSourceInstance(tableConfig);
CommitsBacklog<Long> instantCurrentCommitState =
conversionSource.getCommitsBacklog(instantsForIncrementalSync);
boolean areFilesRemoved = false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@

import org.apache.xtable.GenericTable;
import org.apache.xtable.TestJavaHudiTable;
import org.apache.xtable.conversion.ConversionConfig;
import org.apache.xtable.conversion.ConversionController;
import org.apache.xtable.conversion.ConversionSourceProvider;
import org.apache.xtable.conversion.SourceTable;
import org.apache.xtable.conversion.ConversionConfig;
import org.apache.xtable.conversion.TargetTable;
import org.apache.xtable.hudi.HudiConversionSourceProvider;
import org.apache.xtable.model.storage.TableFormat;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,12 @@ public void syncHoodieTable() {
Properties sourceProperties = new Properties();
sourceProperties.put(PARTITION_FIELD_SPEC_CONFIG, getPartitionSpecConfig());
SourceTable sourceTable =
SourceTable.builder().name(tableName).formatName(HUDI).basePath(basePath).additionalProperties(sourceProperties).build();
SourceTable.builder()
.name(tableName)
.formatName(HUDI)
.basePath(basePath)
.additionalProperties(sourceProperties)
.build();
Duration metadataRetention =
config.contains(XTableSyncConfig.XTABLE_TARGET_METADATA_RETENTION_HOURS)
? Duration.ofHours(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ public static void main(String[] args) throws IOException {
tableFormatList);
Properties sourceProperties = new Properties();
if (table.getPartitionSpec() != null) {
sourceProperties.put(HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG, table.getPartitionSpec());
sourceProperties.put(
HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG, table.getPartitionSpec());
}
SourceTable sourceTable =
SourceTable.builder()
Expand Down

0 comments on commit 437cc15

Please sign in to comment.