[FLINK-34990][cdc-connector][oracle] Oracle cdc support newly add tab…

…le (#3203) * [cdc-connector][oracle] Oracle cdc support newly add table * [cdc-connector][oracle] Fix code style * [cdc-connector][oracle] Address comment
apache · Jul 16, 2024 · 2d1eb0a · 2d1eb0a
1 parent 137dc1b
commit 2d1eb0a
Show file tree

Hide file tree

Showing 10 changed files with 1,078 additions and 67 deletions.
diff --git a/...-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/OracleSourceBuilder.java b/...-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/OracleSourceBuilder.java
@@ -243,6 +243,12 @@ public OracleSourceBuilder<T> skipSnapshotBackfill(boolean skipSnapshotBackfill)
         return this;
     }
 
+    /** Whether the {@link OracleIncrementalSource} should scan the newly added tables or not. */
+    public OracleSourceBuilder<T> scanNewlyAddedTableEnabled(boolean scanNewlyAddedTableEnabled) {
+        this.configFactory.scanNewlyAddedTableEnabled(scanNewlyAddedTableEnabled);
+        return this;
+    }
+
     /**
      * Build the {@link OracleIncrementalSource}.
      *

diff --git a/...rc/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfig.java b/...rc/main/java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfig.java
@@ -63,7 +63,8 @@ public OracleSourceConfig(
             int connectMaxRetries,
             int connectionPoolSize,
             String chunkKeyColumn,
-            boolean skipSnapshotBackfill) {
+            boolean skipSnapshotBackfill,
+            boolean scanNewlyAddedTableEnabled) {
         super(
                 startupOptions,
                 databaseList,
@@ -89,7 +90,7 @@ public OracleSourceConfig(
                 connectionPoolSize,
                 chunkKeyColumn,
                 skipSnapshotBackfill,
-                false);
+                scanNewlyAddedTableEnabled);
         this.url = url;
     }
 

diff --git a/.../java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactory.java b/.../java/org/apache/flink/cdc/connectors/oracle/source/config/OracleSourceConfigFactory.java
@@ -133,6 +133,7 @@ public OracleSourceConfig create(int subtaskId) {
                 connectMaxRetries,
                 connectionPoolSize,
                 chunkKeyColumn,
-                skipSnapshotBackfill);
+                skipSnapshotBackfill,
+                scanNewlyAddedTableEnabled);
     }
 }
diff --git a/...cle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/table/OracleTableSource.java b/...cle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/table/OracleTableSource.java
@@ -79,6 +79,7 @@ public class OracleTableSource implements ScanTableSource, SupportsReadingMetada
     private final String chunkKeyColumn;
     private final boolean closeIdleReaders;
     private final boolean skipSnapshotBackfill;
+    private final boolean scanNewlyAddedTableEnabled;
 
     // --------------------------------------------------------------------------------------------
     // Mutable attributes
@@ -113,7 +114,8 @@ public OracleTableSource(
             double distributionFactorLower,
             @Nullable String chunkKeyColumn,
             boolean closeIdleReaders,
-            boolean skipSnapshotBackfill) {
+            boolean skipSnapshotBackfill,
+            boolean scanNewlyAddedTableEnabled) {
         this.physicalSchema = physicalSchema;
         this.url = url;
         this.port = port;
@@ -139,6 +141,7 @@ public OracleTableSource(
         this.chunkKeyColumn = chunkKeyColumn;
         this.closeIdleReaders = closeIdleReaders;
         this.skipSnapshotBackfill = skipSnapshotBackfill;
+        this.scanNewlyAddedTableEnabled = scanNewlyAddedTableEnabled;
     }
 
     @Override
@@ -187,6 +190,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) {
                             .closeIdleReaders(closeIdleReaders)
                             .skipSnapshotBackfill(skipSnapshotBackfill)
                             .chunkKeyColumn(chunkKeyColumn)
+                            .scanNewlyAddedTableEnabled(scanNewlyAddedTableEnabled)
                             .build();
 
             return SourceProvider.of(oracleChangeEventSource);
@@ -252,7 +256,8 @@ public DynamicTableSource copy() {
                         distributionFactorLower,
                         chunkKeyColumn,
                         closeIdleReaders,
-                        skipSnapshotBackfill);
+                        skipSnapshotBackfill,
+                        scanNewlyAddedTableEnabled);
         source.metadataKeys = metadataKeys;
         source.producedDataType = producedDataType;
         return source;
@@ -291,7 +296,8 @@ public boolean equals(Object o) {
                 && Objects.equals(distributionFactorLower, that.distributionFactorLower)
                 && Objects.equals(chunkKeyColumn, that.chunkKeyColumn)
                 && Objects.equals(closeIdleReaders, that.closeIdleReaders)
-                && Objects.equals(skipSnapshotBackfill, that.skipSnapshotBackfill);
+                && Objects.equals(skipSnapshotBackfill, that.skipSnapshotBackfill)
+                && Objects.equals(scanNewlyAddedTableEnabled, that.scanNewlyAddedTableEnabled);
     }
 
     @Override
@@ -321,7 +327,8 @@ public int hashCode() {
                 distributionFactorLower,
                 chunkKeyColumn,
                 closeIdleReaders,
-                skipSnapshotBackfill);
+                skipSnapshotBackfill,
+                scanNewlyAddedTableEnabled);
     }
 
     @Override

diff --git a/.../src/main/java/org/apache/flink/cdc/connectors/oracle/table/OracleTableSourceFactory.java b/.../src/main/java/org/apache/flink/cdc/connectors/oracle/table/OracleTableSourceFactory.java
@@ -47,6 +47,7 @@
 import static org.apache.flink.cdc.connectors.base.options.SourceOptions.SCAN_INCREMENTAL_SNAPSHOT_BACKFILL_SKIP;
 import static org.apache.flink.cdc.connectors.base.options.SourceOptions.SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE;
 import static org.apache.flink.cdc.connectors.base.options.SourceOptions.SCAN_INCREMENTAL_SNAPSHOT_ENABLED;
+import static org.apache.flink.cdc.connectors.base.options.SourceOptions.SCAN_NEWLY_ADDED_TABLE_ENABLED;
 import static org.apache.flink.cdc.connectors.base.options.SourceOptions.SCAN_SNAPSHOT_FETCH_SIZE;
 import static org.apache.flink.cdc.connectors.base.options.SourceOptions.SCAN_STARTUP_MODE;
 import static org.apache.flink.cdc.connectors.base.options.SourceOptions.SPLIT_KEY_EVEN_DISTRIBUTION_FACTOR_LOWER_BOUND;
@@ -106,6 +107,7 @@ public DynamicTableSource createDynamicTableSource(Context context) {
 
         boolean closeIdlerReaders = config.get(SCAN_INCREMENTAL_CLOSE_IDLE_READER_ENABLED);
         boolean skipSnapshotBackfill = config.get(SCAN_INCREMENTAL_SNAPSHOT_BACKFILL_SKIP);
+        boolean scanNewlyAddedTableEnabled = config.get(SCAN_NEWLY_ADDED_TABLE_ENABLED);
 
         if (enableParallelRead) {
             validateIntegerOption(SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE, splitSize, 1);
@@ -142,7 +144,8 @@ public DynamicTableSource createDynamicTableSource(Context context) {
                 distributionFactorLower,
                 chunkKeyColumn,
                 closeIdlerReaders,
-                skipSnapshotBackfill);
+                skipSnapshotBackfill,
+                scanNewlyAddedTableEnabled);
     }
 
     @Override
@@ -180,6 +183,7 @@ public Set<ConfigOption<?>> optionalOptions() {
         options.add(SCAN_INCREMENTAL_SNAPSHOT_CHUNK_KEY_COLUMN);
         options.add(SCAN_INCREMENTAL_CLOSE_IDLE_READER_ENABLED);
         options.add(SCAN_INCREMENTAL_SNAPSHOT_BACKFILL_SKIP);
+        options.add(SCAN_NEWLY_ADDED_TABLE_ENABLED);
         return options;
     }