Improve WindowCheck memory usage. (#5197)

* Port changes from legacy. * Use iterator for modified row nanosecond retrieval. * Do not reread nanos for modified singleton entries. * test changes 1. * Apply suggestions from code review Co-authored-by: Ryan Caudy <[email protected]> * some changes * some more formatting * rest of review. * Correct two bugs in entry merging. * fix comment formatting * Update engine/table/src/main/java/io/deephaven/engine/util/WindowCheck.java Co-authored-by: Ryan Caudy <[email protected]> * Reduce seed number. * Optimize the getLong away for singletons. --------- Co-authored-by: Ryan Caudy <[email protected]>
deephaven · Feb 28, 2024 · ca4dd87 · ca4dd87
1 parent 38dda6b
commit ca4dd87
Show file tree

Hide file tree

Showing 8 changed files with 889 additions and 144 deletions.
diff --git a/engine/table/build.gradle b/engine/table/build.gradle
@@ -34,6 +34,7 @@ dependencies {
     implementation 'com.tdunning:t-digest:3.2'
     implementation 'com.squareup:javapoet:1.13.0'
     implementation 'io.github.classgraph:classgraph:4.8.165'
+    implementation 'it.unimi.dsi:fastutil:8.5.13'
 
     implementation project(':plugin')
     implementation depCommonsLang3

diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java
@@ -24,7 +24,12 @@
 import java.util.List;
 
 /**
- * This will filter a table for the most recent N nanoseconds (must be on a date time column).
+ * This will filter a table for the most recent N nanoseconds (must be on an {@link Instant} column).
+ *
+ * <p>
+ * Note, this filter rescans the source table. You should prefer to use {@link io.deephaven.engine.util.WindowCheck}
+ * instead.
+ * </p>
  */
 public class TimeSeriesFilter
         extends WhereFilterLivenessArtifactImpl

diff --git a/engine/table/src/main/java/io/deephaven/engine/util/WindowCheck.java b/engine/table/src/main/java/io/deephaven/engine/util/WindowCheck.java
diff --git a/engine/table/src/test/java/io/deephaven/engine/util/TestWindowCheck.java b/engine/table/src/test/java/io/deephaven/engine/util/TestWindowCheck.java
diff --git a/engine/test-utils/build.gradle b/engine/test-utils/build.gradle
@@ -18,7 +18,7 @@ dependencies {
     implementation depCommonsLang3
     implementation depTrove3
 
-    implementation 'it.unimi.dsi:fastutil:8.5.11'
+    implementation 'it.unimi.dsi:fastutil:8.5.13'
 
     Classpaths.inheritJUnitClassic(project, 'implementation')
     Classpaths.inheritJUnitPlatform(project, 'implementation')

diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/GenerateTableUpdates.java b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/GenerateTableUpdates.java
@@ -144,6 +144,15 @@ private void validateGroup(int... opts) {
 
     public static final SimulationProfile DEFAULT_PROFILE = new SimulationProfile();
 
+    public static final SimulationProfile NO_SHIFT_PROFILE =
+            new SimulationProfile() {
+                {
+                    SHIFT_10_PERCENT_KEY_SPACE = 0;
+                    SHIFT_10_PERCENT_POS_SPACE = 0;
+                    SHIFT_AGGRESSIVELY = 0;
+                }
+            };
+
     public static void generateShiftAwareTableUpdates(final SimulationProfile profile, final int targetUpdateSize,
             final Random random, final QueryTable table,
             final ColumnInfo<?, ?>[] columnInfo) {

diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/QueryTableTestBase.java b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/QueryTableTestBase.java
@@ -23,21 +23,13 @@
 public abstract class QueryTableTestBase extends RefreshingTableTestCase {
     public final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
 
-    private static final GenerateTableUpdates.SimulationProfile NO_SHIFT_PROFILE =
-            new GenerateTableUpdates.SimulationProfile() {
-                {
-                    SHIFT_10_PERCENT_KEY_SPACE = 0;
-                    SHIFT_10_PERCENT_POS_SPACE = 0;
-                    SHIFT_AGGRESSIVELY = 0;
-                }
-            };
-
     public final JoinIncrement leftStep = new JoinIncrement() {
         @Override
         public void step(int leftSize, int rightSize, QueryTable leftTable, QueryTable rightTable,
                 ColumnInfo<?, ?>[] leftColumnInfo, ColumnInfo<?, ?>[] rightColumnInfo, EvalNuggetInterface[] en,
                 Random random) {
-            simulateShiftAwareStep(NO_SHIFT_PROFILE, toString(), leftSize, random, leftTable, leftColumnInfo, en);
+            simulateShiftAwareStep(GenerateTableUpdates.NO_SHIFT_PROFILE, toString(), leftSize, random, leftTable,
+                    leftColumnInfo, en);
         }
 
         @Override
@@ -63,7 +55,8 @@ public String toString() {
         public void step(int leftSize, int rightSize, QueryTable leftTable, QueryTable rightTable,
                 ColumnInfo<?, ?>[] leftColumnInfo, ColumnInfo<?, ?>[] rightColumnInfo, EvalNuggetInterface[] en,
                 Random random) {
-            simulateShiftAwareStep(NO_SHIFT_PROFILE, toString(), rightSize, random, rightTable, rightColumnInfo, en);
+            simulateShiftAwareStep(GenerateTableUpdates.NO_SHIFT_PROFILE, toString(), rightSize, random, rightTable,
+                    rightColumnInfo, en);
         }
 
         @Override
@@ -89,8 +82,10 @@ public String toString() {
         public void step(int leftSize, int rightSize, QueryTable leftTable, QueryTable rightTable,
                 ColumnInfo<?, ?>[] leftColumnInfo, ColumnInfo<?, ?>[] rightColumnInfo, EvalNuggetInterface[] en,
                 Random random) {
-            simulateShiftAwareStep(NO_SHIFT_PROFILE, toString(), leftSize, random, leftTable, leftColumnInfo, en);
-            simulateShiftAwareStep(NO_SHIFT_PROFILE, toString(), rightSize, random, rightTable, rightColumnInfo, en);
+            simulateShiftAwareStep(GenerateTableUpdates.NO_SHIFT_PROFILE, toString(), leftSize, random, leftTable,
+                    leftColumnInfo, en);
+            simulateShiftAwareStep(GenerateTableUpdates.NO_SHIFT_PROFILE, toString(), rightSize, random, rightTable,
+                    rightColumnInfo, en);
         }
 
         @Override

diff --git a/...st-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java b/...st-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java
@@ -160,7 +160,7 @@ public static void simulateShiftAwareStep(final String ctxt, int targetUpdateSiz
                 en);
     }
 
-    protected static void simulateShiftAwareStep(final GenerateTableUpdates.SimulationProfile simulationProfile,
+    public static void simulateShiftAwareStep(final GenerateTableUpdates.SimulationProfile simulationProfile,
             final String ctxt, int targetUpdateSize, Random random, QueryTable table, ColumnInfo[] columnInfo,
             EvalNuggetInterface[] en) {
         final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast();