Merge remote-tracking branch 'upstream/main' into release/v0.25.1

.github/workflows/check-ci.yml

@@ -51,19 +51,14 @@ jobs:
           name: check-ci-results
           path: |
             **/build/test-results/**
-            !**/build/test-results/**/binary/**
-
-      - name: Upload Test Reports
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: check-ci-reports
-          path: '**/build/reports/tests/**'
+            **/build/reports/tests/**
 
       - name: Upload JVM Error Logs
         uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: check-ci-jvm-err
-          path: '**/*_pid*.log'
+          path: |
+            **/*_pid*.log
+            **/core.*
           if-no-files-found: ignore

.github/workflows/nightly-check-ci.yml

@@ -68,7 +68,6 @@ jobs:
           path: |
             **/build/test-results/**
             **/build/reports/tests/**
-            !**/build/test-results/**/binary/**
 
       - name: Upload JVM Error Logs
         uses: actions/upload-artifact@v3

authorization-codegen/protoc-gen-contextual-auth-wiring

@@ -1,2 +1,2 @@
 # protoc-gen-contextual-auth-wiring
-java -cp authorization-codegen/build/libs/deephaven-authorization-codegen-0.25.0-all.jar io.deephaven.auth.codegen.GenerateContextualAuthWiring
+java -cp authorization-codegen/build/libs/deephaven-authorization-codegen-0.26.0-all.jar io.deephaven.auth.codegen.GenerateContextualAuthWiring

authorization-codegen/protoc-gen-service-auth-wiring

@@ -1,2 +1,2 @@
 # protoc-gen-service-auth-wiring
-java -cp authorization-codegen/build/libs/deephaven-authorization-codegen-0.25.0-all.jar io.deephaven.auth.codegen.GenerateServiceAuthWiring
+java -cp authorization-codegen/build/libs/deephaven-authorization-codegen-0.26.0-all.jar io.deephaven.auth.codegen.GenerateServiceAuthWiring

buildSrc/src/main/groovy/io.deephaven.common-conventions.gradle

@@ -5,7 +5,7 @@ plugins {
 }
 
 group = 'io.deephaven'
-version = '0.25.0'
+version = '0.26.0'
 
 if (!name.startsWith('deephaven-')) {
     archivesBaseName = "deephaven-${name}"

engine/table/src/main/java/io/deephaven/engine/table/impl/perf/UpdatePerformanceTracker.java

@@ -4,12 +4,18 @@
 package io.deephaven.engine.table.impl.perf;
 
 import io.deephaven.configuration.Configuration;
-import io.deephaven.engine.table.*;
+import io.deephaven.engine.context.ExecutionContext;
+import io.deephaven.engine.table.ShiftObliviousListener;
+import io.deephaven.engine.table.Table;
+import io.deephaven.engine.table.TableUpdateListener;
+import io.deephaven.engine.table.impl.InstrumentedTableUpdateListener;
+import io.deephaven.engine.table.impl.QueryTable;
+import io.deephaven.engine.table.impl.ShiftObliviousInstrumentedListener;
 import io.deephaven.engine.tablelogger.EngineTableLoggers;
 import io.deephaven.engine.tablelogger.UpdatePerformanceLogLogger;
 import io.deephaven.engine.tablelogger.impl.memory.MemoryTableLogger;
+import io.deephaven.engine.updategraph.UpdateGraph;
 import io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph;
-import io.deephaven.engine.table.impl.*;
 import io.deephaven.internal.log.LoggerFactory;
 import io.deephaven.io.logger.Logger;
 import io.deephaven.util.QueryConstants;
@@ -76,7 +82,8 @@ private UpdatePerformanceTracker(@NotNull final Logger logger) {
     }
 
     private void startThread() {
-        Thread driverThread = new Thread(new Driver(), "UpdatePerformanceTracker.Driver");
+        final UpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph();
+        Thread driverThread = new Thread(new Driver(updateGraph), "UpdatePerformanceTracker.Driver");
         driverThread.setDaemon(true);
         driverThread.start();
     }
@@ -90,6 +97,13 @@ public static synchronized void start() {
     }
 
     private class Driver implements Runnable {
+
+        private final UpdateGraph updateGraph;
+
+        public Driver(@NotNull final UpdateGraph updateGraph) {
+            this.updateGraph = updateGraph;
+        }
+
         @Override
         public void run() {
             // noinspection InfiniteLoopStatement
@@ -102,7 +116,7 @@ public void run() {
                     // should log, but no logger handy
                     // ignore
                 }
-                getQueryTable().getUpdateGraph().sharedLock().doLocked(
+                updateGraph.sharedLock().doLocked(
                         () -> finishInterval(intervalStartTimeMillis, System.currentTimeMillis(),
                                 System.nanoTime() - intervalStartTimeNanos));
             }
@@ -232,6 +246,7 @@ public long getIntervalDurationNanos() {
         }
     }
 
+    @NotNull
     public QueryTable getQueryTable() {
         return MemoryTableLogger.maybeGetQueryTable(tableLogger);
     }

engine/table/src/main/java/io/deephaven/engine/table/impl/util/ServerStateTracker.java

@@ -4,13 +4,16 @@
 package io.deephaven.engine.table.impl.util;
 
 import io.deephaven.configuration.Configuration;
+import io.deephaven.engine.context.ExecutionContext;
+import io.deephaven.engine.table.impl.QueryTable;
 import io.deephaven.engine.tablelogger.EngineTableLoggers;
 import io.deephaven.engine.tablelogger.ServerStateLogLogger;
-import io.deephaven.engine.table.impl.QueryTable;
 import io.deephaven.engine.tablelogger.impl.memory.MemoryTableLogger;
+import io.deephaven.engine.updategraph.UpdateGraph;
 import io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph;
-import io.deephaven.io.logger.Logger;
 import io.deephaven.internal.log.LoggerFactory;
+import io.deephaven.io.logger.Logger;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -47,8 +50,9 @@ private ServerStateTracker() {
     }
 
     private void startThread() {
+        final UpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph();
         Thread driverThread = new Thread(
-                new ServerStateTracker.Driver(),
+                new ServerStateTracker.Driver(updateGraph),
                 ServerStateTracker.class.getSimpleName() + ".Driver");
         driverThread.setDaemon(true);
         driverThread.start();
@@ -110,6 +114,12 @@ static void calcStats(final Stats out, final long[] values, final int nValues) {
     }
 
     private class Driver implements Runnable {
+        private final PeriodicUpdateGraph updateGraph;
+
+        public Driver(@NotNull final UpdateGraph updateGraph) {
+            this.updateGraph = updateGraph.cast();
+        }
+
         @Override
         public void run() {
             final RuntimeMemory.Sample memSample = new RuntimeMemory.Sample();
@@ -126,7 +136,6 @@ public void run() {
                 final long prevTotalCollections = memSample.totalCollections;
                 final long prevTotalCollectionTimeMs = memSample.totalCollectionTimeMs;
                 RuntimeMemory.getInstance().read(memSample);
-                PeriodicUpdateGraph updateGraph = getQueryTable().getUpdateGraph().cast();
                 updateGraph.takeAccumulatedCycleStats(ugpAccumCycleStats);
                 final long endTimeMillis = System.currentTimeMillis();
                 logProcessMem(

engine/table/src/main/java/io/deephaven/engine/tablelogger/impl/memory/MemoryTableLogger.java

@@ -8,6 +8,7 @@
 import io.deephaven.io.logger.Logger;
 import io.deephaven.tablelogger.TableLoggerImpl2;
 import io.deephaven.tablelogger.WritableRowContainer;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -16,6 +17,7 @@
  * Base class for memory table loggers that create and initialize a {@link DynamicTableWriter}.
  */
 public abstract class MemoryTableLogger<T extends WritableRowContainer> extends TableLoggerImpl2<T> {
+    @NotNull
     public static QueryTable maybeGetQueryTable(final Object maybeMemoryTableLogger) {
         if (maybeMemoryTableLogger instanceof MemoryTableLogger) {
             return ((MemoryTableLogger) maybeMemoryTableLogger).getQueryTable();

engine/table/src/test/java/io/deephaven/engine/table/impl/util/RowRedirectionLockFreeTest.java

@@ -5,23 +5,32 @@
 
 import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.testutil.ControlledUpdateGraph;
-import io.deephaven.engine.testutil.testcase.RefreshingTableTestCase;
+import io.deephaven.engine.testutil.junit4.EngineCleanup;
 import io.deephaven.engine.updategraph.LogicalClock;
 import gnu.trove.list.array.TLongArrayList;
 import io.deephaven.test.types.OutOfBandTest;
+import junit.framework.TestCase;
 import org.apache.commons.lang3.mutable.MutableInt;
 
 import java.util.Arrays;
 import java.util.Random;
+
+import org.junit.Rule;
+import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 import static io.deephaven.base.ArrayUtil.swap;
 
 @Category(OutOfBandTest.class)
-public class RowRedirectionLockFreeTest extends RefreshingTableTestCase {
+public class RowRedirectionLockFreeTest {
+
+    @Rule
+    public final EngineCleanup framework = new EngineCleanup();
+
     private static final long oneBillion = 1000000000L;
     private static final int testDurationInSeconds = 15;
 
+    @Test
     public void testRowRedirection() throws InterruptedException {
         final WritableRowRedirectionLockFree index = new RowRedirectionLockFreeFactory().createRowRedirection(10);
         index.startTrackingPrevValues();
@@ -44,16 +53,19 @@ public void testRowRedirection() throws InterruptedException {
         for (RWBase rwb : participants) {
             rwb.cancel();
         }
-        for (Thread thread : threads) {
-            thread.join();
+        for (int ii = 0; ii < threads.length; ++ii) {
+            threads[ii].join();
+            if (participants[ii].caughtException != null) {
+                throw participants[ii].caughtException;
+            }
         }
         boolean failed = false;
         for (RWBase rwb : participants) {
             System.out.println(rwb);
             failed |= rwb.hasFailed();
         }
         if (failed) {
-            fail("WritableRowRedirection had some corrupt values");
+            TestCase.fail("WritableRowRedirection had some corrupt values");
         }
     }
 
@@ -67,6 +79,7 @@ private static abstract class RWBase implements Runnable, Cancellable {
         protected final WritableRowRedirectionLockFree index;
         protected int numIterations;
         protected volatile boolean cancelled;
+        protected volatile RuntimeException caughtException;
 
         protected RWBase(String name, long initialStep, WritableRowRedirectionLockFree index) {
             this.name = name;
@@ -77,9 +90,13 @@ protected RWBase(String name, long initialStep, WritableRowRedirectionLockFree i
         }
 
         public final void run() {
-            while (!cancelled) {
-                doOneIteration();
-                ++numIterations;
+            try {
+                while (!cancelled) {
+                    doOneIteration();
+                    ++numIterations;
+                }
+            } catch (RuntimeException e) {
+                caughtException = e;
             }
         }
 
@@ -100,6 +117,8 @@ private static class Reader extends RWBase {
         private int badUpdateCycles;
         private int incoherentCycles;
 
+        protected final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast();
+
         Reader(String name, long initialStep, WritableRowRedirectionLockFree index) {
             super(name, initialStep, index);
             goodIdleCycles = 0;
@@ -112,7 +131,7 @@ private static class Reader extends RWBase {
         @Override
         protected final void doOneIteration() {
             // Figure out what step we're in and what step to read from (current or prev).
-            final long logicalClockStartValue = ExecutionContext.getContext().getUpdateGraph().clock().currentValue();
+            final long logicalClockStartValue = updateGraph.clock().currentValue();
             final long stepFromCycle = LogicalClock.getStep(logicalClockStartValue);
             final LogicalClock.State state = LogicalClock.getState(logicalClockStartValue);
             final long step = state == LogicalClock.State.Updating ? stepFromCycle - 1 : stepFromCycle;
@@ -153,7 +172,7 @@ protected final void doOneIteration() {
             }
 
 
-            final long logicalClockEndValue = ExecutionContext.getContext().getUpdateGraph().clock().currentValue();
+            final long logicalClockEndValue = updateGraph.clock().currentValue();
             if (logicalClockStartValue != logicalClockEndValue) {
                 ++incoherentCycles;
                 return;
@@ -188,6 +207,8 @@ public String toString() {
     }
 
     private static class Writer extends RWBase {
+        final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast();
+
         Writer(String name, long initialStep, WritableRowRedirectionLockFree index) {
             super(name, initialStep, index);
         }
@@ -196,7 +217,6 @@ private static class Writer extends RWBase {
         protected final void doOneIteration() {
             final MutableInt keysInThisGeneration = new MutableInt();
             // A bit of a waste because we only look at the first 'numKeysToInsert' keys, but that's ok.
-            final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast();
             updateGraph.runWithinUnitTestCycle(() -> {
                 final long step = updateGraph.clock().currentStep();
                 keysInThisGeneration.setValue((int) ((step - initialStep) * 1000 + 1000));