Merge remote-tracking branch 'upstream/main' into cfs-update-cpp-agai…

…n-again

.github/CODEOWNERS

@@ -24,3 +24,4 @@
 /py @chipkent @jmao-denver @rcaudy
 /R @chipkent @alexpeters1208 @rcaudy
 *.proto @devinrsmith @nbauernfeind @niloc132 @rcaudy
+*.gwt.xml @niloc132 @rcaudy @nbauernfeind

.github/workflows/cla.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: "CLA Assistant"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: cla-assistant/github-action@v2.4.0
+        uses: cla-assistant/github-action@v2.5.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_PERSONAL_ACCESS_TOKEN }}

.github/workflows/create-docs-issues.yml

@@ -26,7 +26,7 @@ jobs:
             await script({github, context});
 
       - name: Slack Failure Message
-        uses: slackapi/slack-github-action@v1.26.0
+        uses: slackapi/slack-github-action@v1.27.0
         id: slack-failure-message
         if: failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'deephaven'
         env:

.github/workflows/nightly-check-ci.yml

@@ -100,7 +100,7 @@ jobs:
           report_paths: '**/build/test-results/*/TEST-*.xml'
 
       - name: Slack Nightly Failure
-        uses: slackapi/slack-github-action@v1.26.0
+        uses: slackapi/slack-github-action@v1.27.0
         id: slack-nightly-failure
         if: ${{ failure() && github.repository_owner == 'deephaven' }}
         with:

.github/workflows/nightly-image-check.yml

@@ -35,7 +35,7 @@ jobs:
         run: ./gradlew --continue pullImage compareImage
 
       - name: Notify Slack
-        uses: slackapi/slack-github-action@v1.26.0
+        uses: slackapi/slack-github-action@v1.27.0
         id: notify-slack
         if: ${{ failure() }}
         env:

.github/workflows/nightly-publish-ci.yml

@@ -55,7 +55,7 @@ jobs:
           ORG_GRADLE_PROJECT_signingRequired: true
 
       - name: Slack Nightly Failure
-        uses: slackapi/slack-github-action@v1.26.0
+        uses: slackapi/slack-github-action@v1.27.0
         id: slack-nightly-failure
         if: failure()
         with:

.github/workflows/update-web.yml

@@ -33,7 +33,7 @@ jobs:
           sed -i "s/^ARG CHART_VERSION=.*/ARG CHART_VERSION=$CHART_VERSION/" ./web/client-ui/Dockerfile
           sed -i "s/^ARG WIDGET_VERSION=.*/ARG WIDGET_VERSION=$WIDGET_VERSION/" ./web/client-ui/Dockerfile
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         env:
           WEB_VERSION: ${{steps.web_versions.outputs.WEB_VERSION}}
         with:

Base/src/main/resources/io/deephaven/base/Base.gwt.xml

@@ -0,0 +1,4 @@
+<module>
+    <source path="" includes="MathUtil.java" />
+    <source path="verify" />
+</module>

IO/src/main/resources/io/deephaven/io/IO.gwt.xml

@@ -0,0 +1,3 @@
+<module>
+    <source path="streams" includes="ByteBufferInputStream.java" />
+</module>

Integrations/build.gradle

@@ -15,6 +15,7 @@ dependencies {
     implementation project(':plugin')
     implementation project(':Configuration')
     implementation project(':log-factory')
+    implementation libs.commons.lang3
 
     testImplementation project(':engine-test-utils')
     testImplementation project(path: ':Base', configuration: 'tests')

Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java

@@ -57,8 +57,25 @@ public class PythonDeephavenSession extends AbstractScriptSession<PythonSnapshot
     private static final String DEFAULT_SCRIPT_PATH = Configuration.getInstance()
             .getStringWithDefault("PythonDeephavenSession.defaultScriptPath", ".");
 
+    /**
+     * This is following the convention set by io.deephaven.server.config.ConfigServiceGrpcImpl / dh-defaults.prop of
+     * relaying version information to the client via Configuration properties.
+     */
+    private static final String PYTHON_VERSION_PROPERTY = "python.version";
+
     public static String SCRIPT_TYPE = "Python";
 
+    private static void setPythonVersion(Configuration configuration) {
+        final String pythonVersion;
+        try (final PyModule platformModule = PyModule.importModule("platform")) {
+            pythonVersion = platformModule.call(String.class, "python_version", new Class[0], new Object[0]);
+        } catch (RuntimeException e) {
+            log.warn(e).append("Unable to retrieve python version").endl();
+            return;
+        }
+        configuration.setProperty(PYTHON_VERSION_PROPERTY, pythonVersion);
+    }
+
     private final PythonEvaluator evaluator;
     private final PythonScope<PyObject> scope;
     private final PythonScriptSessionModule module;
@@ -67,6 +84,10 @@ public class PythonDeephavenSession extends AbstractScriptSession<PythonSnapshot
     /**
      * Create a Python ScriptSession.
      *
+     * <p>
+     * Sets the configuration property {@value PYTHON_VERSION_PROPERTY} to the value returned from the python code
+     * {@code platform.python_version()}.
+     *
      * @param updateGraph the default update graph to install for the repl
      * @param operationInitializer the default operation initializer to install for the repl
      * @param objectTypeLookup the object type lookup
@@ -96,16 +117,19 @@ public PythonDeephavenSession(
 
         registerJavaExecutor(threadInitializationFactory);
         publishInitial();
+
+        final Configuration configuration = Configuration.getInstance();
         /*
          * And now the user-defined initialization scripts, if any.
          */
         if (runInitScripts) {
-            String[] scripts = Configuration.getInstance().getProperty("PythonDeephavenSession.initScripts").split(",");
+            String[] scripts = configuration.getProperty("PythonDeephavenSession.initScripts").split(",");
 
             for (String script : scripts) {
                 runScript(script);
             }
         }
+        setPythonVersion(configuration);
     }
 
     /**

Integrations/src/main/java/io/deephaven/integrations/python/PythonMergedListenerAdapter.java

@@ -7,14 +7,18 @@
 import io.deephaven.engine.rowset.RowSetFactory;
 import io.deephaven.engine.rowset.RowSetShiftData;
 import io.deephaven.engine.table.ModifiedColumnSet;
+import io.deephaven.engine.table.TableListener;
 import io.deephaven.engine.table.TableUpdate;
 import io.deephaven.engine.table.impl.ListenerRecorder;
 import io.deephaven.engine.table.impl.MergedListener;
 import io.deephaven.engine.table.impl.TableUpdateImpl;
 import io.deephaven.engine.updategraph.NotificationQueue;
 import io.deephaven.engine.updategraph.UpdateGraph;
+import io.deephaven.internal.log.LoggerFactory;
+import io.deephaven.io.logger.Logger;
 import io.deephaven.util.SafeCloseable;
 import io.deephaven.util.annotations.ScriptApi;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jpy.PyObject;
@@ -33,7 +37,10 @@
  */
 @ScriptApi
 public class PythonMergedListenerAdapter extends MergedListener {
-    private final PyObject pyCallable;
+    private static final Logger log = LoggerFactory.getLogger(PythonMergedListenerAdapter.class);
+
+    private final PyObject pyListenerCallable;
+    private final PyObject pyOnFailureCallback;
 
     /**
      * Create a Python merged listener.
@@ -42,23 +49,26 @@ public class PythonMergedListenerAdapter extends MergedListener {
      * @param dependencies The tables that must be satisfied before this listener is executed.
      * @param listenerDescription A description for the UpdatePerformanceTracker to append to its entry description, may
      *        be null.
-     * @param pyObjectIn Python listener object.
+     * @param pyListener Python listener object.
      */
     private PythonMergedListenerAdapter(
             @NotNull ListenerRecorder[] recorders,
             @Nullable NotificationQueue.Dependency[] dependencies,
             @Nullable String listenerDescription,
-            @NotNull PyObject pyObjectIn) {
+            @NotNull PyObject pyListener,
+            @NotNull PyObject pyOnFailureCallback) {
         super(Arrays.asList(recorders), Arrays.asList(dependencies), listenerDescription, null);
         Arrays.stream(recorders).forEach(rec -> rec.setMergedListener(this));
-        this.pyCallable = PythonUtils.pyMergeListenerFunc(pyObjectIn);
+        this.pyListenerCallable = PythonUtils.pyMergeListenerFunc(Objects.requireNonNull(pyListener));
+        this.pyOnFailureCallback = Objects.requireNonNull(pyOnFailureCallback);
     }
 
     public static PythonMergedListenerAdapter create(
             @NotNull ListenerRecorder[] recorders,
             @Nullable NotificationQueue.Dependency[] dependencies,
             @Nullable String listenerDescription,
-            @NotNull PyObject pyObjectIn) {
+            @NotNull PyObject pyListener,
+            @NotNull PyObject pyOnFailureCallback) {
         if (recorders.length < 2) {
             throw new IllegalArgumentException("At least two listener recorders must be provided");
         }
@@ -71,7 +81,8 @@ public static PythonMergedListenerAdapter create(
         final UpdateGraph updateGraph = allItems[0].getUpdateGraph(allItems);
 
         try (final SafeCloseable ignored = ExecutionContext.getContext().withUpdateGraph(updateGraph).open()) {
-            return new PythonMergedListenerAdapter(recorders, dependencies, listenerDescription, pyObjectIn);
+            return new PythonMergedListenerAdapter(recorders, dependencies, listenerDescription, pyListener,
+                    pyOnFailureCallback);
         }
     }
 
@@ -91,6 +102,22 @@ public ArrayList<TableUpdate> currentRowsAsUpdates() {
 
     @Override
     protected void process() {
-        pyCallable.call("__call__");
+        pyListenerCallable.call("__call__");
+    }
+
+    @Override
+    protected void propagateErrorDownstream(boolean fromProcess, @NotNull Throwable error,
+            TableListener.@Nullable Entry entry) {
+        if (!pyOnFailureCallback.isNone()) {
+            try {
+                pyOnFailureCallback.call("__call__", ExceptionUtils.getStackTrace(error));
+            } catch (Exception e2) {
+                // If the Python onFailure callback fails, log the new exception
+                // and continue with the original exception.
+                log.error().append("Python on_error callback failed: ").append(e2).endl();
+            }
+        } else {
+            log.error().append("Python on_error callback is None: ").append(ExceptionUtils.getStackTrace(error)).endl();
+        }
     }
 }

Integrations/src/main/java/io/deephaven/integrations/python/PythonReplayListenerAdapter.java

@@ -3,6 +3,7 @@
 //
 package io.deephaven.integrations.python;
 
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableUpdate;
@@ -14,12 +15,16 @@
 import io.deephaven.engine.rowset.RowSetShiftData;
 import io.deephaven.engine.updategraph.NotificationQueue;
 import io.deephaven.engine.updategraph.UpdateGraph;
+import io.deephaven.internal.log.LoggerFactory;
+import io.deephaven.io.logger.Logger;
 import io.deephaven.util.SafeCloseable;
 import io.deephaven.util.annotations.ScriptApi;
+import org.jetbrains.annotations.NotNull;
 import org.jpy.PyObject;
 
 import javax.annotation.Nullable;
 import java.util.Arrays;
+import java.util.Objects;
 
 
 /**
@@ -33,7 +38,10 @@
 public class PythonReplayListenerAdapter extends InstrumentedTableUpdateListenerAdapter
         implements TableSnapshotReplayer {
     private static final long serialVersionUID = -8882402061960621245L;
-    private final PyObject pyCallable;
+    private static final Logger log = LoggerFactory.getLogger(PythonReplayListenerAdapter.class);
+
+    private final PyObject pyListenerCallable;
+    private final PyObject pyOnFailureCallback;
     private final NotificationQueue.Dependency[] dependencies;
 
     /**
@@ -43,22 +51,34 @@ public class PythonReplayListenerAdapter extends InstrumentedTableUpdateListener
      *        null.
      * @param source The source table to which this listener will subscribe.
      * @param retain Whether a hard reference to this listener should be maintained to prevent it from being collected.
-     * @param pyObjectIn Python listener object.
+     * @param pyListener Python listener object.
      * @param dependencies The tables that must be satisfied before this listener is executed.
      */
-    public static PythonReplayListenerAdapter create(@Nullable String description, Table source, boolean retain,
-            PyObject pyObjectIn, NotificationQueue.Dependency... dependencies) {
+    public static PythonReplayListenerAdapter create(
+            @Nullable String description,
+            @NotNull Table source,
+            boolean retain,
+            @NotNull PyObject pyListener,
+            @NotNull PyObject pyOnFailureCallback,
+            @Nullable NotificationQueue.Dependency... dependencies) {
         final UpdateGraph updateGraph = source.getUpdateGraph(dependencies);
         try (final SafeCloseable ignored = ExecutionContext.getContext().withUpdateGraph(updateGraph).open()) {
-            return new PythonReplayListenerAdapter(description, source, retain, pyObjectIn, dependencies);
+            return new PythonReplayListenerAdapter(description, source, retain, pyListener, pyOnFailureCallback,
+                    dependencies);
         }
     }
 
-    private PythonReplayListenerAdapter(@Nullable String description, Table source, boolean retain, PyObject pyObjectIn,
-            NotificationQueue.Dependency... dependencies) {
+    private PythonReplayListenerAdapter(
+            @Nullable String description,
+            @NotNull Table source,
+            boolean retain,
+            @NotNull PyObject pyListener,
+            @NotNull PyObject pyOnFailureCallback,
+            @Nullable NotificationQueue.Dependency... dependencies) {
         super(description, source, retain);
         this.dependencies = dependencies;
-        this.pyCallable = PythonUtils.pyListenerFunc(pyObjectIn);
+        this.pyListenerCallable = PythonUtils.pyListenerFunc(Objects.requireNonNull(pyListener));
+        this.pyOnFailureCallback = Objects.requireNonNull(pyOnFailureCallback);
     }
 
     @Override
@@ -69,18 +89,39 @@ public void replay() {
         final TableUpdate update =
                 new TableUpdateImpl(source.getRowSet(), emptyRowSet, emptyRowSet, emptyShift, emptyColumnSet);
         final boolean isReplay = true;
-        pyCallable.call("__call__", update, isReplay);
+        pyListenerCallable.call("__call__", update, isReplay);
     }
 
     @Override
     public void onUpdate(final TableUpdate update) {
         final boolean isReplay = false;
-        pyCallable.call("__call__", update, isReplay);
+        pyListenerCallable.call("__call__", update, isReplay);
+    }
+
+    @Override
+    public void onFailureInternal(Throwable originalException, Entry sourceEntry) {
+        if (!pyOnFailureCallback.isNone()) {
+            try {
+                pyOnFailureCallback.call("__call__", ExceptionUtils.getStackTrace(originalException));
+            } catch (Throwable e) {
+                // If the Python onFailure callback fails, log the new exception
+                // and continue with the original exception.
+                log.error().append("Python on_error callback failed: ").append(e).endl();
+            }
+        } else {
+            log.error().append("Python on_error callback is None: ")
+                    .append(ExceptionUtils.getStackTrace(originalException)).endl();
+        }
+        super.onFailureInternal(originalException, sourceEntry);
     }
 
     @Override
     public boolean canExecute(final long step) {
         return super.canExecute(step)
                 && (dependencies.length == 0 || Arrays.stream(dependencies).allMatch(t -> t.satisfied(step)));
     }
+
+    public boolean isFailed() {
+        return failed;
+    }
 }

R/build.gradle

@@ -138,7 +138,7 @@ def rClientDoc = Docker.registerDockerTask(project, 'rClientDoc') {
                    ''')
         runCommand('''echo "status = tryCatch(" \
                            "   {" \
-                           "      install.packages('roxygen2', repos='https://cran.r-project.org'); " \
+                           "      install.packages('roxygen2'); " \
                            "      0" \
                            "   }," \
                            "  error=function(e) 1," \
@@ -179,7 +179,7 @@ def rClientSite = Docker.registerDockerTask(project, 'rClientSite') {
         runCommand("mkdir -p ${prefix}/src/rdeephaven/docs")
         runCommand('''echo "status = tryCatch(" \
                            "   {" \
-                           "      install.packages('pkgdown', repos='https://cran.r-project.org'); " \
+                           "      install.packages('pkgdown'); " \
                            "      0" \
                            "   }," \
                            "  error=function(e) 1," \

R/rdeephaven/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: rdeephaven
 Type: Package
 Title: R Client for Deephaven Core
-Version: 0.36.0
+Version: 0.37.0
 Date: 2023-05-12
 Author: Deephaven Data Labs
 Maintainer: Alex Peters <[email protected]>