UI visual regression testing to cover UI widgets visibility

.github/workflows/ui_notebooks_test.yaml

@@ -0,0 +1,112 @@
+name: UI notebooks tests
+
+on: [pull_request]
+
+concurrency:
+  group: ${{ github.head_ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
+
+jobs:
+  verify-0_basic_ray:
+    runs-on: ubuntu-20.04-4core
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Checkout common repo code
+        uses: actions/checkout@v4
+        with:
+          repository: "project-codeflare/codeflare-common"
+          ref: "main"
+          path: "common"
+
+      - name: Checkout CodeFlare operator repository
+        uses: actions/checkout@v4
+        with:
+          repository: project-codeflare/codeflare-operator
+          path: codeflare-operator
+
+      - name: Set Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: "./codeflare-operator/go.mod"
+          cache-dependency-path: "./codeflare-operator/go.sum"
+
+      - name: Set up gotestfmt
+        uses: gotesttools/gotestfmt-action@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up specific Python version
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+          cache: "pip" # caching pip dependencies
+
+      - name: Setup and start KinD cluster
+        uses: ./common/github-actions/kind
+
+      - name: Deploy CodeFlare stack
+        id: deploy
+        run: |
+          cd codeflare-operator
+          echo Setting up CodeFlare stack
+          make setup-e2e
+          echo Deploying CodeFlare operator
+          make deploy -e IMG="${CODEFLARE_OPERATOR_IMG}" -e ENV="e2e"
+          kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager
+          cd ..
+
+      - name: Setup Guided notebooks execution
+        run: |
+          echo "Installing papermill and dependencies..."
+          pip install poetry ipython ipykernel
+          poetry config virtualenvs.create false
+          echo "Installing SDK..."
+          poetry install --with test,docs
+
+      - name: Install Yarn dependencies
+        run: |
+          poetry run yarn install
+          poetry run yarn playwright install chromium
+        working-directory: ui-tests
+
+      - name: Run UI notebook tests
+        run: |
+          set -euo pipefail
+          poetry run yarn test
+        working-directory: ui-tests
+
+      - name: Print CodeFlare operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing CodeFlare operator logs"
+          kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
+
+      - name: Print KubeRay operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing KubeRay operator logs"
+          kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
+
+      - name: Upload Playwright Test assets
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ipywidgets-test-assets
+          path: |
+            ui-tests/test-results
+
+      - name: Upload Playwright Test report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ipywidgets-test-report
+          path: |
+            ui-tests/playwright-report

.gitignore

@@ -8,3 +8,7 @@ Pipfile.lock
 build/
 tls-cluster-namespace
 quicktest.yaml
+node_modules
+.DS_Store
+ui-tests/playwright-report
+ui-tests/test-results

pyproject.toml

@@ -48,6 +48,7 @@ pytest = "7.4.0"
 coverage = "7.2.7"
 pytest-mock = "3.11.1"
 pytest-timeout = "2.3.1"
+jupyterlab = "4.2.5"
 
 [tool.pytest.ini_options]
 filterwarnings = [

ui-tests/.yarnrc

@@ -0,0 +1,4 @@
+disable-self-update-check true
+ignore-optional true
+network-timeout "300000"
+registry "https://registry.npmjs.org/"

ui-tests/jupyter_server_config.py

@@ -0,0 +1,6 @@
+from jupyterlab.galata import configure_jupyter_server
+
+configure_jupyter_server(c)
+
+# Uncomment to set server log level to debug level
+# c.ServerApp.log_level = "DEBUG"

ui-tests/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@jupyter-widgets/ui-tests",
+  "private": true,
+  "version": "0.1.0",
+  "description": "ipywidgets UI Tests",
+  "scripts": {
+    "start": "jupyter lab --config ./jupyter_server_config.py",
+    "start:detached": "jlpm start&",
+    "test": "npx playwright test",
+    "test:debug": "PWDEBUG=1 npx playwright test",
+    "test:report": "http-server ./playwright-report -a localhost -o",
+    "test:update": "npx playwright test --update-snapshots",
+    "deduplicate": "jlpm && yarn-deduplicate -s fewer --fail"
+  },
+  "author": "Project Jupyter",
+  "license": "BSD-3-Clause",
+  "devDependencies": {
+    "@jupyterlab/galata": "^5.0.1",
+    "@playwright/test": "^1.32.0",
+    "yarn-deduplicate": "^6.0.1"
+  }
+}

ui-tests/playwright.config.js

@@ -0,0 +1,13 @@
+const baseConfig = require('@jupyterlab/galata/lib/playwright-config');
+
+module.exports = {
+  ...baseConfig,
+  timeout: 240000,
+  webServer: {
+    command: 'yarn start',
+    url: 'http://localhost:8888/lab',
+    timeout: 120 * 1000,
+    reuseExistingServer: !process.env.CI,
+  },
+  retries: 0,
+};

ui-tests/tests/0_basic_ray.test.ts

@@ -0,0 +1,52 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+import { test } from "@jupyterlab/galata";
+import { expect } from "@playwright/test";
+import * as path from "path";
+
+test.setTimeout(460000);
+
+test.describe("Visual Regression", () => {
+  test.beforeEach(async ({ page, tmpPath }) => {
+    await page.contents.uploadDirectory(
+      path.resolve(__dirname, "./notebooks"),
+      tmpPath
+    );
+    await page.filebrowser.openDirectory(tmpPath);
+  });
+
+  test("Run notebook 0_basic_ray.ipynb and capture cell outputs", async ({
+    page,
+    tmpPath,
+  }) => {
+    const notebook = "0_basic_ray.ipynb";
+    await page.notebook.openByPath(`${tmpPath}/${notebook}`);
+    await page.notebook.activate(notebook);
+
+    const captures: (Buffer | null)[] = []; // Array to store cell screenshots
+    const cellCount = await page.notebook.getCellCount();
+
+    // Run all cells and capture their screenshots
+    await page.notebook.runCellByCell({
+      onAfterCellRun: async (cellIndex: number) => {
+        const cell = await page.notebook.getCellOutput(cellIndex);
+        if (cell && (await cell.isVisible())) {
+          captures[cellIndex] = await cell.screenshot(); // Save the screenshot by cell index
+        }
+      },
+    });
+
+    await page.notebook.save();
+
+    // Ensure that each cell's screenshot is captured
+    for (let i = 0; i < cellCount; i++) {
+      const image = `widgets-cell-${i}.png`;
+
+      if (captures[i]) {
+        expect.soft(captures[i]).toMatchSnapshot(image); // Compare pre-existing capture
+        continue;
+      }
+    }
+  });
+});

ui-tests/tests/notebooks/0_basic_ray.ipynb

@@ -0,0 +1,117 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import pieces from codeflare-sdk\n",
+    "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0f4bc870-091f-4e11-9642-cba145710159",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create and configure our cluster object\n",
+    "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n",
+    "cluster = Cluster(ClusterConfiguration(\n",
+    "    name='raytest',\n",
+    "    namespace='default',\n",
+    "    head_cpus='500m',\n",
+    "    head_memory=2,\n",
+    "    head_gpus=0, # For GPU enabled workloads set the head_gpus and num_gpus\n",
+    "    num_gpus=0,\n",
+    "    num_workers=2,\n",
+    "    min_cpus='250m',\n",
+    "    max_cpus=1,\n",
+    "    min_memory=1,\n",
+    "    max_memory=2,\n",
+    "    # image=\"\", # Optional Field \n",
+    "    write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources \n",
+    "    # local_queue=\"local-queue-name\" # Specify the local queue manually\n",
+    "))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Bring up the cluster\n",
+    "cluster.up()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0d513912",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cluster.wait_ready()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ce99d84",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cluster.status()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1767a342",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cluster.down()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7e9152ce",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}