Adjust tests for electron playwright

* Update theia dependency to 1.46.1 * Use new TheiaApploader * Add workaround for Electron playwright multi root issue * Support different projects (e.g., chromium or Electron) in performance reports * Remove dependency to deprecated xvfb-action Contributed on behalf of STMicroelectronics Signed-off-by: Olaf Lessenich <[email protected]>
eclipsesource · Mar 27, 2024 · 969c59e · 969c59e
1 parent 97ecba0
commit 969c59e
Show file tree

Hide file tree

Showing 10 changed files with 316 additions and 91 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,7 +14,6 @@ on:
 jobs:
   build:
     name: E2E Tests on ubuntu-latest with Node.js 16.x
-
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
@@ -50,20 +49,102 @@ jobs:
         run: |
           yarn --skip-integrity-check --network-timeout 100000
           yarn browser build
+          yarn electron build
         env:
           NODE_OPTIONS: --max_old_space_size=4096
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
 
+      - name: Cache Build Result
+        uses: actions/cache@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+
+  browser-tests:
+    name: E2E Browser Tests on ubuntu-latest with Node.js 16.x
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: [build]
+
+    steps:
+      - name: Restore Build Result (#${{ github.run_number }})
+        uses: actions/cache/restore@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+          fail-on-cache-miss: true
+
+      - name: Use Node.js "16.x"
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16.x"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Use Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
       - name: Run Theia
         shell: bash
         working-directory: ./theia
         run: yarn browser start &
 
-      - name: Test (playwright)
-        uses: GabrielBB/xvfb-action@v1
+      - name: Test (playwright browser)
         with:
           run: yarn ui-tests-ci
 
+      - name: Upload test results (browser)
+        uses: actions/upload-artifact@v3
+        with:
+          name: ui-tests-browser-${{ github.run_number }}
+          path: allure_results/*
+
+  electron-tests:
+    name: E2E Electron Tests on ubuntu-latest with Node.js 16.x
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: [build]
+
+    steps:
+      - name: Restore Build Result (#${{ github.run_number }})
+        uses: actions/cache/restore@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+          fail-on-cache-miss: true
+
+      - name: Use Node.js "16.x"
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16.x"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Use Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Test (playwright electron)
+        with:
+          run: xvfb-run yarn ui-tests-ci-electron
+
+      - name: Upload test results (electron)
+        uses: actions/upload-artifact@v3
+        with:
+          name: ui-tests-electron-${{ github.run_number }}
+          path: allure_results/*
+
+  generate-report:
+    name: E2E Electron Tests on ubuntu-latest with Node.js 16.x
+    runs-on: ubuntu-latest
+    needs: [browser-tests, electron-tests]
+
+    steps:
+
       - name: Get History
         uses: actions/checkout@v2
         if: always() && github.ref == 'refs/heads/main'
@@ -72,6 +153,13 @@ jobs:
           ref: gh-pages
           path: gh-pages
 
+      - name: Download test results
+        uses: actions/download-artifact@v3
+        with:
+          path: ui-tests-*-${{ github.run_number }}
+          merge-multiple: true
+          path: allure_results
+
       - name: Generate Report
         uses: simple-elf/allure-report-action@master
         if: always() && github.ref == 'refs/heads/main'

diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -34,6 +34,7 @@ jobs:
         shell: bash
         run: |
           cp -f ./configs/theia.production.webpack.config.js ./theia/examples/browser/webpack.config.js
+          cp -f ./configs/theia.production.webpack.config.js ./theia/examples/electron/webpack.config.js
       - name: Build Theia
         shell: bash
         working-directory: ./theia
@@ -51,15 +52,23 @@ jobs:
         env:
           NODE_OPTIONS: --max_old_space_size=4096
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
+      - name: Build Electron App (Production)
+        shell: bash
+        working-directory: ./theia/examples/electron
+        run: |
+          yarn -s compile && yarn rebuild && yarn theia build --mode=production
+        env:
+          NODE_OPTIONS: --max_old_space_size=4096
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
       - name: Cache Build Result
         uses: actions/cache@v3
         id: build-result
         with:
           path: ./*
           key: ${{ github.run_number }}
 
-  run-test:
-    name: Performance benchmark on ubuntu-latest with Node.js 16.x
+  run-browser-tests:
+    name: Performance browser benchmark on ubuntu-latest with Node.js 16.x
     runs-on: ubuntu-latest
     needs: [prepare]
     strategy:
@@ -79,35 +88,75 @@ jobs:
         with:
           node-version: "16.x"
           registry-url: "https://registry.npmjs.org"
-      - name: Use Python 3.x
+      - name: Use Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: "3.x"
+          python-version: "3.11"
       - name: Get History
         uses: actions/checkout@v4
         with:
           ref: gh-pages
           path: gh-pages
-      - name: Run Theia (#${{ matrix.run }})
+      - name: Run Theia browser backend (#${{ matrix.run }})
         shell: bash
         working-directory: ./theia
         run: yarn browser start:debug &
-      - name: Run Performance Measurement (#${{ matrix.run }})
-        uses: GabrielBB/xvfb-action@v1
+      - name: Run Performance Measurement: browser (#${{ matrix.run }})
         env:
           RUN_NO: ${{ matrix.run }}
         with:
           run: yarn performance
-      - name: Upload performance measurement (#${{ matrix.run }})
+      - name: Upload performance measurement: browser (#${{ matrix.run }})
+        uses: actions/upload-artifact@v3
+        with:
+          name: performance-measurement-browser-${{ github.run_number }}
+          path: performance-metrics/*
+
+  run-electron-tests:
+    name: Performance electron benchmark on ubuntu-latest with Node.js 16.x
+    runs-on: ubuntu-latest
+    needs: [prepare]
+    strategy:
+      matrix:
+        run: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    timeout-minutes: 40
+    steps:
+      - name: Restore Build Result (#${{ matrix.run }})
+        uses: actions/cache/restore@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+          fail-on-cache-miss: true
+      - name: Use Node.js "16.x"
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16.x"
+          registry-url: "https://registry.npmjs.org"
+      - name: Use Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Get History
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          path: gh-pages
+      - name: Run Performance Measurement: electron (#${{ matrix.run }})
+        env:
+          RUN_NO: ${{ matrix.run }}
+        with:
+          run: xvfb-run yarn performance-on-electron
+      - name: Upload performance measurement: electron (#${{ matrix.run }})
         uses: actions/upload-artifact@v3
         with:
-          name: performance-measurement-${{ github.run_number }}
+          name: performance-measurement-electron-${{ github.run_number }}
           path: performance-metrics/*
 
   generate-report:
     name: Performance benchmark on ubuntu-latest with Node.js 16.x
     runs-on: ubuntu-latest
-    needs: [run-test]
+    needs: [run-browser-tests, run-electron-tests]
     steps:
       - name: Restore Build Result
         uses: actions/cache/restore@v3
@@ -126,12 +175,17 @@ jobs:
       - name: Download Performance Measurements
         uses: actions/download-artifact@v3
         with:
-          name: performance-measurement-${{ github.run_number }}
+          pattern: performance-measurement-*-${{ github.run_number }}
+          merge-multiple: true
           path: performance-metrics
-      - name: Prepare Report
+      - name: Prepare Browser Report
         if: always() && github.ref == 'refs/heads/main'
         shell: bash
         run: yarn performance-report
+      - name: Prepare Electron Report
+        if: always() && github.ref == 'refs/heads/main'
+        shell: bash
+        run: yarn performance-report-electron
       - name: Publish Report
         if: always() && github.ref == 'refs/heads/main'
         uses: peaceiris/actions-gh-pages@v2

diff --git a/configs/performance.config.ts b/configs/performance.config.ts
@@ -20,7 +20,6 @@ import { MetricsFetchConfig } from '../scripts/fetch-metrics';
 export default defineConfig({
     testDir: '../lib/tests',
     testMatch: ['**/*.performance.js'],
-    globalTeardown: require.resolve('../scripts/fetch-metrics.ts'),
     workers: 1,
     timeout: 60 * 1000,
     use: {
@@ -51,6 +50,9 @@ export default defineConfig({
                 ...devices['Desktop Chrome'],
                 channel: 'chrome'
             },
-        }
+        },
+        {
+            name: 'Electron',
+        },
     ]
 });
diff --git a/configs/playwright.config.ts b/configs/playwright.config.ts
@@ -36,6 +36,14 @@ const config: PlaywrightTestConfig = {
     reporter: [
         ['list'],
         ['allure-playwright']
+    ],
+    projects: [
+        {
+            name: 'Browser',
+        },
+        {
+            name: 'Electron',
+        }
     ]
 };
 

diff --git a/package.json b/package.json
@@ -1,37 +1,41 @@
 {
   "private": true,
   "name": "theia-e2e-test-suite",
-  "version": "1.39.0",
+  "version": "1.46.1",
   "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
   "scripts": {
     "prepare": "yarn clean && yarn build",
     "clean": "rimraf lib tsconfig.tsbuildinfo",
     "build": "tsc --incremental && if-env SKIP_CHROMIUM_INSTALLATION=1 && echo 'Skipping chromium install' || npx playwright install chromium",
-    "ui-tests": "yarn && playwright test --config=./configs/playwright.config.ts",
-    "ui-tests-ci": "yarn && playwright test --config=./configs/playwright.ci.config.ts",
-    "ui-tests-headful": "yarn && playwright test --config=./configs/playwright.headful.config.ts",
+    "ui-tests": "yarn && playwright test --config=./configs/playwright.config.ts --project=Browser",
+    "ui-tests-electron": "yarn && USE_ELECTRON=true playwright test --config=./configs/playwright.config.ts --project=Electron",
+    "ui-tests-ci": "yarn && playwright test --config=./configs/playwright.ci.config.ts --project=Browser",
+    "ui-tests-ci-electron": "yarn && USE_ELECTRON=true playwright test --config=./configs/playwright.ci.config.ts --project=Electron",
+    "ui-tests-headful": "yarn && playwright test --config=./configs/playwright.headful.config.ts --project=Browser",
     "ui-tests-report-generate": "allure generate ./allure-results --clean -o allure-results/allure-report",
     "ui-tests-report": "yarn ui-tests-report-generate && allure open allure-results/allure-report",
     "performance": "yarn run performance-on-chromium",
     "performance-on-chromium": "yarn && playwright test --config=./configs/performance.config.ts --project=chromium",
     "performance-on-chrome": "yarn && playwright test --config=./configs/performance.config.ts --project=Chrome",
-    "performance-report": "yarn && node lib/scripts/performance-report.js"
+    "performance-on-electron": "yarn && USE_ELECTRON=true playwright test --config=./configs/performance.config.ts --project=Electron",
+    "performance-report": "yarn && node lib/scripts/performance-report.js --performanceMetricsPath performance-metrics/chromium --performancePublishPath=performance/chromium",
+    "performance-report-electron": "yarn && node lib/scripts/performance-report.js --performanceMetricsPath performance-metrics/Electron --performancePublishPath=performance/Electron"
   },
   "dependencies": {
-    "@playwright/test": "^1.35.1",
-    "@theia/playwright": "^1.39.0"
+    "@playwright/test": "^1.37.1",
+    "@theia/playwright": "^1.46.1"
   },
   "devDependencies": {
-    "allure-commandline": "^2.23.0",
-    "allure-playwright": "^2.4.0",
-    "rimraf": "^3.0.0",
-    "node-fetch": "^2.6.7",
-    "@types/node-fetch": "^2.6.4",
-    "fs-extra": "^11.1.1",
     "@types/fs-extra": "^11.0.1",
-    "yargs": "^17.7.2",
+    "@types/node-fetch": "^2.6.4",
     "@types/yargs": "^17.0.24",
+    "allure-commandline": "^2.23.1",
+    "allure-playwright": "^2.5.0",
+    "fs-extra": "^11.1.1",
     "if-env": "^1.0.4",
-    "typescript": "~4.5.5"
+    "node-fetch": "^2.6.7",
+    "rimraf": "^3.0.0",
+    "typescript": "~4.5.5",
+    "yargs": "^17.7.2"
   }
 }
diff --git a/scripts/fetch-metrics.ts b/scripts/fetch-metrics.ts
@@ -18,8 +18,8 @@ import { type FullConfig } from '@playwright/test';
 import * as fs from 'fs-extra';
 import fetch from 'node-fetch';
 
-async function fetchMetrics(config: FullConfig) {
-    await fetchPerformanceMetrics(config.metadata.performanceMetrics, config.metadata.totalTime);
+async function fetchMetrics(projectName: string, config: FullConfig) {
+    await fetchPerformanceMetrics(projectName, config.metadata.performanceMetrics, config.metadata.totalTime);
 }
 
 export default fetchMetrics;
@@ -31,13 +31,17 @@ export interface MetricsFetchConfig {
     outputFilePath: string;
 }
 
-export async function fetchPerformanceMetrics({
-    metricsEndpoint,
-    outputFileNamePrefix,
-    outputFileNamePostfix,
-    outputFilePath }: MetricsFetchConfig,
+export async function fetchPerformanceMetrics(
+    projectName: string,
+    {
+        metricsEndpoint,
+        outputFileNamePrefix,
+        outputFileNamePostfix,
+        outputFilePath
+    }: MetricsFetchConfig,
     totalTime?: string
 ) {
+    outputFilePath = `${outputFilePath}/${projectName}`;
     const now = new Date();
     const dateString = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`;
     const timeString = `${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`;