Core: Support HTML/TAP preconfig via QUnit.config.reporters

Allow disabling of HTML Reporter even if `<div id="qunit">` exists.

Ref #1711.

.eslintrc.json

@@ -245,6 +245,7 @@
       "rules": {
         // https://github.com/eslint/eslint/issues/15732
         "array-bracket-spacing": "off",
+        "camelcase": "off",
         "n/handle-callback-err": "off",
         "no-labels": "off",
         "no-undef": "off",

Gruntfile.js

@@ -53,6 +53,7 @@ module.exports = function (grunt) {
       all: {
         options: {
           timeout: 30000,
+          console: false,
           puppeteer: {
             args: isCI
 
@@ -73,6 +74,7 @@ module.exports = function (grunt) {
           urls: [
             'test/index.html',
 
+            'test/config-reporters.html',
             'test/dynamic-import.html',
             'test/events-filters.html',
             'test/events-in-test.html',
@@ -84,6 +86,7 @@ module.exports = function (grunt) {
             'test/only-each.html',
             'test/overload.html',
             'test/performance-mark.html',
+            'test/preconfig-flat-reporters.html',
             'test/preconfig-flat-testId.html',
             'test/preconfig-flat.html',
             'test/preconfig-object.html',

docs/api/config/index.md

@@ -49,7 +49,7 @@ You can configure the test run via the `QUnit.config` object. In the HTML Runner
 
 If you have custom plugins or want to re-use your configuration across multiple HTML test suites, you can also configure your project from an external `/test/bootstrap.js` script. Make sure to place this script before your other test files.
 
-When using the [QUnit CLI](https://qunitjs.com/cli/), you can setup your project and configure QUnit via [`--require`](https://qunitjs.com/cli/#--require).
+When using the [QUnit CLI](../../cli.md), you can setup your project and configure QUnit via [`--require`](../../cli.md#--require).
 
 ```bash
 qunit --require ./test/bootstrap.js

docs/api/config/reporters.md

@@ -0,0 +1,106 @@
+---
+layout: page-api
+title: QUnit.config.reporters
+excerpt: Control which reporters to enable or disable.
+groups:
+  - config
+redirect_from:
+  - "/config/reporters/"
+version_added: "unreleased"
+---
+
+Control which reporters to enable or disable.
+
+<table>
+<tr>
+  <th>type</th>
+  <td markdown="span">`Object<string,bool>`</td>
+</tr>
+<tr>
+  <th>default</th>
+  <td markdown="span">`{}`</td>
+</tr>
+</table>
+
+## Built-in reporters
+
+### tap
+
+The **tap** reporter is a [TAP compliant](https://testanything.org/) reporter. This is the default in the [QUnit CLI](../../cli.md). This allows you to pair QUnit with many [TAP-based reporters](https://github.com/sindresorhus/awesome-tap#reporters), by piping the output. For example:
+
+```sh
+qunit test/ | tap-min
+```
+
+### console
+
+The **console** reporter logs a JSON object for each reporter event from [`QUnit.on`](./api/callbacks/QUnit.on.md). Use this to explore or debug the Reporter API.
+
+```
+runStart {…}
+testStart {…}
+testEnd {…}
+testStart {…}
+testEnd {…}
+runEnd {…}
+```
+
+### perf
+
+The **perf** reporter emits measures for the duration of each QUnit test and each module via the Web Performance API. This allows you to visualize where time is spent during your test run. This uses the [performance.measure()](https://developer.mozilla.org/en-US/docs/Web/API/Performance/measure) method internally. QUnit enables the perf reporter by default in [Browser](../../browser.md) environments. The measures are included in Firefox Profiler and Chrome DevTools (Safari is pending [WebKit #213870](https://bugs.webkit.org/show_bug.cgi?id=213870)).
+
+```
+QUnit Test Run
+└── QUnit Test Suite: Example
+    ├── QUnit Test: apple
+    ├── QUnit Test: banana
+    └── QUnit Test: citron
+```
+
+<figure>
+  <img alt="QUnit profiling in Chrome DevTools Performance tab" src="/resources/perf-chrome.png">
+  <figcaption>Chrome</figcaption>
+</figure>
+
+-------
+
+<figure>
+  <img alt="QUnit performance in Firefox Profiler" src="/resources/perf-firefox.png">
+  <figcaption>Firefox</figcaption>
+</figure>
+
+
+The Web Performance API is also [available in Node.js](https://nodejs.org/docs/latest/api/perf_hooks.html) and the QUnit perf reporter can be enabled in Node.js. You can enable it in the QUnit CLI via `--reporter perf`. Note that the [Node.js inspector](https://nodejs.org/docs/latest/api/debugger.html#v8-inspector-integration-for-nodejs) does not yet send these to Chrome DevTools ([upstream nodejs/node#47813](https://github.com/nodejs/node/issues/47813)).
+
+
+### html
+
+The **html** reporter renders a toolbar and visualizes test results. This is the default in [Browser](../../browser.md) environments, and is documented at [HTML Reporter](../../browser.md#markup).
+
+## Examples
+
+By default, the [HTML Reporter](../../browser.md) is automatically enabled in browser environments if a `<div id="qunit">` element exists, and it remains disabled ("headless") if such element doesn't exist. You can override this to disable the HTML Reporter even if the element does exist.
+
+For example, you can share the same HTML file for both manual testing and CI test runs, and have the CI test run disable the HTML Reporter for improved performance.
+
+```js
+// Set preconfig before loading qunit.js.
+qunit_config_reporters_html = false;
+qunit_config_reporters_perf = false;
+
+// Or, disable at runtime (after qunit.js, but before the first test, i.e. runStart event).
+QUnit.config.reporters.html = false;
+QUnit.config.reporters.perf = false;
+```
+
+Declaratively enable the TAP reporter in a browser environment:
+
+```js
+// Set preconfig before loading qunit.js.
+qunit_config_reporters_tap = true;
+```
+
+## See also
+
+* [Preconfig](./index.md#preconfiguration)
+* [QUnit Reporter API](../callbacks/QUnit.on.md#reporter-api)

docs/cli.md

@@ -127,8 +127,15 @@ Check [`QUnit.config.module`](./api/config/module.md) for more information.
 
 ### `--reporter`
 
-By default, the TAP reporter is used. This allows you to pair QUnit with any [TAP-compatible reporter](https://github.com/sindresorhus/awesome-tap#reporters), by piping the output. For example:
+Built-in reporters:
+
+* `tap`
+* `console`
+* `perf`
+
+Check [`QUnit.config.reporters`](./api/config/reporters.md) for more information.
 
+By default, the TAP reporter is used. This allows you to pair QUnit with many [TAP-based reporters](https://github.com/sindresorhus/awesome-tap#reporters), by piping the output. For example:
 ```sh
 qunit test/ | tap-min
 ```
@@ -144,11 +151,6 @@ qunit --reporter tap
 qunit --reporter qunit-reporter-example
 ```
 
-Built-in reporters:
-
-* `tap`: [TAP compliant](https://testanything.org/) reporter.
-* `console`: Log the JSON object for each reporter event from [`QUnit.on`](./api/callbacks/QUnit.on.md). Use this to explore or debug the Reporter API.
-
 ### `--require`
 
 These modules or scripts will be required before any tests begin running.
@@ -163,11 +165,11 @@ qunit --require ./test/setup.js
 
 ```js
 // test/setup.js
+require('../build/my-custom-reporter.js').init(QUnit);
+
 QUnit.config.noglobals = true;
 QUnit.config.notrycatch = true;
 
-require('../build/my-custom-reporter.js').init(QUnit);
-
 global.MyApp = require('./index');
 ```
 

src/core/browser/browser-runner.js

@@ -75,11 +75,19 @@ export function initBrowser (QUnit, window, document) {
 
   initFixture(QUnit, document);
   initUrlConfig(QUnit);
-  QUnit.reporters.perf.init(QUnit);
-  QUnit.reporters.html.init(QUnit);
+
+  // NOTE:
+  // * It is important to attach error handlers (above) before setting up reporters,
+  //   to ensure reliable reporting of error events.
+  // * Is is important to set up HTML Reporter (if enabled) before calling QUnit.start(),
+  //   as otherwise it will miss the first few or even all synchronous events.
+  //
+  // Priot to QUnit 3.0, the reporter was initialised here, between error handler (above),
+  // and start (below). As of QUnit 3.0, reporters are initialized by doBegin() within
+  // QUnit.start(), which is logically the same place, but decoupled from initBrowser().
 
   function autostart () {
-    // Check as late as possible because if projecst set autostart=false,
+    // Check as late as possible because if projects set autostart=false,
     // they generally do so in their own scripts, after qunit.js.
     if (QUnit.config.autostart) {
       QUnit.start();

src/core/browser/urlparams.js

@@ -18,7 +18,7 @@ export default function initUrlConfig (QUnit) {
       // already set. This prevents internal TypeError from bad urls where keys
       // could otherwise unexpectedly be set to type string or array.
       //
-      // Given that HTML Reporter renders checkboxes based on QUnit.config
+      // Given that HTML Reporter sets checkbox state based on QUnit.config,
       // instead of QUnit.urlParams, this also helps make sure that checkboxes
       // for built-in keys are correctly shown as off if a urlParams value exists
       // but was invalid and discarded by config.js.

src/core/config.js

@@ -47,6 +47,8 @@ const config = {
   // very useful in combination with "Hide passed tests" checked
   reorder: true,
 
+  reporters: {},
+
   // When enabled, all tests must call expect()
   requireExpects: false,
 
@@ -216,6 +218,21 @@ function readFlatPreconfig (obj) {
   readFlatPreconfigString(obj.qunit_config_seed, 'seed');
   readFlatPreconfigStringArray(obj.qunit_config_testid, 'testId');
   readFlatPreconfigNumber(obj.qunit_config_testtimeout, 'testTimeout');
+
+  const reporterKeys = {
+    qunit_config_reporters_console: 'console',
+    qunit_config_reporters_perf: 'perf',
+    qunit_config_reporters_tap: 'tap',
+    qunit_config_reporters_html: 'html'
+  };
+  for (const key in reporterKeys) {
+    const val = obj[key];
+    // Based on readFlatPreconfigBoolean
+    if (typeof val === 'boolean' || (typeof val === 'string' && val !== '')) {
+      const dest = reporterKeys[key];
+      config.reporters[dest] = (val === true || val === 'true' || val === '1');
+    }
+  }
 }
 
 if (process && 'env' in process) {

src/core/core.js

@@ -16,7 +16,7 @@ import { on } from './events.js';
 import onUncaughtException from './on-uncaught-exception.js';
 import diff from './diff.js';
 import version from './version.js';
-import { start } from './start.js';
+import { createStartFunction } from './start.js';
 
 // The "currentModule" object would ideally be defined using the createModule()
 // function. Since it isn't, add the missing suiteReport property to it now that
@@ -61,13 +61,13 @@ const QUnit = {
 
   assert: Assert.prototype,
   module,
-  start,
   test,
 
   // alias other test flavors for easy access
   todo: test.todo,
   skip: test.skip,
   only: test.only
 };
+QUnit.start = createStartFunction(QUnit);
 
 export default QUnit;

src/core/events.js

@@ -46,30 +46,42 @@ export function emit (eventName, data) {
   }
 }
 
+export const prioritySymbol = {};
+
 /**
  * Registers a callback as a listener to the specified event.
  *
  * @public
  * @method on
  * @param {string} eventName
  * @param {Function} callback
+ * @param {Object} [priority] Internal parameter for PerfReporter
  * @return {void}
  */
-export function on (eventName, callback) {
+export function on (eventName, callback, priority = null) {
   if (typeof eventName !== 'string') {
     throw new TypeError('eventName must be a string when registering a listener');
-  } else if (!inArray(eventName, SUPPORTED_EVENTS)) {
+  }
+  if (!inArray(eventName, SUPPORTED_EVENTS)) {
     const events = SUPPORTED_EVENTS.join(', ');
     throw new Error(`"${eventName}" is not a valid event; must be one of: ${events}.`);
-  } else if (typeof callback !== 'function') {
+  }
+  if (typeof callback !== 'function') {
     throw new TypeError('callback must be a function when registering a listener');
   }
+  if (priority && priority !== prioritySymbol) {
+    throw new TypeError('invalid priority parameter');
+  }
 
   const listeners = config._event_listeners[eventName] || (config._event_listeners[eventName] = []);
 
   // Don't register the same callback more than once
   if (!inArray(callback, listeners)) {
-    listeners.push(callback);
+    if (priority === prioritySymbol) {
+      listeners.unshift(callback);
+    } else {
+      listeners.push(callback);
+    }
 
     if (config._event_memory[eventName] !== undefined) {
       callback(config._event_memory[eventName]);

src/core/reporters/HtmlReporter.js

@@ -1,6 +1,7 @@
 import { extend, errorString, escapeText } from '../utilities.js';
 import diff from '../diff.js';
 import dump from '../dump.js';
+import { prioritySymbol } from '../events.js';
 import { window, document, navigator, StringMap } from '../globals.js';
 import { urlParams } from '../urlparams.js';
 import version from '../version.js';
@@ -198,14 +199,16 @@ export default class HtmlReporter {
     // potential internal errors when the HTML Reporter is disabled.
     this.listen = function () {
       this.listen = null;
-      QUnit.begin(this.onBegin.bind(this));
-      QUnit.testStart(this.onTestStart.bind(this));
-      QUnit.log(this.onLog.bind(this));
+      QUnit.begin(this.onBegin.bind(this), prioritySymbol);
+      // Use prioritySignal for testStart() to increase availability
+      // of the HTML API for TESTID elements toward other event listeners.
+      QUnit.testStart(this.onTestStart.bind(this), prioritySymbol);
+      QUnit.log(this.onLog.bind(this), prioritySymbol);
       QUnit.testDone(this.onTestDone.bind(this));
       QUnit.on('runEnd', this.onRunEnd.bind(this));
     };
-    QUnit.on('error', this.onError.bind(this));
-    QUnit.on('runStart', this.onRunStart.bind(this));
+    QUnit.on('error', this.onError.bind(this), prioritySymbol);
+    QUnit.on('runStart', this.onRunStart.bind(this), prioritySymbol);
   }
 
   // Handle "submit" event from "filter" or "moduleFilter" field.

src/core/reporters/PerfReporter.js

@@ -1,14 +1,10 @@
 import { window } from '../globals.js';
+import { prioritySymbol } from '../events.js';
 import Logger from '../logger.js';
 
-// TODO: Consider using globalThis instead of window, so that the reporter
-// works for Node.js as well. As this can add overhead, we should make
-// this opt-in before we enable it for CLI.
-//
-// QUnit 3 will switch from `window` to `globalThis` and then make it
-// no longer an implicit feature of the HTML Reporter, but rather let
-// it be opt-in via `QUnit.config.reporters = ['perf']` or something
-// like that.
+// TODO: Change from window to globalThis in QUnit 3.0, so that the reporter
+// works for Node.js as well. As this can add overhead, we should keep
+// this opt-in on the CLI.
 const nativePerf = (
   window &&
     typeof window.performance !== 'undefined' &&
@@ -39,11 +35,11 @@ export default class PerfReporter {
   constructor (runner, options = {}) {
     this.perf = options.perf || perf;
 
-    runner.on('runStart', this.onRunStart.bind(this));
+    runner.on('runStart', this.onRunStart.bind(this), prioritySymbol);
     runner.on('runEnd', this.onRunEnd.bind(this));
-    runner.on('suiteStart', this.onSuiteStart.bind(this));
+    runner.on('suiteStart', this.onSuiteStart.bind(this), prioritySymbol);
     runner.on('suiteEnd', this.onSuiteEnd.bind(this));
-    runner.on('testStart', this.onTestStart.bind(this));
+    runner.on('testStart', this.onTestStart.bind(this), prioritySymbol);
     runner.on('testEnd', this.onTestEnd.bind(this));
   }