Merge pull request #99 from DataDog/yoann/improve-tests-experience

[DX] Improve tests experience

packages/core/package.json

@@ -23,12 +23,14 @@
         "watch": "tsc -w"
     },
     "dependencies": {
+        "async-retry": "1.3.3",
         "chalk": "2.3.1",
         "glob": "11.0.0",
         "simple-git": "3.25.0",
         "unplugin": "1.11.0"
     },
     "devDependencies": {
+        "@types/async-retry": "1.4.8",
         "@types/chalk": "2.2.0",
         "@types/node": "^18",
         "typescript": "5.4.3"

packages/core/src/helpers.ts

@@ -2,6 +2,11 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2019-Present Datadog, Inc.
 
+import retry from 'async-retry';
+import type { RequestInit } from 'undici-types';
+
+import type { RequestOpts } from './types';
+
 // Format a duration 0h 0m 0s 0ms
 export const formatDuration = (duration: number) => {
     const days = Math.floor(duration / 1000 / 60 / 60 / 24);
@@ -15,3 +20,100 @@ export const formatDuration = (duration: number) => {
         seconds ? `${seconds}s ` : ''
     }${milliseconds}ms`.trim();
 };
+
+export const getResolvedPath = (filepath: string) => {
+    try {
+        return require.resolve(filepath);
+    } catch (e) {
+        return filepath;
+    }
+};
+
+export const ERROR_CODES_NO_RETRY = [400, 403, 413];
+export const NB_RETRIES = 5;
+// Do a retriable fetch.
+export const doRequest = <T>(opts: RequestOpts): Promise<T> => {
+    const { url, method = 'GET', getData, onRetry, type = 'text' } = opts;
+    return retry(
+        async (bail: (e: Error) => void, attempt: number) => {
+            let response: Response;
+            try {
+                const requestInit: RequestInit = {
+                    method,
+                    // This is needed for sending body in NodeJS' Fetch.
+                    // https://github.com/nodejs/node/issues/46221
+                    duplex: 'half',
+                };
+
+                if (typeof getData === 'function') {
+                    const { data, headers } = await getData();
+                    requestInit.body = data;
+                    requestInit.headers = headers;
+                }
+
+                response = await fetch(url, requestInit);
+            } catch (error: any) {
+                // We don't want to retry if there is a non-fetch related error.
+                bail(error);
+                // bail(error) throws so the return is never executed.
+                return {} as T;
+            }
+
+            if (!response.ok) {
+                // Not instantiating the error here, as it will make Jest throw in the tests.
+                const errorMessage = `HTTP ${response.status} ${response.statusText}`;
+                if (ERROR_CODES_NO_RETRY.includes(response.status)) {
+                    bail(new Error(errorMessage));
+                    // bail(error) throws so the return is never executed.
+                    return {} as T;
+                } else {
+                    // Trigger the retry.
+                    throw new Error(errorMessage);
+                }
+            }
+
+            try {
+                let result;
+                // Await it so we catch any parsing error and bail.
+                if (type === 'json') {
+                    result = await response.json();
+                } else {
+                    result = await response.text();
+                }
+
+                return result as T;
+            } catch (error: any) {
+                // We don't want to retry on parsing errors.
+                bail(error);
+                // bail(error) throws so the return is never executed.
+                return {} as T;
+            }
+        },
+        {
+            retries: NB_RETRIES,
+            onRetry,
+        },
+    );
+};
+
+// Truncate a string to a certain length.
+// Placing a [...] placeholder in the middle.
+// "A way too long sentence could be truncated a bit." => "A way too[...]could be truncated a bit."
+export const truncateString = (
+    str: string,
+    maxLength: number = 60,
+    placeholder: string = '[...]',
+) => {
+    if (str.length <= maxLength) {
+        return str;
+    }
+
+    // We want to keep at the very least 4 characters.
+    const stringLength = Math.max(4, maxLength - placeholder.length);
+
+    // We want to keep most of the end of the string, hence the 10 chars top limit for left.
+    const leftStop = Math.min(10, Math.floor(stringLength / 2));
+    const rightStop = stringLength - leftStop;
+
+    return `${str.slice(0, leftStop)}${placeholder}${str.slice(-rightStop)}`;
+};

packages/core/src/plugins/build-report/esbuild.ts

@@ -2,18 +2,18 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2019-Present Datadog, Inc.
 
+import { getResolvedPath } from '@dd/core/helpers';
 import type { Logger } from '@dd/core/log';
 import type { Entry, GlobalContext, Input, Output, PluginOptions } from '@dd/core/types';
 import { glob } from 'glob';
-import path from 'path';
 
-import { cleanName, getResolvedPath, getType } from './helpers';
+import { cleanName, getAbsolutePath, getType } from './helpers';
 
 // Re-index metafile data for easier access.
 const reIndexMeta = <T>(obj: Record<string, T>, cwd: string) =>
     Object.fromEntries(
         Object.entries(obj).map(([key, value]) => {
-            const newKey = path.join(cwd, key);
+            const newKey = getAbsolutePath(cwd, key);
             return [newKey, value];
         }),
     );
@@ -95,9 +95,11 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt
 
                 // Loop through inputs.
                 for (const [filename, input] of Object.entries(result.metafile.inputs)) {
-                    const filepath = path.join(cwd, filename);
+                    const filepath = getAbsolutePath(cwd, filename);
+                    const name = cleanName(context, filename);
+
                     const file: Input = {
-                        name: cleanName(context, filename),
+                        name,
                         filepath,
                         dependents: new Set(),
                         dependencies: new Set(),
@@ -110,12 +112,12 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt
 
                 // Loop through outputs.
                 for (const [filename, output] of Object.entries(result.metafile.outputs)) {
-                    const fullPath = path.join(cwd, filename);
+                    const fullPath = getAbsolutePath(cwd, filename);
                     const cleanedName = cleanName(context, fullPath);
                     // Get inputs of this output.
                     const inputFiles: Input[] = [];
                     for (const inputName of Object.keys(output.inputs)) {
-                        const inputFound = reportInputsIndexed[path.join(cwd, inputName)];
+                        const inputFound = reportInputsIndexed[getAbsolutePath(cwd, inputName)];
                         if (!inputFound) {
                             warn(`Input ${inputName} not found for output ${cleanedName}`);
                             continue;
@@ -127,7 +129,8 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt
                     // When splitting, esbuild creates an empty entryPoint wrapper for the chunk.
                     // It has no inputs, but still relates to its entryPoint.
                     if (output.entryPoint && !inputFiles.length) {
-                        const inputFound = reportInputsIndexed[path.join(cwd, output.entryPoint!)];
+                        const inputFound =
+                            reportInputsIndexed[getAbsolutePath(cwd, output.entryPoint!)];
                         if (!inputFound) {
                             warn(`Input ${output.entryPoint} not found for output ${cleanedName}`);
                             continue;
@@ -156,7 +159,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt
                         continue;
                     }
 
-                    const inputFile = reportInputsIndexed[path.join(cwd, output.entryPoint!)];
+                    const inputFile = reportInputsIndexed[getAbsolutePath(cwd, output.entryPoint!)];
 
                     if (inputFile) {
                         // In the case of "splitting: true", all the files are considered entries to esbuild.
@@ -246,7 +249,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt
                     }
 
                     for (const imported of metaFile.imports) {
-                        const importPath = path.join(cwd, imported.path);
+                        const importPath = getAbsolutePath(cwd, imported.path);
                         // Look for the other inputs.
                         getAllImports<T>(importPath, ref, allImports);
                     }
@@ -296,7 +299,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt
                         if (!isFileSupported(dependency.path)) {
                             continue;
                         }
-                        const dependencyPath = path.join(cwd, dependency.path);
+                        const dependencyPath = getAbsolutePath(cwd, dependency.path);
                         const dependencyFile = references.inputs.report[dependencyPath];
 
                         if (!dependencyFile) {

packages/core/src/plugins/build-report/helpers.ts

@@ -13,7 +13,8 @@ import type {
     Entry,
     Input,
     Output,
-} from '../../types';
+} from '@dd/core/types';
+import path from 'path';
 
 // Will match any last part of a path after a dot or slash and is a word character.
 const EXTENSION_RX = /\.(?!.*(?:\.|\/|\\))(\w{1,})/g;
@@ -234,12 +235,12 @@ export const cleanPath = (filepath: string) => {
     );
 };
 
-export const getResolvedPath = (filepath: string) => {
-    try {
-        return require.resolve(filepath);
-    } catch (e) {
+// Will only prepend the cwd if not already there.
+export const getAbsolutePath = (cwd: string, filepath: string) => {
+    if (filepath.startsWith(cwd)) {
         return filepath;
     }
+    return path.resolve(cwd, filepath);
 };
 
 // Extract a name from a path based on the context (out dir and cwd).

packages/core/src/plugins/build-report/rollup.ts

@@ -2,12 +2,10 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2019-Present Datadog, Inc.
 
-import path from 'path';
-
 import type { Logger } from '../../log';
 import type { Entry, GlobalContext, Input, Output, PluginOptions } from '../../types';
 
-import { cleanName, cleanPath, cleanReport, getType } from './helpers';
+import { cleanName, cleanPath, cleanReport, getAbsolutePath, getType } from './helpers';
 
 export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOptions['rollup'] => {
     const importsReport: Record<
@@ -109,7 +107,7 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti
 
             // Fill in inputs and outputs.
             for (const [filename, asset] of Object.entries(bundle)) {
-                const filepath = path.join(context.bundler.outDir, filename);
+                const filepath = getAbsolutePath(context.bundler.outDir, filename);
                 const size =
                     'code' in asset
                         ? Buffer.byteLength(asset.code, 'utf8')
@@ -240,7 +238,7 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti
                 }
 
                 for (const importName of imports) {
-                    getAllOutputs(path.join(context.bundler.outDir, importName), allOutputs);
+                    getAllOutputs(getAbsolutePath(context.bundler.outDir, importName), allOutputs);
                 }
 
                 return allOutputs;

packages/core/src/plugins/build-report/webpack.ts

@@ -4,9 +4,8 @@
 
 import type { Logger } from '@dd/core/log';
 import type { Entry, GlobalContext, Input, Output, PluginOptions } from '@dd/core/types';
-import path from 'path';
 
-import { cleanName, cleanReport, getType } from './helpers';
+import { cleanName, cleanReport, getAbsolutePath, getType } from './helpers';
 
 export const getWebpackPlugin =
     (context: GlobalContext, PLUGIN_NAME: string, log: Logger): PluginOptions['webpack'] =>
@@ -103,7 +102,7 @@ export const getWebpackPlugin =
                     size: asset.info.size || 0,
                     name: asset.name,
                     inputs: [],
-                    filepath: path.join(context.bundler.outDir, asset.name),
+                    filepath: getAbsolutePath(context.bundler.outDir, asset.name),
                     type: getType(asset.name),
                 };
 
@@ -131,7 +130,7 @@ export const getWebpackPlugin =
                 return module.nameForCondition
                     ? module.nameForCondition
                     : module.name
-                      ? path.join(context.cwd, module.name)
+                      ? getAbsolutePath(context.cwd, module.name)
                       : module.identifier
                         ? module.identifier
                         : 'unknown';
@@ -291,9 +290,9 @@ export const getWebpackPlugin =
                     // Webpack 5 is a list of objects.
                     // Webpack 4 is a list of strings.
                     if (typeof asset === 'string') {
-                        assetPath = path.join(context.bundler.outDir, asset);
+                        assetPath = getAbsolutePath(context.bundler.outDir, asset);
                     } else if (typeof asset.name === 'string') {
-                        assetPath = path.join(context.bundler.outDir, asset.name);
+                        assetPath = getAbsolutePath(context.bundler.outDir, asset.name);
                     }
 
                     if (!assetPath || !reportOutputsIndexed[assetPath]) {
@@ -319,7 +318,7 @@ export const getWebpackPlugin =
                 const file: Entry = {
                     name,
                     filepath: entryFilename
-                        ? path.join(context.bundler.outDir, entryFilename)
+                        ? getAbsolutePath(context.bundler.outDir, entryFilename)
                         : 'unknown',
                     size,
                     inputs: Array.from(new Set(entryInputs)),

packages/core/src/plugins/global-context/index.ts → packages/core/src/plugins/bundler-report/index.ts

@@ -2,11 +2,9 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2019-Present Datadog, Inc.
 
-import type { GlobalContext, Meta, Options, PluginOptions } from '@dd/core/types';
+import type { GlobalContext, Options, PluginOptions } from '@dd/core/types';
 import path from 'path';
 
-// TODO: Add universal config report with list of plugins (names), loaders.
-
 const PLUGIN_NAME = 'datadog-context-plugin';
 
 const rollupPlugin: (context: GlobalContext) => PluginOptions['rollup'] = (context) => ({
@@ -24,29 +22,9 @@ const rollupPlugin: (context: GlobalContext) => PluginOptions['rollup'] = (conte
     },
 });
 
-export const getGlobalContextPlugin = (opts: Options, meta: Meta) => {
-    const cwd = process.cwd();
-    const variant =
-        meta.framework === 'webpack' ? (meta.webpack.compiler['webpack'] ? '5' : '4') : '';
-
-    const globalContext: GlobalContext = {
-        auth: opts.auth,
-        bundler: {
-            name: meta.framework,
-            fullName: `${meta.framework}${variant}`,
-            variant,
-            outDir: cwd,
-        },
-        build: {
-            errors: [],
-            warnings: [],
-        },
-        cwd,
-        start: Date.now(),
-        version: meta.version,
-    };
-
-    const globalContextPlugin: PluginOptions = {
+// TODO: Add universal config report with list of plugins (names), loaders.
+export const getBundlerReportPlugin = (opts: Options, globalContext: GlobalContext) => {
+    const bundlerReportPlugin: PluginOptions = {
         name: PLUGIN_NAME,
         enforce: 'pre',
         esbuild: {
@@ -75,16 +53,7 @@ export const getGlobalContextPlugin = (opts: Options, meta: Meta) => {
         // Vite and Rollup have the same API.
         vite: rollupPlugin(globalContext),
         rollup: rollupPlugin(globalContext),
-        // TODO: Add support and add outputFiles to the context.
-        rspack(compiler) {
-            globalContext.bundler.rawConfig = compiler.options;
-        },
-        farm: {
-            configResolved(config: any) {
-                globalContext.bundler.rawConfig = config;
-            },
-        },
     };
 
-    return { globalContext, globalContextPlugin };
+    return bundlerReportPlugin;
 };