diff --git a/core/computed/metrics/lantern-metric.js b/core/computed/metrics/lantern-metric.js index f431d3523674..869592d97323 100644 --- a/core/computed/metrics/lantern-metric.js +++ b/core/computed/metrics/lantern-metric.js @@ -10,7 +10,7 @@ import {LoadSimulator} from '../load-simulator.js'; import {ProcessedNavigation} from '../processed-navigation.js'; import {PageDependencyGraph} from '../page-dependency-graph.js'; import {TraceEngineResult} from '../trace-engine-result.js'; -import {createProcessedNavigation} from '../../lib/lantern/lantern.js'; +import {createProcessedNavigation} from '../../lib/lantern/trace-engine-computation-data.js'; /** * @param {LH.Artifacts.MetricComputationDataInput} data diff --git a/core/computed/page-dependency-graph.js b/core/computed/page-dependency-graph.js index 04072f00b376..0bb46c373cbf 100644 --- a/core/computed/page-dependency-graph.js +++ b/core/computed/page-dependency-graph.js @@ -10,6 +10,7 @@ import {NetworkRequest} from '../lib/network-request.js'; import {ProcessedTrace} from './processed-trace.js'; import {NetworkRecords} from './network-records.js'; import {TraceEngineResult} from './trace-engine-result.js'; +import * as TraceEngineComputationData from '../lib/lantern/trace-engine-computation-data.js'; /** @typedef {import('../lib/lantern/base-node.js').Node} Node */ @@ -28,8 +29,9 @@ class PageDependencyGraph { if (data.fromTrace) { const traceEngineResult = await TraceEngineResult.request({trace}, context); - const {graph} = await LanternPageDependencyGraph.createGraphFromTrace( - trace, traceEngineResult, URL); + const requests = TraceEngineComputationData.createNetworkRequests(trace, traceEngineResult); + const graph = + TraceEngineComputationData.createGraph(requests, trace, traceEngineResult, URL); return graph; } diff --git a/core/lib/lantern/lantern.js b/core/lib/lantern/lantern.js index c4004be6324a..63312ed5cdec 100644 --- a/core/lib/lantern/lantern.js +++ b/core/lib/lantern/lantern.js @@ -4,11 +4,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import * as Lantern from './types/lantern.js'; - -/** @typedef {import('@paulirish/trace_engine/models/trace/handlers/PageLoadMetricsHandler.js').MetricName} MetricName */ -/** @typedef {import('@paulirish/trace_engine/models/trace/handlers/PageLoadMetricsHandler.js').MetricScore} MetricScore */ - /** @type {LH.Util.SelfMap} */ const NetworkRequestTypes = { XHR: 'XHR', @@ -31,48 +26,6 @@ const NetworkRequestTypes = { Prefetch: 'Prefetch', }; -/** - * @param {LH.Artifacts.TraceEngineResult} traceEngineResult - * @return {Lantern.Simulation.ProcessedNavigation} - */ -function createProcessedNavigation(traceEngineResult) { - const Meta = traceEngineResult.data.Meta; - const frameId = Meta.mainFrameId; - const scoresByNav = traceEngineResult.data.PageLoadMetrics.metricScoresByFrameId.get(frameId); - if (!scoresByNav) { - throw new Error('missing metric scores for main frame'); - } - - const lastNavigationId = Meta.mainFrameNavigations.at(-1)?.args.data?.navigationId; - const scores = lastNavigationId && scoresByNav.get(lastNavigationId); - if (!scores) { - throw new Error('missing metric scores for specified navigation'); - } - - /** @param {MetricName} metric */ - const getTimestampOrUndefined = metric => { - const metricScore = scores.get(metric); - if (!metricScore?.event) return; - return metricScore.event.ts; - }; - /** @param {MetricName} metric */ - const getTimestamp = metric => { - const metricScore = scores.get(metric); - if (!metricScore?.event) throw new Error(`missing metric: ${metric}`); - return metricScore.event.ts; - }; - // TODO: should use `MetricName.LCP`, but it is a const enum. - const FCP = /** @type {MetricName} */('FCP'); - const LCP = /** @type {MetricName} */('LCP'); - return { - timestamps: { - firstContentfulPaint: getTimestamp(FCP), - largestContentfulPaint: getTimestampOrUndefined(LCP), - }, - }; -} - export { NetworkRequestTypes, - createProcessedNavigation, }; diff --git a/core/lib/lantern/page-dependency-graph.js b/core/lib/lantern/page-dependency-graph.js index e745dfe3111c..c7359c84257a 100644 --- a/core/lib/lantern/page-dependency-graph.js +++ b/core/lib/lantern/page-dependency-graph.js @@ -10,7 +10,6 @@ import {NetworkNode} from './network-node.js'; import {CPUNode} from './cpu-node.js'; import {TraceProcessor} from '../tracehouse/trace-processor.js'; import {NetworkAnalyzer} from './simulator/network-analyzer.js'; -import {RESOURCE_TYPES} from '../network-request.js'; /** @typedef {import('./base-node.js').Node} Node */ /** @typedef {Omit} URLArtifact */ @@ -543,456 +542,6 @@ class PageDependencyGraph { } /** - * @param {Lantern.NetworkRequest} request The request to find the initiator of - * @param {Map} requestsByURL - * @return {Lantern.NetworkRequest|null} - */ - static chooseInitiatorRequest(request, requestsByURL) { - if (request.redirectSource) { - return request.redirectSource; - } - - const initiatorURL = PageDependencyGraph.getNetworkInitiators(request)[0]; - let candidates = requestsByURL.get(initiatorURL) || []; - // The (valid) initiator must come before the initiated request. - candidates = candidates.filter(c => { - return c.responseHeadersEndTime <= request.rendererStartTime && - c.finished && !c.failed; - }); - if (candidates.length > 1) { - // Disambiguate based on prefetch. Prefetch requests have type 'Other' and cannot - // initiate requests, so we drop them here. - const nonPrefetchCandidates = candidates.filter( - cand => cand.resourceType !== RESOURCE_TYPES.Other); - if (nonPrefetchCandidates.length) { - candidates = nonPrefetchCandidates; - } - } - if (candidates.length > 1) { - // Disambiguate based on frame. It's likely that the initiator comes from the same frame. - const sameFrameCandidates = candidates.filter(cand => cand.frameId === request.frameId); - if (sameFrameCandidates.length) { - candidates = sameFrameCandidates; - } - } - if (candidates.length > 1 && request.initiator.type === 'parser') { - // Filter to just Documents when initiator type is parser. - const documentCandidates = candidates.filter(cand => - cand.resourceType === RESOURCE_TYPES.Document); - if (documentCandidates.length) { - candidates = documentCandidates; - } - } - if (candidates.length > 1) { - // If all real loads came from successful preloads (url preloaded and - // loads came from the cache), filter to link rel=preload request(s). - const linkPreloadCandidates = candidates.filter(c => c.isLinkPreload); - if (linkPreloadCandidates.length) { - const nonPreloadCandidates = candidates.filter(c => !c.isLinkPreload); - const allPreloaded = nonPreloadCandidates.every(c => c.fromDiskCache || c.fromMemoryCache); - if (nonPreloadCandidates.length && allPreloaded) { - candidates = linkPreloadCandidates; - } - } - } - - // Only return an initiator if the result is unambiguous. - return candidates.length === 1 ? candidates[0] : null; - } - - /** - * Returns a map of `pid` -> `tid[]`. - * @param {LH.Trace} trace - * @return {Map} - */ - static _findWorkerThreads(trace) { - // TODO: WorkersHandler in TraceEngine needs to be updated to also include `pid` (only had `tid`). - const workerThreads = new Map(); - const workerCreationEvents = ['ServiceWorker thread', 'DedicatedWorker thread']; - - for (const event of trace.traceEvents) { - if (event.name !== 'thread_name' || !event.args.name) { - continue; - } - if (!workerCreationEvents.includes(event.args.name)) { - continue; - } - - const tids = workerThreads.get(event.pid); - if (tids) { - tids.push(event.tid); - } else { - workerThreads.set(event.pid, [event.tid]); - } - } - - return workerThreads; - } - - /** - * @param {URL|string} url - */ - static _createParsedUrl(url) { - if (typeof url === 'string') { - url = new URL(url); - } - return { - scheme: url.protocol.split(':')[0], - // Intentional, DevTools uses different terminology - host: url.hostname, - securityOrigin: url.origin, - }; - } - - /** - * @param {LH.Artifacts.TraceEngineResult} traceEngineResult - * @param {Map} workerThreads - * @param {import('@paulirish/trace_engine/models/trace/types/TraceEvents.js').SyntheticNetworkRequest} request - * @return {Lantern.NetworkRequest=} - */ - static _createLanternRequest(traceEngineResult, workerThreads, request) { - if (request.args.data.connectionId === undefined || - request.args.data.connectionReused === undefined) { - throw new Error('Trace is too old'); - } - - let url; - try { - url = new URL(request.args.data.url); - } catch (e) { - return; - } - - const timing = request.args.data.timing ? { - // These two timings are not included in the trace. - workerFetchStart: -1, - workerRespondWithSettled: -1, - ...request.args.data.timing, - } : undefined; - - const networkRequestTime = timing ? - timing.requestTime * 1000 : - request.args.data.syntheticData.downloadStart / 1000; - - let fromWorker = false; - const tids = workerThreads.get(request.pid); - if (tids?.includes(request.tid)) { - fromWorker = true; - } - - // TraceEngine collects worker thread ids in a different manner than `workerThreads` does. - // AFAIK these should be equivalent, but in case they are not let's also check this for now. - if (traceEngineResult.data.Workers.workerIdByThread.has(request.tid)) { - fromWorker = true; - } - - // `initiator` in the trace does not contain the stack trace for JS-initiated - // requests. Instead, that is stored in the `stackTrace` property of the SyntheticNetworkRequest. - // There are some minor differences in the fields, accounted for here. - // Most importantly, there seems to be fewer frames in the trace than the equivalent - // events over the CDP. This results in less accuracy in determining the initiator request, - // which means less edges in the graph, which mean worse results. - // TODO: Should fix in Chromium. - /** @type {Lantern.NetworkRequest['initiator']} */ - const initiator = request.args.data.initiator ?? {type: 'other'}; - if (request.args.data.stackTrace) { - const callFrames = request.args.data.stackTrace.map(f => { - return { - scriptId: String(f.scriptId), - url: f.url, - lineNumber: f.lineNumber - 1, - columnNumber: f.columnNumber - 1, - functionName: f.functionName, - }; - }); - initiator.stack = {callFrames}; - } - - let resourceType = request.args.data.resourceType; - if (request.args.data.initiator?.fetchType === 'xmlhttprequest') { - // @ts-expect-error yes XHR is a valid ResourceType. TypeScript const enums are so unhelpful. - resourceType = 'XHR'; - } else if (request.args.data.initiator?.fetchType === 'fetch') { - // @ts-expect-error yes Fetch is a valid ResourceType. TypeScript const enums are so unhelpful. - resourceType = 'Fetch'; - } - - // TODO: set decodedBodyLength for data urls in Trace Engine. - let resourceSize = request.args.data.decodedBodyLength ?? 0; - if (url.protocol === 'data:' && resourceSize === 0) { - const needle = 'base64,'; - const index = url.pathname.indexOf(needle); - if (index !== -1) { - resourceSize = atob(url.pathname.substring(index + needle.length)).length; - } - } - - return { - rawRequest: request, - requestId: request.args.data.requestId, - connectionId: request.args.data.connectionId, - connectionReused: request.args.data.connectionReused, - url: request.args.data.url, - protocol: request.args.data.protocol, - parsedURL: this._createParsedUrl(url), - documentURL: request.args.data.requestingFrameUrl, - rendererStartTime: request.ts / 1000, - networkRequestTime, - responseHeadersEndTime: request.args.data.syntheticData.downloadStart / 1000, - networkEndTime: request.args.data.syntheticData.finishTime / 1000, - transferSize: request.args.data.encodedDataLength, - resourceSize, - fromDiskCache: request.args.data.syntheticData.isDiskCached, - fromMemoryCache: request.args.data.syntheticData.isMemoryCached, - isLinkPreload: request.args.data.isLinkPreload, - finished: request.args.data.finished, - failed: request.args.data.failed, - statusCode: request.args.data.statusCode, - initiator, - timing, - resourceType, - mimeType: request.args.data.mimeType, - priority: request.args.data.priority, - frameId: request.args.data.frame, - fromWorker, - // Set later. - redirects: undefined, - redirectSource: undefined, - redirectDestination: undefined, - initiatorRequest: undefined, - }; - } - - /** - * - * @param {Lantern.NetworkRequest[]} lanternRequests - */ - static _linkInitiators(lanternRequests) { - /** @type {Map} */ - const requestsByURL = new Map(); - for (const request of lanternRequests) { - const requests = requestsByURL.get(request.url) || []; - requests.push(request); - requestsByURL.set(request.url, requests); - } - - for (const request of lanternRequests) { - const initiatorRequest = PageDependencyGraph.chooseInitiatorRequest(request, requestsByURL); - if (initiatorRequest) { - request.initiatorRequest = initiatorRequest; - } - } - } - - /** - * Sorts and filters trace events by timestamp and respecting the nesting structure inherent to - * parent/child event relationships. - * - * @param {LH.TraceEvent[]} traceEvents - * @param {(e: LH.TraceEvent) => boolean} filter - */ - static _filteredTraceSort(traceEvents, filter) { - // create an array of the indices that we want to keep - const indices = []; - for (let srcIndex = 0; srcIndex < traceEvents.length; srcIndex++) { - if (filter(traceEvents[srcIndex])) { - indices.push(srcIndex); - } - } - - // Sort by ascending timestamp first. - indices.sort((indexA, indexB) => traceEvents[indexA].ts - traceEvents[indexB].ts); - - // Now we find groups with equal timestamps and order them by their nesting structure. - for (let i = 0; i < indices.length - 1; i++) { - const ts = traceEvents[indices[i]].ts; - const tsGroupIndices = [i]; - for (let j = i + 1; j < indices.length; j++) { - if (traceEvents[indices[j]].ts !== ts) break; - tsGroupIndices.push(j); - } - - // We didn't find any other events with the same timestamp, just keep going. - if (tsGroupIndices.length === 1) continue; - - // Sort the group by other criteria and replace our index array with it. - const finalIndexOrder = TraceProcessor._sortTimestampEventGroup( - tsGroupIndices, - indices, - i, - traceEvents - ); - indices.splice(i, finalIndexOrder.length, ...finalIndexOrder); - // We just sorted this set of identical timestamps, so skip over the rest of the group. - // -1 because we already have i++. - i += tsGroupIndices.length - 1; - } - - // create a new array using the target indices from previous sort step - const sorted = []; - for (let i = 0; i < indices.length; i++) { - sorted.push(traceEvents[indices[i]]); - } - - return sorted; - } - - /** - * @param {LH.Trace} trace - * @param {LH.Artifacts.TraceEngineResult} traceEngineResult - * @return {LH.TraceEvent[]} - */ - static _collectMainThreadEvents(trace, traceEngineResult) { - const Meta = traceEngineResult.data.Meta; - const mainFramePids = Meta.mainFrameNavigations.length - ? new Set(Meta.mainFrameNavigations.map(nav => nav.pid)) - : Meta.topLevelRendererIds; - - const rendererPidToTid = new Map(); - for (const pid of mainFramePids) { - const threads = Meta.threadsInProcess.get(pid) ?? []; - - let found = false; - for (const [tid, thread] of threads) { - if (thread.args.name === 'CrRendererMain') { - rendererPidToTid.set(pid, tid); - found = true; - break; - } - } - - if (found) continue; - - // `CrRendererMain` can be missing if chrome is launched with the `--single-process` flag. - // In this case, page tasks will be run in the browser thread. - for (const [tid, thread] of threads) { - if (thread.args.name === 'CrBrowserMain') { - rendererPidToTid.set(pid, tid); - found = true; - break; - } - } - } - - return this._filteredTraceSort(trace.traceEvents, e => rendererPidToTid.get(e.pid) === e.tid); - } - - /** - * @param {LH.Trace} trace - * @param {LH.Artifacts.TraceEngineResult} traceEngineResult - * @param {LH.Artifacts.URL=} URL - */ - static async createGraphFromTrace(trace, traceEngineResult, URL) { - const mainThreadEvents = this._collectMainThreadEvents(trace, traceEngineResult); - const workerThreads = this._findWorkerThreads(trace); - - /** @type {Lantern.NetworkRequest[]} */ - const lanternRequests = []; - for (const request of traceEngineResult.data.NetworkRequests.byTime) { - const lanternRequest = this._createLanternRequest(traceEngineResult, workerThreads, request); - if (lanternRequest) { - lanternRequests.push(lanternRequest); - } - } - - // TraceEngine consolidates all redirects into a single request object, but lantern needs - // an entry for each redirected request. - for (const request of [...lanternRequests]) { - if (!request.rawRequest) continue; - - const redirects = request.rawRequest.args.data.redirects; - if (!redirects.length) continue; - - const requestChain = []; - for (const redirect of redirects) { - const redirectedRequest = structuredClone(request); - - redirectedRequest.networkRequestTime = redirect.ts / 1000; - redirectedRequest.rendererStartTime = redirectedRequest.networkRequestTime; - - redirectedRequest.networkEndTime = (redirect.ts + redirect.dur) / 1000; - redirectedRequest.responseHeadersEndTime = redirectedRequest.networkEndTime; - - redirectedRequest.timing = { - requestTime: redirectedRequest.networkRequestTime / 1000, - receiveHeadersStart: redirectedRequest.responseHeadersEndTime, - receiveHeadersEnd: redirectedRequest.responseHeadersEndTime, - proxyStart: -1, - proxyEnd: -1, - dnsStart: -1, - dnsEnd: -1, - connectStart: -1, - connectEnd: -1, - sslStart: -1, - sslEnd: -1, - sendStart: -1, - sendEnd: -1, - workerStart: -1, - workerReady: -1, - workerFetchStart: -1, - workerRespondWithSettled: -1, - pushStart: -1, - pushEnd: -1, - }; - - redirectedRequest.url = redirect.url; - redirectedRequest.parsedURL = this._createParsedUrl(redirect.url); - // TODO: TraceEngine is not retaining the actual status code. - redirectedRequest.statusCode = 302; - redirectedRequest.resourceType = undefined; - // TODO: TraceEngine is not retaining transfer size of redirected request. - redirectedRequest.transferSize = 400; - requestChain.push(redirectedRequest); - lanternRequests.push(redirectedRequest); - } - requestChain.push(request); - - for (let i = 0; i < requestChain.length; i++) { - const request = requestChain[i]; - if (i > 0) { - request.redirectSource = requestChain[i - 1]; - request.redirects = requestChain.slice(0, i); - } - if (i !== requestChain.length - 1) { - request.redirectDestination = requestChain[i + 1]; - } - } - - // Apply the `:redirect` requestId convention: only redirects[0].requestId is the actual - // requestId, all the rest have n occurences of `:redirect` as a suffix. - for (let i = 1; i < requestChain.length; i++) { - requestChain[i].requestId = `${requestChain[i - 1].requestId}:redirect`; - } - } - - this._linkInitiators(lanternRequests); - - // This would already be sorted by rendererStartTime, if not for the redirect unwrapping done - // above. - lanternRequests.sort((a, b) => a.rendererStartTime - b.rendererStartTime); - - // URL defines the initial request that the Lantern graph starts at (the root node) and the - // main document request. These are equal if there are no redirects. - if (!URL) { - URL = { - requestedUrl: lanternRequests[0].url, - mainDocumentUrl: '', - finalDisplayedUrl: '', - }; - - let request = lanternRequests[0]; - while (request.redirectDestination) { - request = request.redirectDestination; - } - URL.mainDocumentUrl = request.url; - } - - const graph = PageDependencyGraph.createGraph(mainThreadEvents, lanternRequests, URL); - return {graph, requests: lanternRequests}; - } - - /** - * * @param {Node} rootNode */ static printGraph(rootNode, widthInCharacters = 100) { diff --git a/core/lib/lantern/trace-engine-computation-data.js b/core/lib/lantern/trace-engine-computation-data.js new file mode 100644 index 000000000000..af35fbf11c13 --- /dev/null +++ b/core/lib/lantern/trace-engine-computation-data.js @@ -0,0 +1,516 @@ +/** + * @license + * Copyright 2024 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as Lantern from './types/lantern.js'; +import {PageDependencyGraph} from './page-dependency-graph.js'; +import {RESOURCE_TYPES} from '../network-request.js'; +import {TraceProcessor} from '../tracehouse/trace-processor.js'; + +/** @typedef {import('@paulirish/trace_engine/models/trace/handlers/PageLoadMetricsHandler.js').MetricName} MetricName */ +/** @typedef {import('@paulirish/trace_engine/models/trace/handlers/PageLoadMetricsHandler.js').MetricScore} MetricScore */ + +/** + * @param {LH.Artifacts.TraceEngineResult} traceEngineResult + * @return {Lantern.Simulation.ProcessedNavigation} + */ +function createProcessedNavigation(traceEngineResult) { + const Meta = traceEngineResult.data.Meta; + const frameId = Meta.mainFrameId; + const scoresByNav = traceEngineResult.data.PageLoadMetrics.metricScoresByFrameId.get(frameId); + if (!scoresByNav) { + throw new Error('missing metric scores for main frame'); + } + + const lastNavigationId = Meta.mainFrameNavigations.at(-1)?.args.data?.navigationId; + const scores = lastNavigationId && scoresByNav.get(lastNavigationId); + if (!scores) { + throw new Error('missing metric scores for specified navigation'); + } + + /** @param {MetricName} metric */ + const getTimestampOrUndefined = metric => { + const metricScore = scores.get(metric); + if (!metricScore?.event) return; + return metricScore.event.ts; + }; + /** @param {MetricName} metric */ + const getTimestamp = metric => { + const metricScore = scores.get(metric); + if (!metricScore?.event) throw new Error(`missing metric: ${metric}`); + return metricScore.event.ts; + }; + // TODO: should use `MetricName.LCP`, but it is a const enum. + const FCP = /** @type {MetricName} */('FCP'); + const LCP = /** @type {MetricName} */('LCP'); + return { + timestamps: { + firstContentfulPaint: getTimestamp(FCP), + largestContentfulPaint: getTimestampOrUndefined(LCP), + }, + }; +} + +/** + * @param {URL|string} url + */ +function createParsedUrl(url) { + if (typeof url === 'string') { + url = new URL(url); + } + return { + scheme: url.protocol.split(':')[0], + // Intentional, DevTools uses different terminology + host: url.hostname, + securityOrigin: url.origin, + }; +} + +/** + * Returns a map of `pid` -> `tid[]`. + * @param {LH.Trace} trace + * @return {Map} + */ +function findWorkerThreads(trace) { + // TODO: WorkersHandler in TraceEngine needs to be updated to also include `pid` (only had `tid`). + const workerThreads = new Map(); + const workerCreationEvents = ['ServiceWorker thread', 'DedicatedWorker thread']; + + for (const event of trace.traceEvents) { + if (event.name !== 'thread_name' || !event.args.name) { + continue; + } + if (!workerCreationEvents.includes(event.args.name)) { + continue; + } + + const tids = workerThreads.get(event.pid); + if (tids) { + tids.push(event.tid); + } else { + workerThreads.set(event.pid, [event.tid]); + } + } + + return workerThreads; +} + +/** + * @param {LH.Artifacts.TraceEngineResult} traceEngineResult + * @param {Map} workerThreads + * @param {import('@paulirish/trace_engine/models/trace/types/TraceEvents.js').SyntheticNetworkRequest} request + * @return {Lantern.NetworkRequest=} + */ +function createLanternRequest(traceEngineResult, workerThreads, request) { + if (request.args.data.connectionId === undefined || + request.args.data.connectionReused === undefined) { + throw new Error('Trace is too old'); + } + + let url; + try { + url = new URL(request.args.data.url); + } catch (e) { + return; + } + + const timing = request.args.data.timing ? { + // These two timings are not included in the trace. + workerFetchStart: -1, + workerRespondWithSettled: -1, + ...request.args.data.timing, + } : undefined; + + const networkRequestTime = timing ? + timing.requestTime * 1000 : + request.args.data.syntheticData.downloadStart / 1000; + + let fromWorker = false; + const tids = workerThreads.get(request.pid); + if (tids?.includes(request.tid)) { + fromWorker = true; + } + + // TraceEngine collects worker thread ids in a different manner than `workerThreads` does. + // AFAIK these should be equivalent, but in case they are not let's also check this for now. + if (traceEngineResult.data.Workers.workerIdByThread.has(request.tid)) { + fromWorker = true; + } + + // `initiator` in the trace does not contain the stack trace for JS-initiated + // requests. Instead, that is stored in the `stackTrace` property of the SyntheticNetworkRequest. + // There are some minor differences in the fields, accounted for here. + // Most importantly, there seems to be fewer frames in the trace than the equivalent + // events over the CDP. This results in less accuracy in determining the initiator request, + // which means less edges in the graph, which mean worse results. + // TODO: Should fix in Chromium. + /** @type {Lantern.NetworkRequest['initiator']} */ + const initiator = request.args.data.initiator ?? {type: 'other'}; + if (request.args.data.stackTrace) { + const callFrames = request.args.data.stackTrace.map(f => { + return { + scriptId: String(f.scriptId), + url: f.url, + lineNumber: f.lineNumber - 1, + columnNumber: f.columnNumber - 1, + functionName: f.functionName, + }; + }); + initiator.stack = {callFrames}; + } + + let resourceType = request.args.data.resourceType; + if (request.args.data.initiator?.fetchType === 'xmlhttprequest') { + // @ts-expect-error yes XHR is a valid ResourceType. TypeScript const enums are so unhelpful. + resourceType = 'XHR'; + } else if (request.args.data.initiator?.fetchType === 'fetch') { + // @ts-expect-error yes Fetch is a valid ResourceType. TypeScript const enums are so unhelpful. + resourceType = 'Fetch'; + } + + // TODO: set decodedBodyLength for data urls in Trace Engine. + let resourceSize = request.args.data.decodedBodyLength ?? 0; + if (url.protocol === 'data:' && resourceSize === 0) { + const needle = 'base64,'; + const index = url.pathname.indexOf(needle); + if (index !== -1) { + resourceSize = atob(url.pathname.substring(index + needle.length)).length; + } + } + + return { + rawRequest: request, + requestId: request.args.data.requestId, + connectionId: request.args.data.connectionId, + connectionReused: request.args.data.connectionReused, + url: request.args.data.url, + protocol: request.args.data.protocol, + parsedURL: createParsedUrl(url), + documentURL: request.args.data.requestingFrameUrl, + rendererStartTime: request.ts / 1000, + networkRequestTime, + responseHeadersEndTime: request.args.data.syntheticData.downloadStart / 1000, + networkEndTime: request.args.data.syntheticData.finishTime / 1000, + transferSize: request.args.data.encodedDataLength, + resourceSize, + fromDiskCache: request.args.data.syntheticData.isDiskCached, + fromMemoryCache: request.args.data.syntheticData.isMemoryCached, + isLinkPreload: request.args.data.isLinkPreload, + finished: request.args.data.finished, + failed: request.args.data.failed, + statusCode: request.args.data.statusCode, + initiator, + timing, + resourceType, + mimeType: request.args.data.mimeType, + priority: request.args.data.priority, + frameId: request.args.data.frame, + fromWorker, + // Set later. + redirects: undefined, + redirectSource: undefined, + redirectDestination: undefined, + initiatorRequest: undefined, + }; +} + +/** + * @param {Lantern.NetworkRequest} request The request to find the initiator of + * @param {Map} requestsByURL + * @return {Lantern.NetworkRequest|null} + */ +function chooseInitiatorRequest(request, requestsByURL) { + if (request.redirectSource) { + return request.redirectSource; + } + + const initiatorURL = PageDependencyGraph.getNetworkInitiators(request)[0]; + let candidates = requestsByURL.get(initiatorURL) || []; + // The (valid) initiator must come before the initiated request. + candidates = candidates.filter(c => { + return c.responseHeadersEndTime <= request.rendererStartTime && + c.finished && !c.failed; + }); + if (candidates.length > 1) { + // Disambiguate based on prefetch. Prefetch requests have type 'Other' and cannot + // initiate requests, so we drop them here. + const nonPrefetchCandidates = candidates.filter( + cand => cand.resourceType !== RESOURCE_TYPES.Other); + if (nonPrefetchCandidates.length) { + candidates = nonPrefetchCandidates; + } + } + if (candidates.length > 1) { + // Disambiguate based on frame. It's likely that the initiator comes from the same frame. + const sameFrameCandidates = candidates.filter(cand => cand.frameId === request.frameId); + if (sameFrameCandidates.length) { + candidates = sameFrameCandidates; + } + } + if (candidates.length > 1 && request.initiator.type === 'parser') { + // Filter to just Documents when initiator type is parser. + const documentCandidates = candidates.filter(cand => + cand.resourceType === RESOURCE_TYPES.Document); + if (documentCandidates.length) { + candidates = documentCandidates; + } + } + if (candidates.length > 1) { + // If all real loads came from successful preloads (url preloaded and + // loads came from the cache), filter to link rel=preload request(s). + const linkPreloadCandidates = candidates.filter(c => c.isLinkPreload); + if (linkPreloadCandidates.length) { + const nonPreloadCandidates = candidates.filter(c => !c.isLinkPreload); + const allPreloaded = nonPreloadCandidates.every(c => c.fromDiskCache || c.fromMemoryCache); + if (nonPreloadCandidates.length && allPreloaded) { + candidates = linkPreloadCandidates; + } + } + } + + // Only return an initiator if the result is unambiguous. + return candidates.length === 1 ? candidates[0] : null; +} + +/** + * @param {Lantern.NetworkRequest[]} lanternRequests + */ +function linkInitiators(lanternRequests) { +/** @type {Map} */ + const requestsByURL = new Map(); + for (const request of lanternRequests) { + const requests = requestsByURL.get(request.url) || []; + requests.push(request); + requestsByURL.set(request.url, requests); + } + + for (const request of lanternRequests) { + const initiatorRequest = chooseInitiatorRequest(request, requestsByURL); + if (initiatorRequest) { + request.initiatorRequest = initiatorRequest; + } + } +} + +/** + * @param {LH.Trace} trace + * @param {LH.Artifacts.TraceEngineResult} traceEngineResult + * @return {Lantern.NetworkRequest[]} + */ +function createNetworkRequests(trace, traceEngineResult) { + const workerThreads = findWorkerThreads(trace); + + /** @type {Lantern.NetworkRequest[]} */ + const lanternRequests = []; + for (const request of traceEngineResult.data.NetworkRequests.byTime) { + const lanternRequest = createLanternRequest(traceEngineResult, workerThreads, request); + if (lanternRequest) { + lanternRequests.push(lanternRequest); + } + } + + // TraceEngine consolidates all redirects into a single request object, but lantern needs + // an entry for each redirected request. + for (const request of [...lanternRequests]) { + if (!request.rawRequest) continue; + + const redirects = request.rawRequest.args.data.redirects; + if (!redirects.length) continue; + + const requestChain = []; + for (const redirect of redirects) { + const redirectedRequest = structuredClone(request); + + redirectedRequest.networkRequestTime = redirect.ts / 1000; + redirectedRequest.rendererStartTime = redirectedRequest.networkRequestTime; + + redirectedRequest.networkEndTime = (redirect.ts + redirect.dur) / 1000; + redirectedRequest.responseHeadersEndTime = redirectedRequest.networkEndTime; + + redirectedRequest.timing = { + requestTime: redirectedRequest.networkRequestTime / 1000, + receiveHeadersStart: redirectedRequest.responseHeadersEndTime, + receiveHeadersEnd: redirectedRequest.responseHeadersEndTime, + proxyStart: -1, + proxyEnd: -1, + dnsStart: -1, + dnsEnd: -1, + connectStart: -1, + connectEnd: -1, + sslStart: -1, + sslEnd: -1, + sendStart: -1, + sendEnd: -1, + workerStart: -1, + workerReady: -1, + workerFetchStart: -1, + workerRespondWithSettled: -1, + pushStart: -1, + pushEnd: -1, + }; + + redirectedRequest.url = redirect.url; + redirectedRequest.parsedURL = createParsedUrl(redirect.url); + // TODO: TraceEngine is not retaining the actual status code. + redirectedRequest.statusCode = 302; + redirectedRequest.resourceType = undefined; + // TODO: TraceEngine is not retaining transfer size of redirected request. + redirectedRequest.transferSize = 400; + requestChain.push(redirectedRequest); + lanternRequests.push(redirectedRequest); + } + requestChain.push(request); + + for (let i = 0; i < requestChain.length; i++) { + const request = requestChain[i]; + if (i > 0) { + request.redirectSource = requestChain[i - 1]; + request.redirects = requestChain.slice(0, i); + } + if (i !== requestChain.length - 1) { + request.redirectDestination = requestChain[i + 1]; + } + } + + // Apply the `:redirect` requestId convention: only redirects[0].requestId is the actual + // requestId, all the rest have n occurences of `:redirect` as a suffix. + for (let i = 1; i < requestChain.length; i++) { + requestChain[i].requestId = `${requestChain[i - 1].requestId}:redirect`; + } + } + + linkInitiators(lanternRequests); + + // This would already be sorted by rendererStartTime, if not for the redirect unwrapping done + // above. + return lanternRequests.sort((a, b) => a.rendererStartTime - b.rendererStartTime); +} + +/** + * Sorts and filters trace events by timestamp and respecting the nesting structure inherent to + * parent/child event relationships. + * + * @param {LH.TraceEvent[]} traceEvents + * @param {(e: LH.TraceEvent) => boolean} filter + */ +function filteredTraceSort(traceEvents, filter) { + // create an array of the indices that we want to keep + const indices = []; + for (let srcIndex = 0; srcIndex < traceEvents.length; srcIndex++) { + if (filter(traceEvents[srcIndex])) { + indices.push(srcIndex); + } + } + + // Sort by ascending timestamp first. + indices.sort((indexA, indexB) => traceEvents[indexA].ts - traceEvents[indexB].ts); + + // Now we find groups with equal timestamps and order them by their nesting structure. + for (let i = 0; i < indices.length - 1; i++) { + const ts = traceEvents[indices[i]].ts; + const tsGroupIndices = [i]; + for (let j = i + 1; j < indices.length; j++) { + if (traceEvents[indices[j]].ts !== ts) break; + tsGroupIndices.push(j); + } + + // We didn't find any other events with the same timestamp, just keep going. + if (tsGroupIndices.length === 1) continue; + + // Sort the group by other criteria and replace our index array with it. + const finalIndexOrder = TraceProcessor._sortTimestampEventGroup( + tsGroupIndices, + indices, + i, + traceEvents + ); + indices.splice(i, finalIndexOrder.length, ...finalIndexOrder); + // We just sorted this set of identical timestamps, so skip over the rest of the group. + // -1 because we already have i++. + i += tsGroupIndices.length - 1; + } + + // create a new array using the target indices from previous sort step + const sorted = []; + for (let i = 0; i < indices.length; i++) { + sorted.push(traceEvents[indices[i]]); + } + + return sorted; +} + +/** + * @param {LH.Trace} trace + * @param {LH.Artifacts.TraceEngineResult} traceEngineResult + * @return {LH.TraceEvent[]} + */ +function collectMainThreadEvents(trace, traceEngineResult) { + const Meta = traceEngineResult.data.Meta; + const mainFramePids = Meta.mainFrameNavigations.length + ? new Set(Meta.mainFrameNavigations.map(nav => nav.pid)) + : Meta.topLevelRendererIds; + + const rendererPidToTid = new Map(); + for (const pid of mainFramePids) { + const threads = Meta.threadsInProcess.get(pid) ?? []; + + let found = false; + for (const [tid, thread] of threads) { + if (thread.args.name === 'CrRendererMain') { + rendererPidToTid.set(pid, tid); + found = true; + break; + } + } + + if (found) continue; + + // `CrRendererMain` can be missing if chrome is launched with the `--single-process` flag. + // In this case, page tasks will be run in the browser thread. + for (const [tid, thread] of threads) { + if (thread.args.name === 'CrBrowserMain') { + rendererPidToTid.set(pid, tid); + found = true; + break; + } + } + } + + return filteredTraceSort(trace.traceEvents, e => rendererPidToTid.get(e.pid) === e.tid); +} + +/** + * @param {Lantern.NetworkRequest[]} requests + * @param {LH.Trace} trace + * @param {LH.Artifacts.TraceEngineResult} traceEngineResult + * @param {LH.Artifacts.URL=} URL + */ +function createGraph(requests, trace, traceEngineResult, URL) { + const mainThreadEvents = collectMainThreadEvents(trace, traceEngineResult); + + // URL defines the initial request that the Lantern graph starts at (the root node) and the + // main document request. These are equal if there are no redirects. + if (!URL) { + URL = { + requestedUrl: requests[0].url, + mainDocumentUrl: '', + finalDisplayedUrl: '', + }; + + let request = requests[0]; + while (request.redirectDestination) { + request = request.redirectDestination; + } + URL.mainDocumentUrl = request.url; + } + + return PageDependencyGraph.createGraph(mainThreadEvents, requests, URL); +} + +export { + createProcessedNavigation, + createNetworkRequests, + createGraph, +}; diff --git a/core/test/lib/lantern/metrics/metric-test-utils.js b/core/test/lib/lantern/metrics/metric-test-utils.js index d7cdacf332c8..a6e850b2e5e2 100644 --- a/core/test/lib/lantern/metrics/metric-test-utils.js +++ b/core/test/lib/lantern/metrics/metric-test-utils.js @@ -5,10 +5,9 @@ */ import {TraceEngineResult} from '../../../../computed/trace-engine-result.js'; -import {createProcessedNavigation} from '../../../../lib/lantern/lantern.js'; -import {PageDependencyGraph} from '../../../../lib/lantern/page-dependency-graph.js'; import {NetworkAnalyzer} from '../../../../lib/lantern/simulator/network-analyzer.js'; import {Simulator} from '../../../../lib/lantern/simulator/simulator.js'; +import * as TraceEngineComputationData from '../../../../lib/lantern/trace-engine-computation-data.js'; import * as Lantern from '../../../../lib/lantern/types/lantern.js'; /** @typedef {Lantern.NetworkRequest} NetworkRequest */ @@ -24,13 +23,14 @@ async function getComputationDataFromFixture({trace, settings, URL}) { const context = {settings, computedCache: new Map()}; const traceEngineResult = await TraceEngineResult.request({trace}, context); - const {graph, requests} = - await PageDependencyGraph.createGraphFromTrace(trace, traceEngineResult, URL); - const processedNavigation = createProcessedNavigation(traceEngineResult); + const requests = TraceEngineComputationData.createNetworkRequests(trace, traceEngineResult); const networkAnalysis = NetworkAnalyzer.analyze(requests); - const simulator = Simulator.createSimulator({...settings, networkAnalysis}); - return {simulator, graph, processedNavigation}; + return { + simulator: Simulator.createSimulator({...settings, networkAnalysis}), + graph: TraceEngineComputationData.createGraph(requests, trace, traceEngineResult, URL), + processedNavigation: TraceEngineComputationData.createProcessedNavigation(traceEngineResult), + }; } export {getComputationDataFromFixture};