diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts index 6718c6baea29..643143590e94 100644 --- a/packages/core/src/request.ts +++ b/packages/core/src/request.ts @@ -193,7 +193,7 @@ export class Request { this.id = id; this.url = url; this.loadedUrl = loadedUrl; - this.uniqueKey = uniqueKey || this._computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }); + this.uniqueKey = uniqueKey || Request.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }); this.method = method; this.payload = payload; this.noRetry = noRetry; @@ -378,7 +378,18 @@ export class Request { this.errorMessages.push(message); } - protected _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }: ComputeUniqueKeyOptions) { + // TODO: only for better BC, remove in v4 + protected _computeUniqueKey(options: ComputeUniqueKeyOptions) { + return Request.computeUniqueKey(options); + } + + // TODO: only for better BC, remove in v4 + protected _hashPayload(payload: BinaryLike): string { + return Request.hashPayload(payload); + } + + /** @internal */ + static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false }: ComputeUniqueKeyOptions) { const normalizedMethod = method.toUpperCase(); const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors. if (!useExtendedUniqueKey) { @@ -390,11 +401,12 @@ export class Request { } return normalizedUrl; } - const payloadHash = payload ? this._hashPayload(payload) : ''; + const payloadHash = payload ? Request.hashPayload(payload) : ''; return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`; } - protected _hashPayload(payload: BinaryLike): string { + /** @internal */ + static hashPayload(payload: BinaryLike): string { return crypto .createHash('sha256') .update(payload) diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.ts index 9eeb4f80e189..6c2d6b0e90d0 100644 --- a/packages/core/src/storages/request_list.ts +++ b/packages/core/src/storages/request_list.ts @@ -9,8 +9,7 @@ import type { EventManager } from '../events'; import { EventType } from '../events'; import { log } from '../log'; import type { ProxyConfiguration } from '../proxy_configuration'; -import type { InternalSource, RequestOptions, Source } from '../request'; -import { Request } from '../request'; +import { type InternalSource, type RequestOptions, Request, type Source } from '../request'; import { createDeserialize, serializeArray } from '../serialization'; /** @internal */ @@ -238,7 +237,7 @@ export class RequestList { * All requests in the array have distinct uniqueKey! * @internal */ - requests: Request[] = []; + requests: (Request | RequestOptions)[] = []; /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */ private nextIndex = 0; @@ -551,7 +550,7 @@ export class RequestList { return { nextIndex: this.nextIndex, nextUniqueKey: this.nextIndex < this.requests.length - ? this.requests[this.nextIndex].uniqueKey + ? this.requests[this.nextIndex].uniqueKey! : null, inProgress: [...this.inProgress], }; @@ -593,21 +592,31 @@ export class RequestList { if (uniqueKey) { this.reclaimed.delete(uniqueKey); const index = this.uniqueKeyToIndex[uniqueKey]; - return this.requests[index]; + return this.ensureRequest(this.requests[index], index); } // Otherwise return next request. if (this.nextIndex < this.requests.length) { - const request = this.requests[this.nextIndex]; - this.inProgress.add(request.uniqueKey); + const index = this.nextIndex; + const request = this.requests[index]; + this.inProgress.add(request.uniqueKey!); this.nextIndex++; this.isStatePersisted = false; - return request; + return this.ensureRequest(request, index); } return null; } + private ensureRequest(requestLike: Request | RequestOptions, index: number): Request { + if (requestLike instanceof Request) { + return requestLike; + } + + this.requests[index] = new Request(requestLike); + return this.requests[index] as Request; + } + /** * Marks request as handled after successful processing. */ @@ -694,19 +703,21 @@ export class RequestList { * of a `Request`, then the function creates a `Request` instance. */ protected _addRequest(source: RequestListSource) { - let request; + let request: Request | RequestOptions; const type = typeof source; + if (type === 'string') { - request = new Request({ url: source as string }); + request = { url: source as string }; } else if (source instanceof Request) { request = source; } else if (source && type === 'object') { - request = new Request(source as RequestOptions); + request = source as RequestOptions; } else { throw new Error(`Cannot create Request from type: ${type}`); } const hasUniqueKey = Reflect.has(Object(source), 'uniqueKey'); + request.uniqueKey ??= Request.computeUniqueKey(request as any); // Add index to uniqueKey if duplicates are to be kept if (this.keepDuplicateUrls && !hasUniqueKey) { diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.ts index 539695b885fa..2f60e6613153 100644 --- a/test/core/request_list.test.ts +++ b/test/core/request_list.test.ts @@ -728,7 +728,7 @@ describe('RequestList', () => { const name = 'xxx'; const SDK_KEY = `SDK_${name}`; const sources = ['https://example.com']; - const requests = sources.map((url) => new Request({ url })); + const requests = sources.map((url) => ({ url, uniqueKey: url })); const rl = await RequestList.open(name, sources); expect(rl).toBeInstanceOf(RequestList); @@ -752,7 +752,7 @@ describe('RequestList', () => { const SDK_KEY = `SDK_${name}`; let counter = 0; const sources = [{ url: 'https://example.com' }]; - const requests = sources.map(({ url }) => new Request({ url, uniqueKey: `${url}-${counter++}` })); + const requests = sources.map(({ url }) => ({ url, uniqueKey: `${url}-${counter++}` })); const options = { keepDuplicateUrls: true, persistStateKey: 'yyy', @@ -780,7 +780,7 @@ describe('RequestList', () => { const name: string = null; const sources = [{ url: 'https://example.com' }]; - const requests = sources.map(({ url }) => new Request({ url })); + const requests = sources.map(({ url }) => ({ url, uniqueKey: url })); const rl = await RequestList.open(name, sources); expect(rl).toBeInstanceOf(RequestList);