perf: optimize RequestList memory footprint (#2466)

The request list now delays the conversion of the source items into the
`Request` objects, resulting in a significantly less memory footprint.

Related: https://apify.slack.com/archives/C0L33UM7Z/p1715109984834079

packages/core/src/request.ts

@@ -193,7 +193,7 @@ export class Request<UserData extends Dictionary = Dictionary> {
         this.id = id;
         this.url = url;
         this.loadedUrl = loadedUrl;
-        this.uniqueKey = uniqueKey || this._computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+        this.uniqueKey = uniqueKey || Request.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
         this.method = method;
         this.payload = payload;
         this.noRetry = noRetry;
@@ -378,7 +378,18 @@ export class Request<UserData extends Dictionary = Dictionary> {
         this.errorMessages.push(message);
     }
 
-    protected _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }: ComputeUniqueKeyOptions) {
+    // TODO: only for better BC, remove in v4
+    protected _computeUniqueKey(options: ComputeUniqueKeyOptions) {
+        return Request.computeUniqueKey(options);
+    }
+
+    // TODO: only for better BC, remove in v4
+    protected _hashPayload(payload: BinaryLike): string {
+        return Request.hashPayload(payload);
+    }
+
+    /** @internal */
+    static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false }: ComputeUniqueKeyOptions) {
         const normalizedMethod = method.toUpperCase();
         const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
         if (!useExtendedUniqueKey) {
@@ -390,11 +401,12 @@ export class Request<UserData extends Dictionary = Dictionary> {
             }
             return normalizedUrl;
         }
-        const payloadHash = payload ? this._hashPayload(payload) : '';
+        const payloadHash = payload ? Request.hashPayload(payload) : '';
         return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
     }
 
-    protected _hashPayload(payload: BinaryLike): string {
+    /** @internal */
+    static hashPayload(payload: BinaryLike): string {
         return crypto
             .createHash('sha256')
             .update(payload)

packages/core/src/storages/request_list.ts

@@ -9,8 +9,7 @@ import type { EventManager } from '../events';
 import { EventType } from '../events';
 import { log } from '../log';
 import type { ProxyConfiguration } from '../proxy_configuration';
-import type { InternalSource, RequestOptions, Source } from '../request';
-import { Request } from '../request';
+import { type InternalSource, type RequestOptions, Request, type Source } from '../request';
 import { createDeserialize, serializeArray } from '../serialization';
 
 /** @internal */
@@ -238,7 +237,7 @@ export class RequestList {
      * All requests in the array have distinct uniqueKey!
      * @internal
      */
-    requests: Request[] = [];
+    requests: (Request | RequestOptions)[] = [];
 
     /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
     private nextIndex = 0;
@@ -551,7 +550,7 @@ export class RequestList {
         return {
             nextIndex: this.nextIndex,
             nextUniqueKey: this.nextIndex < this.requests.length
-                ? this.requests[this.nextIndex].uniqueKey
+                ? this.requests[this.nextIndex].uniqueKey!
                 : null,
             inProgress: [...this.inProgress],
         };
@@ -593,21 +592,31 @@ export class RequestList {
         if (uniqueKey) {
             this.reclaimed.delete(uniqueKey);
             const index = this.uniqueKeyToIndex[uniqueKey];
-            return this.requests[index];
+            return this.ensureRequest(this.requests[index], index);
         }
 
         // Otherwise return next request.
         if (this.nextIndex < this.requests.length) {
-            const request = this.requests[this.nextIndex];
-            this.inProgress.add(request.uniqueKey);
+            const index = this.nextIndex;
+            const request = this.requests[index];
+            this.inProgress.add(request.uniqueKey!);
             this.nextIndex++;
             this.isStatePersisted = false;
-            return request;
+            return this.ensureRequest(request, index);
         }
 
         return null;
     }
 
+    private ensureRequest(requestLike: Request | RequestOptions, index: number): Request {
+        if (requestLike instanceof Request) {
+            return requestLike;
+        }
+
+        this.requests[index] = new Request(requestLike);
+        return this.requests[index] as Request;
+    }
+
     /**
      * Marks request as handled after successful processing.
      */
@@ -694,19 +703,21 @@ export class RequestList {
      * of a `Request`, then the function creates a `Request` instance.
      */
     protected _addRequest(source: RequestListSource) {
-        let request;
+        let request: Request | RequestOptions;
         const type = typeof source;
+
         if (type === 'string') {
-            request = new Request({ url: source as string });
+            request = { url: source as string };
         } else if (source instanceof Request) {
             request = source;
         } else if (source && type === 'object') {
-            request = new Request(source as RequestOptions);
+            request = source as RequestOptions;
         } else {
             throw new Error(`Cannot create Request from type: ${type}`);
         }
 
         const hasUniqueKey = Reflect.has(Object(source), 'uniqueKey');
+        request.uniqueKey ??= Request.computeUniqueKey(request as any);
 
         // Add index to uniqueKey if duplicates are to be kept
         if (this.keepDuplicateUrls && !hasUniqueKey) {

test/core/request_list.test.ts

@@ -728,7 +728,7 @@ describe('RequestList', () => {
             const name = 'xxx';
             const SDK_KEY = `SDK_${name}`;
             const sources = ['https://example.com'];
-            const requests = sources.map((url) => new Request({ url }));
+            const requests = sources.map((url) => ({ url, uniqueKey: url }));
 
             const rl = await RequestList.open(name, sources);
             expect(rl).toBeInstanceOf(RequestList);
@@ -752,7 +752,7 @@ describe('RequestList', () => {
             const SDK_KEY = `SDK_${name}`;
             let counter = 0;
             const sources = [{ url: 'https://example.com' }];
-            const requests = sources.map(({ url }) => new Request({ url, uniqueKey: `${url}-${counter++}` }));
+            const requests = sources.map(({ url }) => ({ url, uniqueKey: `${url}-${counter++}` }));
             const options = {
                 keepDuplicateUrls: true,
                 persistStateKey: 'yyy',
@@ -780,7 +780,7 @@ describe('RequestList', () => {
 
             const name: string = null;
             const sources = [{ url: 'https://example.com' }];
-            const requests = sources.map(({ url }) => new Request({ url }));
+            const requests = sources.map(({ url }) => ({ url, uniqueKey: url }));
 
             const rl = await RequestList.open(name, sources);
             expect(rl).toBeInstanceOf(RequestList);