fix: RQ request count is consistent after migration (#2116)

Until now, the `assumedTotalCount` got reset on a migration (it was a
purely local counter). This causes #1855, which is not too bad, but
manifests at a very prominent place in the console.

I'm not *way too confident* about the async iife in the constructor, but
I think it should work just fine, as the new `totalCount` property is
purely approximative (and is marked as such it the comment).
Closes #1855

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -783,7 +783,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                 // eslint-disable-next-line max-len
                 message = `Experiencing problems, ${this.stats.state.requestsFailed - previousState.requestsFailed || this.stats.state.requestsFailed} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
             } else {
-                const total = this.requestQueue?.assumedTotalCount || this.requestList?.length();
+                const total = this.requestQueue?.getTotalCount() || this.requestList?.length();
                 message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests.`;
             }
 

packages/core/src/storages/request_provider.ts

@@ -41,6 +41,8 @@ export abstract class RequestProvider implements IStorage {
     assumedTotalCount = 0;
     assumedHandledCount = 0;
 
+    private initialCount = 0;
+
     protected queueHeadIds = new ListDictionary<string>();
     protected requestCache: LruCache<RequestLruItem>;
     /** @internal */
@@ -77,6 +79,15 @@ export abstract class RequestProvider implements IStorage {
         return this.inProgress.size;
     }
 
+    /**
+     * Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).
+     *
+     * Survives restarts and actor migrations.
+     */
+    getTotalCount() {
+        return this.assumedTotalCount + this.initialCount;
+    }
+
     /**
      * Adds a request to the queue.
      *
@@ -631,6 +642,9 @@ export abstract class RequestProvider implements IStorage {
         const queue = await manager.openStorage(queueIdOrName, options.storageClient);
         queue.proxyConfiguration = options.proxyConfiguration;
 
+        // eslint-disable-next-line dot-notation
+        queue['initialCount'] = (await queue.client.get())?.totalRequestCount ?? 0;
+
         return queue;
     }
 }