From f63ccc018c9e9046531287c47d11283a8e71a6ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= <banan23@gmail.com>
Date: Wed, 23 Aug 2023 17:38:55 +0200
Subject: [PATCH] fix: clean up `inProgress` cache when delaying requests via
 `sameDomainDelaySecs` (#2045)

---
 packages/basic-crawler/src/internals/basic-crawler.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 80a87b20d06c..90ff27c73e77 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -982,7 +982,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
      * adding it back to the queue after the timeout passes. Returns `true` if the request
      * should be ignored and will be reclaimed to the queue once ready.
      */
-    protected delayRequest(request:Request, source: RequestQueue | RequestList) {
+    protected delayRequest(request: Request, source: RequestQueue | RequestList) {
         const domain = getDomain(request.url);
 
         if (!domain || !request) {
@@ -997,10 +997,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             return false;
         }
 
+        source.inProgress.delete(request.id!);
         const delay = lastAccessTime + this.sameDomainDelayMillis - now;
         this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
         setTimeout(async () => {
             this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
+            source?.inProgress.add(request.id!);
             await source?.reclaimRequest(request);
         }, delay);