diff --git a/.eslintrc.json b/.eslintrc.json index 022995a54db7..6f62b37f8bff 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -18,6 +18,15 @@ "rules": { "@typescript-eslint/array-type": "error", "@typescript-eslint/ban-ts-comment": 0, + "@typescript-eslint/ban-types": [ + "error", + { + "types": { + "{}": false + }, + "extendDefaults": true + } + ], "@typescript-eslint/consistent-type-imports": [ "error", { diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts index f6457e7aa78a..37d54e7b2672 100644 --- a/packages/core/src/crawlers/crawler_commons.ts +++ b/packages/core/src/crawlers/crawler_commons.ts @@ -14,7 +14,6 @@ import { KeyValueStore } from '../storages'; export interface RestrictedCrawlingContext // we need `Record` here, otherwise `Omit` is resolved badly - // eslint-disable-next-line extends Record { /** * The original {@apilink Request} object. diff --git a/packages/core/src/typedefs.ts b/packages/core/src/typedefs.ts index 6f2dcaf39a5a..49f7f49c1d2a 100644 --- a/packages/core/src/typedefs.ts +++ b/packages/core/src/typedefs.ts @@ -1,5 +1,3 @@ -/* eslint-disable @typescript-eslint/ban-types */ - /** @ignore */ export type Constructor = new (...args: any[]) => T; diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts index a1dbf2a94e73..d6e0f87782b3 100644 --- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts +++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts @@ -1,5 +1,5 @@ import { addTimeoutToPromise } from '@apify/timeout'; -import { extractUrlsFromPage } from '@crawlee/browser'; +import { extractUrlsFromPage, type RouterHandler } from '@crawlee/browser'; import type { RestrictedCrawlingContext, StatisticState, @@ -98,7 +98,7 @@ export interface AdaptivePlaywrightCrawlerOptions * If the function throws an exception, the crawler will try to re-crawl the * request later, up to `option.maxRequestRetries` times. */ - requestHandler: (crawlingContext: AdaptivePlaywrightCrawlerContext) => Awaitable; + requestHandler?: (crawlingContext: AdaptivePlaywrightCrawlerContext) => Awaitable; /** * Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests. @@ -156,12 +156,20 @@ export interface AdaptivePlaywrightCrawlerOptions * @experimental */ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { - private adaptiveRequestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler']; + private adaptiveRequestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] & {}; private renderingTypePredictor: NonNullable; private resultChecker: NonNullable; private resultComparator: NonNullable; override readonly stats: AdaptivePlaywrightCrawlerStatistics; + /** + * Default {@apilink Router} instance that will be used if we don't specify any {@apilink AdaptivePlaywrightCrawlerOptions.requestHandler|`requestHandler`}. + * See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}. + */ + // @ts-ignore + override readonly router: RouterHandler = + Router.create(); + constructor( { requestHandler, @@ -175,7 +183,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { override readonly config = Configuration.getGlobalConfig(), ) { super(options, config); - this.adaptiveRequestHandler = requestHandler; + this.adaptiveRequestHandler = requestHandler ?? this.router; this.renderingTypePredictor = renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }); this.resultChecker = resultChecker ?? (() => true); @@ -322,7 +330,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { log: crawlingContext.log, querySelector: async (selector, timeoutMs) => { const locator = playwrightContext.page.locator(selector).first(); - await locator.waitFor({ timeout: timeoutMs }); + await locator.waitFor({ timeout: timeoutMs, state: 'attached' }); return (await playwrightContext.parseWithCheerio())( selector, ) as Cheerio; diff --git a/packages/utils/src/internals/typedefs.ts b/packages/utils/src/internals/typedefs.ts index f5483b2ce0c2..e79602e4efce 100644 --- a/packages/utils/src/internals/typedefs.ts +++ b/packages/utils/src/internals/typedefs.ts @@ -1,5 +1,3 @@ -/* eslint-disable @typescript-eslint/ban-types */ - /** @ignore */ export function entries(obj: T) { return Object.entries(obj) as [keyof T, T[keyof T]][];