diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index dc2130cfa8a7..c4c6c0ca9a08 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -356,6 +356,10 @@ export interface BasicCrawlerOptions; -// Omitted (https://github.com/sindresorhus/got/blob/main/documentation/2-options.md): -// - decompress, -// - resolveBodyOnly, -// - allowGetBody, -// - dnsLookup, -// - dnsCache, -// - dnsLookupIpVersion, -// - retry, -// - hooks, -// - parseJson, -// - stringifyJson, -// - request, -// - cache, -// - cacheOptions, -// - http2 -// - https -// - agent -// - localAddress -// - createConnection -// - pagination -// - setHost -// - maxHeaderSize -// - methodRewriting -// - enableUnixSockets -// - context +/** + * HTTP Request as accepted by {@apilink BaseHttpClient} methods. + */ export interface HttpRequest { [k: string]: unknown; // TODO BC with got - remove in 4.0 @@ -124,17 +105,28 @@ export interface HttpRequest sessionToken?: object; } +/** + * Additional options for HTTP requests that need to be handled separately before passing to {@apilink BaseHttpClient}. + */ export interface HttpRequestOptions extends HttpRequest { + /** Search (query string) parameters to be appended to the request URL */ searchParams?: SearchParams; + /** A form to be sent in the HTTP request body (URL encoding will be used) */ form?: Record; + /** Artbitrary object to be JSON-serialized and sent as the HTTP request body */ json?: unknown; + /** Basic HTTP Auth username */ username?: string; + /** Basic HTTP Auth password */ password?: string; } +/** + * HTTP response data, without a body, as returned by {@apilink BaseHttpClient} methods. + */ export interface BaseHttpResponseData { redirectUrls: URL[]; url: string; @@ -154,6 +146,9 @@ interface HttpResponseWithoutBody; } +/** + * HTTP response data as returned by the {@apilink BaseHttpClient.sendRequest} method. + */ export interface HttpResponse extends HttpResponseWithoutBody { [k: string]: any; // TODO BC with got - remove in 4.0 @@ -161,25 +156,43 @@ export interface HttpResponse void; +/** + * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl. + */ export interface BaseHttpClient { + /** + * Perform an HTTP Request and return the complete response. + */ sendRequest( request: HttpRequest, ): Promise>; + /** + * Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response. + */ stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise; } +/** + * Converts {@apilink HttpRequestOptions} to a {@apilink HttpRequest}. + */ export function processHttpRequestOptions({ searchParams, form, diff --git a/packages/core/src/http_clients/got_scraping_http_client.ts b/packages/core/src/http_clients/got_scraping_http_client.ts index 07e75a313734..0ac359c28972 100644 --- a/packages/core/src/http_clients/got_scraping_http_client.ts +++ b/packages/core/src/http_clients/got_scraping_http_client.ts @@ -11,7 +11,13 @@ import type { BaseHttpClient, } from './base_http_client'; +/** + * A HTTP client implementation based on the `got-scraping` library. + */ export class GotScrapingHttpClient implements BaseHttpClient { + /** + * @inheritDoc + */ async sendRequest( request: HttpRequest, ): Promise> { @@ -30,6 +36,9 @@ export class GotScrapingHttpClient implements BaseHttpClient { }; } + /** + * @inheritDoc + */ async stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise { // eslint-disable-next-line no-async-promise-executor return new Promise(async (resolve, reject) => { diff --git a/packages/utils/src/internals/url.ts b/packages/utils/src/internals/url.ts index ec9ec29efe7b..a17224d5bb57 100644 --- a/packages/utils/src/internals/url.ts +++ b/packages/utils/src/internals/url.ts @@ -1,5 +1,11 @@ export type SearchParams = string | URLSearchParams | Record; +/** + * Appends search (query string) parameters to a URL, replacing the original value (if any). + * + * @param url The URL to append to. + * @param searchParams The search parameters to be appended. + */ export function applySearchParams(url: URL, searchParams: SearchParams | undefined): void { if (searchParams === undefined) { return;