diff --git a/docs/examples/http_crawler.mdx b/docs/examples/http_crawler.mdx index e62793ae554a..41f5d25394f3 100644 --- a/docs/examples/http_crawler.mdx +++ b/docs/examples/http_crawler.mdx @@ -7,7 +7,7 @@ import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import ApiLink from '@site/src/components/ApiLink'; import HttpCrawlerSource from '!!raw-loader!roa-loader!./http_crawler.ts'; -This example demonstrates how to use `HttpCrawler` to crawl a list of URLs from an external file, load each URL using a plain HTTP request, and save HTML. +This example demonstrates how to use `HttpCrawler` to build a crawler that crawls a list of URLs from an external file, load each URL using a plain HTTP request, and save HTML. {HttpCrawlerSource} diff --git a/docs/examples/http_crawler.ts b/docs/examples/http_crawler.ts index 4052df13136b..3940fc5de650 100644 --- a/docs/examples/http_crawler.ts +++ b/docs/examples/http_crawler.ts @@ -35,8 +35,8 @@ const crawler = new HttpCrawler({ // Store the results to the dataset. In local configuration, // the data will be stored as JSON files in ./storage/datasets/default await Dataset.pushData({ - url: request.url, - body, + url: request.url, // URL of the page + body, // HTML code of the page }); }, @@ -47,6 +47,7 @@ const crawler = new HttpCrawler({ }); // Run the crawler and wait for it to finish. +// It will crawl a list of URLs from an external file, load each URL using a plain HTTP request, and save HTML await crawler.run([ 'https://crawlee.dev', ]);