Skip to content

Commit

Permalink
chore: templates use new dataset helpers (#2059)
Browse files Browse the repository at this point in the history
  • Loading branch information
barjin authored Aug 31, 2023
1 parent 742f32e commit 9369a85
Show file tree
Hide file tree
Showing 8 changed files with 24 additions and 24 deletions.
6 changes: 3 additions & 3 deletions packages/templates/templates/cheerio-js/src/routes.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Dataset, createCheerioRouter } from 'crawlee';
import { createCheerioRouter } from 'crawlee';

export const router = createCheerioRouter();

Expand All @@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});

router.addHandler('detail', async ({ request, $, log }) => {
router.addHandler('detail', async ({ request, $, log, pushData }) => {
const title = $('title').text();
log.info(`${title}`, { url: request.loadedUrl });

await Dataset.pushData({
await pushData({
url: request.loadedUrl,
title,
});
Expand Down
6 changes: 3 additions & 3 deletions packages/templates/templates/cheerio-ts/src/routes.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Dataset, createCheerioRouter } from 'crawlee';
import { createCheerioRouter } from 'crawlee';

export const router = createCheerioRouter();

Expand All @@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});

router.addHandler('detail', async ({ request, $, log }) => {
router.addHandler('detail', async ({ request, $, log, pushData }) => {
const title = $('title').text();
log.info(`${title}`, { url: request.loadedUrl });

await Dataset.pushData({
await pushData({
url: request.loadedUrl,
title,
});
Expand Down
6 changes: 3 additions & 3 deletions packages/templates/templates/getting-started-js/src/main.js
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
// For more information, see https://crawlee.dev/
import { PlaywrightCrawler, Dataset } from 'crawlee';
import { PlaywrightCrawler } from 'crawlee';

// PlaywrightCrawler crawls the web using a headless
// browser controlled by the Playwright library.
const crawler = new PlaywrightCrawler({
// Use the requestHandler to process each of the crawled pages.
async requestHandler({ request, page, enqueueLinks, log }) {
async requestHandler({ request, page, enqueueLinks, log, pushData }) {
const title = await page.title();
log.info(`Title of ${request.loadedUrl} is '${title}'`);

// Save results as JSON to ./storage/datasets/default
await Dataset.pushData({ title, url: request.loadedUrl });
await pushData({ title, url: request.loadedUrl });

// Extract links from the current page
// and add them to the crawling queue.
Expand Down
6 changes: 3 additions & 3 deletions packages/templates/templates/getting-started-ts/src/main.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
// For more information, see https://crawlee.dev/
import { PlaywrightCrawler, Dataset } from 'crawlee';
import { PlaywrightCrawler } from 'crawlee';

// PlaywrightCrawler crawls the web using a headless
// browser controlled by the Playwright library.
const crawler = new PlaywrightCrawler({
// Use the requestHandler to process each of the crawled pages.
async requestHandler({ request, page, enqueueLinks, log }) {
async requestHandler({ request, page, enqueueLinks, log, pushData }) {
const title = await page.title();
log.info(`Title of ${request.loadedUrl} is '${title}'`);

// Save results as JSON to ./storage/datasets/default
await Dataset.pushData({ title, url: request.loadedUrl });
await pushData({ title, url: request.loadedUrl });

// Extract links from the current page
// and add them to the crawling queue.
Expand Down
6 changes: 3 additions & 3 deletions packages/templates/templates/playwright-js/src/routes.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Dataset, createPlaywrightRouter } from 'crawlee';
import { createPlaywrightRouter } from 'crawlee';

export const router = createPlaywrightRouter();

Expand All @@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});

router.addHandler('detail', async ({ request, page, log }) => {
router.addHandler('detail', async ({ request, page, log, pushData }) => {
const title = await page.title();
log.info(`${title}`, { url: request.loadedUrl });

await Dataset.pushData({
await pushData({
url: request.loadedUrl,
title,
});
Expand Down
6 changes: 3 additions & 3 deletions packages/templates/templates/playwright-ts/src/routes.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Dataset, createPlaywrightRouter } from 'crawlee';
import { createPlaywrightRouter } from 'crawlee';

export const router = createPlaywrightRouter();

Expand All @@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});

router.addHandler('detail', async ({ request, page, log }) => {
router.addHandler('detail', async ({ request, page, log, pushData }) => {
const title = await page.title();
log.info(`${title}`, { url: request.loadedUrl });

await Dataset.pushData({
await pushData({
url: request.loadedUrl,
title,
});
Expand Down
6 changes: 3 additions & 3 deletions packages/templates/templates/puppeteer-js/src/routes.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Dataset, createPuppeteerRouter } from 'crawlee';
import { createPuppeteerRouter } from 'crawlee';

export const router = createPuppeteerRouter();

Expand All @@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});

router.addHandler('detail', async ({ request, page, log }) => {
router.addHandler('detail', async ({ request, page, log, pushData }) => {
const title = await page.title();
log.info(`${title}`, { url: request.loadedUrl });

await Dataset.pushData({
await pushData({
url: request.loadedUrl,
title,
});
Expand Down
6 changes: 3 additions & 3 deletions packages/templates/templates/puppeteer-ts/src/routes.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Dataset, createPuppeteerRouter } from 'crawlee';
import { createPuppeteerRouter } from 'crawlee';

export const router = createPuppeteerRouter();

Expand All @@ -10,11 +10,11 @@ router.addDefaultHandler(async ({ enqueueLinks, log }) => {
});
});

router.addHandler('detail', async ({ request, page, log }) => {
router.addHandler('detail', async ({ request, page, log, pushData }) => {
const title = await page.title();
log.info(`${title}`, { url: request.loadedUrl });

await Dataset.pushData({
await pushData({
url: request.loadedUrl,
title,
});
Expand Down

0 comments on commit 9369a85

Please sign in to comment.