1import { setTimeout } from 'node:timers/promises';
2
3// Crawlee - web scraping and browser automation library (Read more at https://crawlee.dev)
4import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
5// Apify SDK - toolkit for building Apify Actors (Read more at https://docs.apify.com/sdk/js/)
6import { Actor } from 'apify';
7
8// The init() call configures the Actor to correctly work with the Apify-provided environment - mainly the storage infrastructure. It is necessary that every Actor performs an init() call.
9await Actor.init();
10
11// Handle graceful abort - Actor is being stopped by user or platform
12Actor.on('aborting', async () => {
13    // Persist any state, do any cleanup you need, and terminate the Actor using `await Actor.exit()` explicitly as soon as possible
14    // This will help ensure that the Actor is doing best effort to honor any potential limits on costs of a single run set by the user
15    // Wait 1 second to allow Crawlee/SDK useState and other state persistence operations to complete
16    // This is a temporary workaround until SDK implements proper state persistence in the aborting event
17    await setTimeout(1000);
18    await Actor.exit();
19});
20
21// Structure of input is defined in input_schema.json
22const { startUrls = ['https://apify.com'], maxRequestsPerCrawl = 100 } = (await Actor.getInput()) ?? {};
23
24// Proxy configuration to rotate IP addresses and prevent blocking (https://docs.apify.com/platform/proxy)
25// `checkAccess` flag ensures the proxy credentials are valid, but the check can take a few hundred milliseconds.
26// Disable it for short runs if you are sure your proxy configuration is correct
27const proxyConfiguration = await Actor.createProxyConfiguration({ checkAccess: true });
28
29const crawler = new CheerioCrawler({
30    proxyConfiguration,
31    maxRequestsPerCrawl,
32    async requestHandler({ enqueueLinks, request, $, log }) {
33        log.info('enqueueing new URLs');
34        await enqueueLinks();
35
36        // Extract title from the page.
37        const title = $('title').text();
38        log.info(`${title}`, { url: request.loadedUrl });
39
40        // Save url and title to Dataset - a table-like storage.
41        await Dataset.pushData({ url: request.loadedUrl, title });
42    },
43});
44
45await crawler.run(startUrls);
46
47// Gracefully exit the Actor process. It's recommended to quit all Actors with an exit()
48await Actor.exit();