Web Scraper

1import { ApifyClient } from 'apify-client';
2
3// Initialize the ApifyClient with your Apify API token
4// Replace the '<YOUR_API_TOKEN>' with your token
5const client = new ApifyClient({
6    token: '<YOUR_API_TOKEN>',
7});
8
9// Prepare Actor input
10const input = {
11    "runMode": "DEVELOPMENT",
12    "startUrls": [
13        {
14            "url": "https://apify.com"
15        }
16    ],
17    "linkSelector": "a[href]",
18    "pseudoUrls": [
19        {
20            "purl": "https://apify.com[(/[\\w-]+)?]"
21        }
22    ],
23    "pageFunction": `// The function accepts a single argument: the "context" object.
24        // For a complete list of its properties and functions,
25        // see https://apify.com/apify/web-scraper#page-function 
26        async function pageFunction(context) {
27            // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!
28            // debugger; 
29        
30            // jQuery is handy for finding DOM elements and extracting data from them.
31            // To use it, make sure to enable the "Inject jQuery" option.
32            const $ = context.jQuery;
33            const pageTitle = $('title').first().text();
34        
35            // Print some information to actor log
36            context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);
37        
38            // Manually add a new page to the queue for scraping.
39           await context.enqueueRequest({ url: 'http://www.example.com' });
40        
41            // Return an object with the data extracted from the page.
42            // It will be stored to the resulting dataset.
43            return {
44                url: context.request.url,
45                pageTitle
46            };
47        }`,
48    "proxyConfiguration": {
49        "useApifyProxy": true
50    },
51    "initialCookies": [],
52    "waitUntil": [
53        "networkidle2"
54    ],
55    "preNavigationHooks": `// We need to return array of (possibly async) functions here.
56        // The functions accept two arguments: the "crawlingContext" object
57        // and "gotoOptions".
58        [
59            async (crawlingContext, gotoOptions) => {
60                // ...
61            },
62        ]`,
63    "postNavigationHooks": `// We need to return array of (possibly async) functions here.
64        // The functions accept a single argument: the "crawlingContext" object.
65        [
66            async (crawlingContext) => {
67                // ...
68            },
69        ]`,
70    "breakpointLocation": "NONE",
71    "customData": {}
72};
73
74// Run the Actor and wait for it to finish
75const run = await client.actor("apify/web-scraper").call(input);
76
77// Fetch and print Actor results from the run's dataset (if any)
78console.log('Results from dataset');
79console.log(`💾 Check your data here: https://console.apify.com/storage/datasets/${run.defaultDatasetId}`);
80const { items } = await client.dataset(run.defaultDatasetId).listItems();
81items.forEach((item) => {
82    console.dir(item);
83});
84
85// 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/js/docs