Web Scraper

1from apify_client import ApifyClient
2
3# Initialize the ApifyClient with your Apify API token
4# Replace '<YOUR_API_TOKEN>' with your token.
5client = ApifyClient("<YOUR_API_TOKEN>")
6
7# Prepare the Actor input
8run_input = {
9    "runMode": "DEVELOPMENT",
10    "startUrls": [{ "url": "https://apify.com" }],
11    "linkSelector": "a[href]",
12    "pseudoUrls": [{ "purl": "https://apify.com[(/[\\w-]+)?]" }],
13    "pageFunction": """// The function accepts a single argument: the \"context\" object.
14// For a complete list of its properties and functions,
15// see https://apify.com/apify/web-scraper#page-function 
16async function pageFunction(context) {
17    // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!
18    // debugger; 
19
20    // jQuery is handy for finding DOM elements and extracting data from them.
21    // To use it, make sure to enable the \"Inject jQuery\" option.
22    const $ = context.jQuery;
23    const pageTitle = $('title').first().text();
24
25    // Print some information to actor log
26    context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);
27
28    // Manually add a new page to the queue for scraping.
29   await context.enqueueRequest({ url: 'http://www.example.com' });
30
31    // Return an object with the data extracted from the page.
32    // It will be stored to the resulting dataset.
33    return {
34        url: context.request.url,
35        pageTitle
36    };
37}""",
38    "proxyConfiguration": { "useApifyProxy": True },
39    "initialCookies": [],
40    "waitUntil": ["networkidle2"],
41    "preNavigationHooks": """// We need to return array of (possibly async) functions here.
42// The functions accept two arguments: the \"crawlingContext\" object
43// and \"gotoOptions\".
44[
45    async (crawlingContext, gotoOptions) => {
46        // ...
47    },
48]
49""",
50    "postNavigationHooks": """// We need to return array of (possibly async) functions here.
51// The functions accept a single argument: the \"crawlingContext\" object.
52[
53    async (crawlingContext) => {
54        // ...
55    },
56]""",
57    "breakpointLocation": "NONE",
58    "customData": {},
59}
60
61# Run the Actor and wait for it to finish
62run = client.actor("apify/web-scraper").call(run_input=run_input)
63
64# Fetch and print Actor results from the run's dataset (if there are any)
65print("💾 Check your data here: https://console.apify.com/storage/datasets/" + run["defaultDatasetId"])
66for item in client.dataset(run["defaultDatasetId"]).iterate_items():
67    print(item)
68
69# 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/python/docs/quick-start