Python site crawler (Scrapy)

1"""Module defines the main entry point for the Apify Actor.
2
3Module defines the main coroutine for the Apify Scrapy Actor, executed from the __main__.py file. The coroutine
4processes the Actor's input and executes the Scrapy spider. Additionally, it updates Scrapy project settings by
5applying Apify-related settings. Which includes adding a custom scheduler, retry middleware, and an item pipeline
6for pushing data to the Apify dataset.
7
8Customization:
9--------------
10
11Feel free to customize this file to add specific functionality to the Actor, such as incorporating your own Scrapy
12components like spiders and handling Actor input. However, make sure you have a clear understanding of your
13modifications. For instance, removing `apply_apify_settings` break the integration between Scrapy and Apify.
14
15Documentation:
16--------------
17
18For an in-depth description of the Apify-Scrapy integration process, our Scrapy components, known limitations and
19other stuff, please refer to the following documentation page: https://docs.apify.com/cli/docs/integrating-scrapy.
20"""
21
22from __future__ import annotations
23
24from apify import Actor
25from apify.scrapy import apply_apify_settings
26from scrapy.crawler import AsyncCrawlerRunner
27
28# Import your Scrapy spider here.
29from .spiders import TitleSpider as Spider
30
31
32async def main() -> None:
33    """Apify Actor main coroutine for executing the Scrapy spider."""
34    async with Actor:
35        # Retrieve and process Actor input.
36        actor_input = await Actor.get_input() or {}
37        start_urls = [url['url'] for url in actor_input.get('startUrls', [])]
38        allowed_domains = actor_input.get('allowedDomains')
39        proxy_config = actor_input.get('proxyConfiguration')
40
41        # Apply Apify settings, which will override the Scrapy project settings.
42        settings = apply_apify_settings(proxy_config=proxy_config)
43
44        # Create AsyncCrawlerRunner and execute the Scrapy spider.
45        crawler_runner = AsyncCrawlerRunner(settings)
46        await crawler_runner.crawl(
47            Spider,
48            start_urls=start_urls,
49            allowed_domains=allowed_domains,
50        )