Test OG image
Try for free
No credit card required
Go to Store
.editorconfig
1root = true
2
3[*]
4indent_style = space
5indent_size = 4
6charset = utf-8
7trim_trailing_whitespace = true
8insert_final_newline = true
9end_of_line = lf.eslintrc
1{
2 "extends": "@apify"
3}.gitignore
1# This file tells Git which files shouldn't be added to source control
2
3.idea
4node_modulesDockerfile
1# First, specify the base Docker image. You can read more about
2# the available images at https://sdk.apify.com/docs/guides/docker-images
3# You can also use any other image from Docker Hub.
4FROM apify/actor-node-puppeteer-chrome:16
5
6# Second, copy just package.json and package-lock.json since it should be
7# the only file that affects "npm install" in the next step, to speed up the build
8COPY package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14 && npm install --only=prod --no-optional \
15 && echo "Installed NPM packages:" \
16 && (npm list --all || true) \
17 && echo "Node.js version:" \
18 && node --version \
19 && echo "NPM version:" \
20 && npm --version
21
22# Next, copy the remaining files and directories with the source code.
23# Since we do this after NPM install, quick build will be really fast
24# for most source file changes.
25COPY . ./
26
27# Optionally, specify how to launch the source code of your actor.
28# By default, Apify's base Docker images define the CMD instruction
29# that runs the Node.js source code using the command specified
30# in the "scripts.start" section of the package.json file.
31# In short, the instruction looks something like this:
32#
33# CMD npm startINPUT_SCHEMA.json
1{
2 "title": "PuppeteerCrawler Template",
3 "description": "lorem ipsum",
4 "type": "object",
5 "schemaVersion": 1,
6 "properties": {
7 "startUrls": {
8 "title": "Start URLs",
9 "type": "array",
10 "description": "URLs to start with.",
11 "editor": "requestListSources",
12 "prefill": [
13 { "url": "https://apify.com" }
14 ]
15 }
16 },
17 "required": [
18 "startUrls"
19 ]
20}apify.json
1{
2 "env": { "npm_config_loglevel": "silent" }
3}main.js
1/**
2 * This template is a production ready boilerplate for developing with `PuppeteerCrawler`.
3 * Use this to bootstrap your projects using the most up-to-date code.
4 * If you're looking for examples or want to learn more, see README.
5 */
6
7const Apify = require('apify');
8const { handleStart, handleList, handleDetail } = require('./src/routes');
9
10const { utils: { log } } = Apify;
11
12Apify.main(async () => {
13 const { startUrls } = await Apify.getInput();
14
15 const requestList = await Apify.openRequestList('start-urls', startUrls);
16 const requestQueue = await Apify.openRequestQueue();
17 const proxyConfiguration = await Apify.createProxyConfiguration();
18
19 const crawler = new Apify.PuppeteerCrawler({
20 requestList,
21 requestQueue,
22 proxyConfiguration,
23 launchContext: {
24 // Chrome with stealth should work for most websites.
25 // If it doesn't, feel free to remove this.
26 useChrome: true,
27 stealth: true,
28 },
29 handlePageFunction: async (context) => {
30 const { url, userData: { label } } = context.request;
31 log.info('Page opened.', { label, url });
32 switch (label) {
33 case 'LIST':
34 return handleList(context);
35 case 'DETAIL':
36 return handleDetail(context);
37 default:
38 return handleStart(context);
39 }
40 },
41 });
42
43 log.info('Starting the crawl.');
44 await crawler.run();
45 log.info('Crawl finished.');
46});package.json
1{
2 "name": "project-puppeteer-crawler",
3 "version": "0.0.1",
4 "description": "This is a boilerplate of an Apify actor.",
5 "dependencies": {
6 "apify": "^2.0.7",
7 "puppeteer": "*"
8 },
9 "devDependencies": {
10 "@apify/eslint-config": "^0.1.3",
11 "eslint": "^7.0.0"
12 },
13 "scripts": {
14 "start": "node main.js",
15 "lint": "./node_modules/.bin/eslint ./src --ext .js,.jsx",
16 "lint:fix": "./node_modules/.bin/eslint ./src --ext .js,.jsx --fix",
17 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
18 },
19 "author": "It's not you it's me",
20 "license": "ISC"
21}src/routes.js
1const Apify = require('apify');
2
3const { utils: { log } } = Apify;
4
5exports.handleStart = async ({ request, page }) => {
6 // Handle Start URLs
7};
8
9exports.handleList = async ({ request, page }) => {
10 // Handle pagination
11};
12
13exports.handleDetail = async ({ request, page }) => {
14 // Handle details
15};
brownies
