diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e22fa1b791..67d558f5a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,43 +82,11 @@ TODO: ## Documentation -We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). +We follow the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for code documentation. All user-facing classes and functions must be documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). -Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages. +Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with custom post-processing. Additional content is provided through markdown files in the `docs/` directory. The final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub pages. -To run the documentation locally, you need to have Node.js version 20 or higher installed. Once you have the correct version of Node.js, follow these steps: - -Navigate to the `website/` directory: - -```sh -cd website/ -``` - -Enable Corepack, which installs Yarn automatically: - -```sh -corepack enable -``` - -Build the API reference: - -```sh -./build_api_reference.sh -``` - -Install the necessary dependencies: - -```sh -yarn -``` - -Start the project in development mode with Hot Module Replacement (HMR): - -```sh -yarn start -``` - -Or using `make`: +To run the documentation locally, ensure you have `Node.js` 20+ installed, then run: ```sh make run-docs diff --git a/docs/deployment/code_examples/apify/crawler_as_actor_example.py b/docs/deployment/code_examples/apify/crawler_as_actor_example.py index d14356623d..53527d555b 100644 --- a/docs/deployment/code_examples/apify/crawler_as_actor_example.py +++ b/docs/deployment/code_examples/apify/crawler_as_actor_example.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -6,7 +8,7 @@ async def main() -> None: # Wrap the crawler code in an Actor context manager. async with Actor: - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: @@ -19,3 +21,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await context.enqueue_links() await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/deployment/code_examples/apify/get_public_url.py b/docs/deployment/code_examples/apify/get_public_url.py index 061cbf4e97..d12cfba300 100644 --- a/docs/deployment/code_examples/apify/get_public_url.py +++ b/docs/deployment/code_examples/apify/get_public_url.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -8,3 +10,7 @@ async def main() -> None: url = store.get_public_url('your-file') Actor.log.info(f'KVS public URL: {url}') # https://api.apify.com/v2/key-value-stores//records/your-file + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/deployment/code_examples/apify/log_with_config_example.py b/docs/deployment/code_examples/apify/log_with_config_example.py index c1241fdc2e..dfefa7b5ae 100644 --- a/docs/deployment/code_examples/apify/log_with_config_example.py +++ b/docs/deployment/code_examples/apify/log_with_config_example.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor, Configuration @@ -11,3 +13,7 @@ async def main() -> None: async with Actor(config): Actor.log.info('Hello from Apify platform!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/deployment/code_examples/apify/proxy_advanced_example.py b/docs/deployment/code_examples/apify/proxy_advanced_example.py index 91064e2fb1..1b5306bd39 100644 --- a/docs/deployment/code_examples/apify/proxy_advanced_example.py +++ b/docs/deployment/code_examples/apify/proxy_advanced_example.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -12,3 +14,7 @@ async def main() -> None: ) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/deployment/code_examples/apify/proxy_example.py b/docs/deployment/code_examples/apify/proxy_example.py index 6535c2e672..d546c5cc45 100644 --- a/docs/deployment/code_examples/apify/proxy_example.py +++ b/docs/deployment/code_examples/apify/proxy_example.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -16,3 +18,7 @@ async def main() -> None: proxy_url = await proxy_configuration.new_url() Actor.log.info(f'Proxy URL: {proxy_url}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/add_data_to_dataset.mdx b/docs/examples/add_data_to_dataset.mdx index c5e3f1472d..aa4164cacf 100644 --- a/docs/examples/add_data_to_dataset.mdx +++ b/docs/examples/add_data_to_dataset.mdx @@ -6,24 +6,24 @@ title: Add data to dataset import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/add_data_to_dataset_bs.py'; -import PlaywrightExample from '!!raw-loader!./code_examples/add_data_to_dataset_pw.py'; -import DatasetExample from '!!raw-loader!./code_examples/add_data_to_dataset_dataset.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_bs.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_pw.py'; +import DatasetExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_dataset.py'; This example demonstrates how to store extracted data into datasets using the `context.push_data` helper function. If the specified dataset does not already exist, it will be created automatically. Additionally, you can save data to custom datasets by providing `dataset_id` or `dataset_name` parameters to the `push_data` function. - + {BeautifulSoupExample} - + - + {PlaywrightExample} - + @@ -35,6 +35,6 @@ Each item in the dataset will be stored in its own file within the following dir For more control, you can also open a dataset manually using the asynchronous constructor `Dataset.open` - + {DatasetExample} - + diff --git a/docs/examples/beautifulsoup_crawler.mdx b/docs/examples/beautifulsoup_crawler.mdx index 7ab92083ac..160e4c4d65 100644 --- a/docs/examples/beautifulsoup_crawler.mdx +++ b/docs/examples/beautifulsoup_crawler.mdx @@ -4,12 +4,12 @@ title: BeautifulSoup crawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/beautifulsoup_crawler.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler.py'; This example demonstrates how to use `BeautifulSoupCrawler` to crawl a list of URLs, load each URL using a plain HTTP request, parse the HTML using the [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) library and extract some data from it - the page title and all `

`, `

` and `

` tags. This setup is perfect for scraping specific elements from web pages. Thanks to the well-known BeautifulSoup, you can easily navigate the HTML structure and retrieve the data you need with minimal code. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request. - + {BeautifulSoupExample} - + diff --git a/docs/examples/capture_screenshot_using_playwright.mdx b/docs/examples/capture_screenshot_using_playwright.mdx index ffbaffcd93..614693b1e8 100644 --- a/docs/examples/capture_screenshot_using_playwright.mdx +++ b/docs/examples/capture_screenshot_using_playwright.mdx @@ -4,9 +4,9 @@ title: Capture screenshots using Playwright --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import CaptureScreenshotExample from '!!raw-loader!./code_examples/capture_screenshot_using_playwright.py'; +import CaptureScreenshotExample from '!!raw-loader!roa-loader!./code_examples/capture_screenshot_using_playwright.py'; This example demonstrates how to capture screenshots of web pages using `PlaywrightCrawler` and store them in the key-value store. @@ -14,6 +14,6 @@ The `PlaywrightCrawler` is confi The captured screenshots are stored in the key-value store, which is suitable for managing and storing files in various formats. In this case, screenshots are stored as PNG images with a unique key generated from the URL of the page. - + {CaptureScreenshotExample} - + diff --git a/docs/examples/code_examples/adaptive_playwright_crawler.py b/docs/examples/code_examples/adaptive_playwright_crawler.py index add09041b2..f2851d502b 100644 --- a/docs/examples/code_examples/adaptive_playwright_crawler.py +++ b/docs/examples/code_examples/adaptive_playwright_crawler.py @@ -14,7 +14,7 @@ async def main() -> None: # Crawler created by following factory method will use `beautifulsoup` # for parsing static content. crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( - max_requests_per_crawl=5, playwright_crawler_specific_kwargs={'headless': False} + max_requests_per_crawl=10, playwright_crawler_specific_kwargs={'headless': False} ) @crawler.router.default_handler diff --git a/docs/examples/code_examples/add_data_to_dataset_dataset.py b/docs/examples/code_examples/add_data_to_dataset_dataset.py index 66234d2953..b1d9aba923 100644 --- a/docs/examples/code_examples/add_data_to_dataset_dataset.py +++ b/docs/examples/code_examples/add_data_to_dataset_dataset.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.storages import Dataset @@ -7,3 +9,7 @@ async def main() -> None: # Interact with dataset directly. await dataset.push_data({'key': 'value'}) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/fill_and_submit_web_form_request.py b/docs/examples/code_examples/fill_and_submit_web_form_request.py index 0c992f8294..14dc6c479d 100644 --- a/docs/examples/code_examples/fill_and_submit_web_form_request.py +++ b/docs/examples/code_examples/fill_and_submit_web_form_request.py @@ -1,21 +1,28 @@ +import asyncio from urllib.parse import urlencode from crawlee import Request -# Prepare a POST request to the form endpoint. -request = Request.from_url( - url='https://httpbin.org/post', - method='POST', - headers={'content-type': 'application/x-www-form-urlencoded'}, - payload=urlencode( - { - 'custname': 'John Doe', - 'custtel': '1234567890', - 'custemail': 'johndoe@example.com', - 'size': 'large', - 'topping': ['bacon', 'cheese', 'mushroom'], - 'delivery': '13:00', - 'comments': 'Please ring the doorbell upon arrival.', - } - ).encode(), -) + +async def main() -> None: + # Prepare a POST request to the form endpoint. + request = Request.from_url( + url='https://httpbin.org/post', + method='POST', + headers={'content-type': 'application/x-www-form-urlencoded'}, + payload=urlencode( + { + 'custname': 'John Doe', + 'custtel': '1234567890', + 'custemail': 'johndoe@example.com', + 'size': 'large', + 'topping': ['bacon', 'cheese', 'mushroom'], + 'delivery': '13:00', + 'comments': 'Please ring the doorbell upon arrival.', + } + ).encode(), + ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/crawl_all_links_on_website.mdx b/docs/examples/crawl_all_links_on_website.mdx index f50ea953a9..f17c63920f 100644 --- a/docs/examples/crawl_all_links_on_website.mdx +++ b/docs/examples/crawl_all_links_on_website.mdx @@ -6,10 +6,10 @@ title: Crawl all links on website import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_all_links_on_website_bs.py'; -import PlaywrightExample from '!!raw-loader!./code_examples/crawl_all_links_on_website_pw.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_bs.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_pw.py'; This example uses the `enqueue_links` helper to add new links to the `RequestQueue` as the crawler navigates from page to page. By automatically discovering and enqueuing all links on a given page, the crawler can systematically scrape an entire website. This approach is ideal for web scraping tasks where you need to collect data from multiple interconnected pages. @@ -21,13 +21,13 @@ If no options are given, by default the method will only add links that are unde - + {BeautifulSoupExample} - + - + {PlaywrightExample} - + diff --git a/docs/examples/crawl_multiple_urls.mdx b/docs/examples/crawl_multiple_urls.mdx index b5c1c7f8f7..2d3d370283 100644 --- a/docs/examples/crawl_multiple_urls.mdx +++ b/docs/examples/crawl_multiple_urls.mdx @@ -6,22 +6,22 @@ title: Crawl multiple URLs import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_multiple_urls_bs.py'; -import PlaywrightExample from '!!raw-loader!./code_examples/crawl_multiple_urls_pw.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_bs.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_pw.py'; This example demonstrates how to crawl a specified list of URLs using different crawlers. You'll learn how to set up the crawler, define a request handler, and run the crawler with multiple URLs. This setup is useful for scraping data from multiple pages or websites concurrently. - + {BeautifulSoupExample} - + - + {PlaywrightExample} - + diff --git a/docs/examples/crawl_specific_links_on_website.mdx b/docs/examples/crawl_specific_links_on_website.mdx index b2e206b811..c1dcd8446c 100644 --- a/docs/examples/crawl_specific_links_on_website.mdx +++ b/docs/examples/crawl_specific_links_on_website.mdx @@ -6,22 +6,22 @@ title: Crawl specific links on website import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_specific_links_on_website_bs.py'; -import PlaywrightExample from '!!raw-loader!./code_examples/crawl_specific_links_on_website_pw.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_bs.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_pw.py'; This example demonstrates how to crawl a website while targeting specific patterns of links. By utilizing the `enqueue_links` helper, you can pass `include` or `exclude` parameters to improve your crawling strategy. This approach ensures that only the links matching the specified patterns are added to the `RequestQueue`. Both `include` and `exclude` support lists of globs or regular expressions. This functionality is great for focusing on relevant sections of a website and avoiding scraping unnecessary or irrelevant content. - + {BeautifulSoupExample} - + - + {PlaywrightExample} - + diff --git a/docs/examples/crawl_website_with_relative_links.mdx b/docs/examples/crawl_website_with_relative_links.mdx index 96d1301924..4cf7bee845 100644 --- a/docs/examples/crawl_website_with_relative_links.mdx +++ b/docs/examples/crawl_website_with_relative_links.mdx @@ -6,12 +6,12 @@ title: Crawl website with relative links import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import AllLinksExample from '!!raw-loader!./code_examples/crawl_website_with_relative_links_all_links.py'; -import SameDomainExample from '!!raw-loader!./code_examples/crawl_website_with_relative_links_same_domain.py'; -import SameHostnameExample from '!!raw-loader!./code_examples/crawl_website_with_relative_links_same_hostname.py'; -import SameOriginExample from '!!raw-loader!./code_examples/crawl_website_with_relative_links_same_origin.py'; +import AllLinksExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_all_links.py'; +import SameDomainExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_domain.py'; +import SameHostnameExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_hostname.py'; +import SameOriginExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_origin.py'; When crawling a website, you may encounter various types of links that you wish to include in your crawl. To facilitate this, we provide the `enqueue_links` method on the crawler context, which will automatically find and add these links to the crawler's `RequestQueue`. This method simplifies the process of handling different types of links, including relative links, by automatically resolving them based on the page's context. @@ -30,23 +30,23 @@ For these examples, we are using the `B - + {AllLinksExample} - + - + {SameDomainExample} - + - + {SameHostnameExample} - + - + {SameOriginExample} - + diff --git a/docs/examples/crawler_keep_alive.mdx b/docs/examples/crawler_keep_alive.mdx index 60127d9849..2e6c6640c7 100644 --- a/docs/examples/crawler_keep_alive.mdx +++ b/docs/examples/crawler_keep_alive.mdx @@ -4,12 +4,12 @@ title: Keep a Crawler alive waiting for more requests --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/beautifulsoup_crawler_keep_alive.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_keep_alive.py'; This example demonstrates how to keep crawler alive even when there are no requests at the moment by using `keep_alive=True` argument of `BasicCrawler.__init__`. This is available to all crawlers that inherit from `BasicCrawler` and in the example below it is shown on `BeautifulSoupCrawler`. To stop the crawler that was started with `keep_alive=True` you can call `crawler.stop()`. - + {BeautifulSoupExample} - + diff --git a/docs/examples/crawler_stop.mdx b/docs/examples/crawler_stop.mdx index 9a0f635ab9..4ea7f28565 100644 --- a/docs/examples/crawler_stop.mdx +++ b/docs/examples/crawler_stop.mdx @@ -4,12 +4,12 @@ title: Stopping a Crawler with stop method --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/beautifulsoup_crawler_stop.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_stop.py'; This example demonstrates how to use `stop` method of `BasicCrawler` to stop crawler once the crawler finds what it is looking for. This method is available to all crawlers that inherit from `BasicCrawler` and in the example below it is shown on `BeautifulSoupCrawler`. Simply call `crawler.stop()` to stop the crawler. It will not continue to crawl through new requests. Requests that are already being concurrently processed are going to get finished. It is possible to call `stop` method with optional argument `reason` that is a string that will be used in logs and it can improve logs readability especially if you have multiple different conditions for triggering `stop`. - + {BeautifulSoupExample} - + diff --git a/docs/examples/export_entire_dataset_to_file.mdx b/docs/examples/export_entire_dataset_to_file.mdx index 5c673e90ff..72418ebe66 100644 --- a/docs/examples/export_entire_dataset_to_file.mdx +++ b/docs/examples/export_entire_dataset_to_file.mdx @@ -6,10 +6,10 @@ title: Export entire dataset to file import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import JsonExample from '!!raw-loader!./code_examples/export_entire_dataset_to_file_json.py'; -import CsvExample from '!!raw-loader!./code_examples/export_entire_dataset_to_file_csv.py'; +import JsonExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_json.py'; +import CsvExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_csv.py'; This example demonstrates how to use the `BasicCrawler.export_data` method of the crawler to export the entire default dataset to a single file. This method supports exporting data in either CSV or JSON format. @@ -21,13 +21,13 @@ For these examples, we are using the `B - + {JsonExample} - + - + {CsvExample} - + diff --git a/docs/examples/fill_and_submit_web_form.mdx b/docs/examples/fill_and_submit_web_form.mdx index f8b1fdbc1f..841a2616ee 100644 --- a/docs/examples/fill_and_submit_web_form.mdx +++ b/docs/examples/fill_and_submit_web_form.mdx @@ -6,10 +6,10 @@ title: Fill and submit web form import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RequestExample from '!!raw-loader!./code_examples/fill_and_submit_web_form_request.py'; -import CrawlerExample from '!!raw-loader!./code_examples/fill_and_submit_web_form_crawler.py'; +import RequestExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_request.py'; +import CrawlerExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_crawler.py'; This example demonstrates how to fill and submit a web form using the `HttpCrawler` crawler. The same approach applies to any crawler that inherits from it, such as the `BeautifulSoupCrawler` or `ParselCrawler`. @@ -42,9 +42,9 @@ The "Payload" tab will display the form fields and their submitted values. This Now, let's create a POST request with the form fields and their values using the `Request` class, specifically its `Request.from_url` constructor: - + {RequestExample} - + Alternatively, you can send form data as URL parameters using the `url` argument. It depends on the form and how it is implemented. However, sending the data as a POST request body using the `payload` is generally a better approach. @@ -52,9 +52,9 @@ Alternatively, you can send form data as URL parameters using the `url` argument Finally, let's implement the crawler and run it with the prepared request. Although we are using the `HttpCrawler`, the process is the same for any crawler that inherits from it. - + {CrawlerExample} - + ## Running the crawler diff --git a/docs/examples/json_logging.mdx b/docs/examples/json_logging.mdx index 8db5a3a1c1..06dd2ac492 100644 --- a/docs/examples/json_logging.mdx +++ b/docs/examples/json_logging.mdx @@ -4,17 +4,17 @@ title: Сonfigure JSON logging --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import JsonLoggingExample from '!!raw-loader!./code_examples/configure_json_logging.py'; +import JsonLoggingExample from '!!raw-loader!roa-loader!./code_examples/configure_json_logging.py'; This example demonstrates how to configure JSON line (JSONL) logging with Crawlee. By using the `use_table_logs=False` parameter, you can disable table-formatted statistics logs, which makes it easier to parse logs with external tools or to serialize them as JSON. The example shows how to integrate with the popular [`loguru`](https://github.com/delgan/loguru) library to capture Crawlee logs and format them as JSONL (one JSON object per line). This approach works well when you need to collect logs for analysis, monitoring, or when integrating with logging platforms like ELK Stack, Grafana Loki, or similar systems. - + {JsonLoggingExample} - + Here's an example of what a crawler statistics log entry in JSONL format. diff --git a/docs/examples/parsel_crawler.mdx b/docs/examples/parsel_crawler.mdx index f59b58f62c..b0eca7eb28 100644 --- a/docs/examples/parsel_crawler.mdx +++ b/docs/examples/parsel_crawler.mdx @@ -4,12 +4,12 @@ title: Parsel crawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import ParselCrawlerExample from '!!raw-loader!./code_examples/parsel_crawler.py'; +import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler.py'; This example shows how to use `ParselCrawler` to crawl a website or a list of URLs. Each URL is loaded using a plain HTTP request and the response is parsed using [Parsel](https://pypi.org/project/parsel/) library which supports CSS and XPath selectors for HTML responses and JMESPath for JSON responses. We can extract data from all kinds of complex HTML structures using XPath. In this example, we will use Parsel to crawl github.com and extract page title, URL and emails found in the webpage. The default handler will scrape data from the current webpage and enqueue all the links found in the webpage for continuous scraping. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request. - + {ParselCrawlerExample} - + diff --git a/docs/examples/playwright_crawler.mdx b/docs/examples/playwright_crawler.mdx index 912e411bde..70b0bc8afb 100644 --- a/docs/examples/playwright_crawler.mdx +++ b/docs/examples/playwright_crawler.mdx @@ -4,9 +4,9 @@ title: Playwright crawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightCrawlerExample from '!!raw-loader!./code_examples/playwright_crawler.py'; +import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler.py'; This example demonstrates how to use `PlaywrightCrawler` to recursively scrape the Hacker news website using headless Chromium and Playwright. @@ -14,6 +14,6 @@ The `PlaywrightCrawler` manages A **pre-navigation hook** can be used to perform actions before navigating to the URL. This hook provides further flexibility in controlling environment and preparing for navigation. - + {PlaywrightCrawlerExample} - + diff --git a/docs/examples/playwright_crawler_adaptive.mdx b/docs/examples/playwright_crawler_adaptive.mdx index a967354516..c1f8875df8 100644 --- a/docs/examples/playwright_crawler_adaptive.mdx +++ b/docs/examples/playwright_crawler_adaptive.mdx @@ -4,9 +4,9 @@ title: AdaptivePlaywrightCrawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import AdaptivePlaywrightCrawlerExample from '!!raw-loader!./code_examples/adaptive_playwright_crawler.py'; +import AdaptivePlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/adaptive_playwright_crawler.py'; This example demonstrates how to use `AdaptivePlaywrightCrawler`. An `AdaptivePlaywrightCrawler` is a combination of `PlaywrightCrawler` and some implementation of HTTP-based crawler such as `ParselCrawler` or `BeautifulSoupCrawler`. It uses a more limited crawling context interface so that it is able to switch to HTTP-only crawling when it detects that it may bring a performance benefit. @@ -15,6 +15,6 @@ A [pre-navigation hook](/python/docs/guides/adaptive-playwright-crawler#page-con For more detailed description please see [AdaptivePlaywrightCrawler guide](/python/docs/guides/adaptive-playwright-crawler 'AdaptivePlaywrightCrawler guide') - + {AdaptivePlaywrightCrawlerExample} - + diff --git a/docs/examples/playwright_crawler_with_block_requests.mdx b/docs/examples/playwright_crawler_with_block_requests.mdx index e4f2ab33b7..d7d5e15928 100644 --- a/docs/examples/playwright_crawler_with_block_requests.mdx +++ b/docs/examples/playwright_crawler_with_block_requests.mdx @@ -4,9 +4,9 @@ title: Playwright crawler with block requests --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightBlockRequests from '!!raw-loader!./code_examples/playwright_block_requests.py'; +import PlaywrightBlockRequests from '!!raw-loader!roa-loader!./code_examples/playwright_block_requests.py'; This example demonstrates how to optimize your `PlaywrightCrawler` performance by blocking unnecessary network requests. @@ -22,6 +22,6 @@ By default, `block_requests` You can also replace the default patterns list with your own by providing `url_patterns`, or extend it by passing additional patterns in `extra_url_patterns`. - + {PlaywrightBlockRequests} - + diff --git a/docs/examples/playwright_crawler_with_camoufox.mdx b/docs/examples/playwright_crawler_with_camoufox.mdx index 9c4a30d2d2..b627c9ba34 100644 --- a/docs/examples/playwright_crawler_with_camoufox.mdx +++ b/docs/examples/playwright_crawler_with_camoufox.mdx @@ -4,9 +4,9 @@ title: Playwright crawler with Camoufox --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightCrawlerExampleWithCamoufox from '!!raw-loader!./code_examples/playwright_crawler_with_camoufox.py'; +import PlaywrightCrawlerExampleWithCamoufox from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_camoufox.py'; This example demonstrates how to integrate Camoufox into `PlaywrightCrawler` using `BrowserPool` with custom `PlaywrightBrowserPlugin`. @@ -21,6 +21,6 @@ For more details please refer to: https://github.com/daijro/camoufox/tree/main/p The example code after PlayWrightCrawler instantiation is similar to example describing the use of Playwright Crawler. The main difference is that in this example Camoufox will be used as the browser through BrowserPool. - + {PlaywrightCrawlerExampleWithCamoufox} - + diff --git a/docs/examples/playwright_crawler_with_fingerprint_generator.mdx b/docs/examples/playwright_crawler_with_fingerprint_generator.mdx index 0551149ec2..e1c41e4c06 100644 --- a/docs/examples/playwright_crawler_with_fingerprint_generator.mdx +++ b/docs/examples/playwright_crawler_with_fingerprint_generator.mdx @@ -4,14 +4,14 @@ title: Playwright crawler with fingerprint generator --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightCrawlerExample from '!!raw-loader!./code_examples/playwright_crawler_with_fingerprint_generator.py'; +import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_fingerprint_generator.py'; This example demonstrates how to use `PlaywrightCrawler` together with `FingerprintGenerator` that will populate several browser attributes to mimic real browser fingerprint. To read more about fingerprints please see: https://docs.apify.com/academy/anti-scraping/techniques/fingerprinting. You can implement your own fingerprint generator or use `DefaultFingerprintGenerator`. To use the generator initialize it with the desired fingerprint options. The generator will try to create fingerprint based on those options. Unspecified options will be automatically selected by the generator from the set of reasonable values. If some option is important for you, do not rely on the default and explicitly define it. - + {PlaywrightCrawlerExample} - + diff --git a/docs/guides/avoid_blocking.mdx b/docs/guides/avoid_blocking.mdx index daccf1c4d8..3ada4e2446 100644 --- a/docs/guides/avoid_blocking.mdx +++ b/docs/guides/avoid_blocking.mdx @@ -3,13 +3,15 @@ id: avoid-blocking title: Avoid getting blocked description: How to avoid getting blocked when scraping --- + import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightDefaultFingerprintGenerator from '!!raw-loader!./code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py'; -import PlaywrightDefaultFingerprintGeneratorWithArgs from '!!raw-loader!./code_examples/browser_fingerprint/default_fingerprint_generator_with_args.py'; -import PlaywrightWithCamoufox from '!!raw-loader!../examples/code_examples/playwright_crawler_with_camoufox.py'; +import PlaywrightDefaultFingerprintGenerator from '!!raw-loader!roa-loader!./code_examples/avoid_blocking/playwright_with_fingerprint_generator.py'; +import PlaywrightWithCamoufox from '!!raw-loader!roa-loader!../examples/code_examples/playwright_crawler_with_camoufox.py'; +import PlaywrightDefaultFingerprintGeneratorWithArgs from '!!raw-loader!./code_examples/avoid_blocking/default_fingerprint_generator_with_args.py'; A scraper might get blocked for numerous reasons. Let's narrow it down to the two main ones. The first is a bad or blocked IP address. You can learn about this topic in the [proxy management guide](./proxy-management). The second reason is [browser fingerprints](https://pixelprivacy.com/resources/browser-fingerprinting/) (or signatures), which we will explore more in this guide. Check the [Apify Academy anti-scraping course](https://docs.apify.com/academy/anti-scraping) to gain a deeper theoretical understanding of blocking and learn a few tips and tricks. @@ -19,9 +21,9 @@ Browser fingerprint is a collection of browser attributes and significant featur Changing browser fingerprints can be a tedious job. Luckily, Crawlee provides this feature with minimal configuration necessary - the usage of fingerprints in `PlaywrightCrawler` is enabled by default. You can customize the fingerprints by using the `fingerprint_generator` argument of the `PlaywrightCrawler.__init__`, either pass your own implementation of `FingerprintGenerator` or use `DefaultFingerprintGenerator`. - + {PlaywrightDefaultFingerprintGenerator} - + In certain cases we want to narrow down the fingerprints used - e.g. specify a certain operating system, locale or browser. This is also possible with Crawlee - the crawler can have the generation algorithm customized to reflect the particular browser version and many more. For description of fingerprint generation options please see `HeaderGeneratorOptions`, `ScreenOptions` and `DefaultFingerprintGenerator.__init__` See the example bellow: @@ -35,9 +37,9 @@ If you do not want to use fingerprints, then pass `fingerprint_generator=None` a In some cases even `PlaywrightCrawler` with fingerprints is not enough. You can try using `PlaywrightCrawler` together with [Camoufox](https://camoufox.com/). See the example integration below: - + {PlaywrightWithCamoufox} - + **Related links** diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py b/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py deleted file mode 100644 index 5dc12e54ef..0000000000 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py +++ /dev/null @@ -1,13 +0,0 @@ -from datetime import timedelta - -from crawlee.crawlers import AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext - -crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser() - - -@crawler.router.default_handler -async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: - # Locate element h2 within 5 seconds - h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000)) - # Do stuff with element found by the selector - context.log.info(h2) diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py b/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py deleted file mode 100644 index 68aa981eab..0000000000 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py +++ /dev/null @@ -1,8 +0,0 @@ -from crawlee.crawlers import AdaptivePlaywrightCrawler - -crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( - # Arguments relevant only for PlaywrightCrawler - playwright_crawler_specific_kwargs={'headless': False, 'browser_type': 'chromium'}, - # Common arguments relevant to all crawlers - max_crawl_depth=5, -) diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py b/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py deleted file mode 100644 index 1cb64576b3..0000000000 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py +++ /dev/null @@ -1,8 +0,0 @@ -from crawlee.crawlers import AdaptivePlaywrightCrawler - -crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( - # Arguments relevant only for PlaywrightCrawler - playwright_crawler_specific_kwargs={'headless': False, 'browser_type': 'chromium'}, - # Common arguments relevant to all crawlers - max_crawl_depth=5, -) diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py b/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py deleted file mode 100644 index 656997576d..0000000000 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py +++ /dev/null @@ -1,29 +0,0 @@ -from playwright.async_api import Route - -from crawlee.crawlers import ( - AdaptivePlaywrightCrawler, - AdaptivePlaywrightPreNavCrawlingContext, -) - -crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser() - - -@crawler.pre_navigation_hook -async def hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None: - """Hook executed both in static sub crawler and playwright sub crawler. - - Trying to access `context.page` in this hook would raise `AdaptiveContextError` - for pages crawled without playwright.""" - - context.log.info(f'pre navigation hook for: {context.request.url}') - - -@crawler.pre_navigation_hook(playwright_only=True) -async def hook_playwright(context: AdaptivePlaywrightPreNavCrawlingContext) -> None: - """Hook executed only in playwright sub crawler.""" - - async def some_routing_function(route: Route) -> None: - await route.continue_() - - await context.page.route('*/**', some_routing_function) - context.log.info(f'Playwright only pre navigation hook for: {context.request.url}') diff --git a/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py b/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py new file mode 100644 index 0000000000..a6d2072ad3 --- /dev/null +++ b/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py @@ -0,0 +1,20 @@ +import asyncio + +from crawlee.fingerprint_suite import ( + DefaultFingerprintGenerator, + HeaderGeneratorOptions, + ScreenOptions, +) + + +async def main() -> None: + fingerprint_generator = DefaultFingerprintGenerator( + header_options=HeaderGeneratorOptions(browsers=['chromium']), + screen_options=ScreenOptions(min_width=400), + ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py b/docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py similarity index 100% rename from docs/guides/code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py rename to docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py diff --git a/docs/guides/code_examples/browser_fingerprint/default_fingerprint_generator_with_args.py b/docs/guides/code_examples/browser_fingerprint/default_fingerprint_generator_with_args.py deleted file mode 100644 index 14241e76ab..0000000000 --- a/docs/guides/code_examples/browser_fingerprint/default_fingerprint_generator_with_args.py +++ /dev/null @@ -1,10 +0,0 @@ -from crawlee.fingerprint_suite import ( - DefaultFingerprintGenerator, - HeaderGeneratorOptions, - ScreenOptions, -) - -fingerprint_generator = DefaultFingerprintGenerator( - header_options=HeaderGeneratorOptions(browsers=['chromium']), - screen_options=ScreenOptions(min_width=400), -) diff --git a/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py b/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py index eac1f8850a..6db2fb589d 100644 --- a/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py +++ b/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py @@ -1,25 +1,35 @@ +import asyncio + from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin from crawlee.crawlers import PlaywrightCrawler -crawler = PlaywrightCrawler( - browser_pool=BrowserPool( - plugins=[ - PlaywrightBrowserPlugin( - browser_type='chromium', - browser_launch_options={ - 'headless': False, - 'channel': 'msedge', - 'slow_mo': 200, - }, - browser_new_context_options={ - 'color_scheme': 'dark', - 'extra_http_headers': { - 'Custom-Header': 'my-header', - 'Accept-Language': 'en', + +async def main() -> None: + crawler = PlaywrightCrawler( + browser_pool=BrowserPool( + plugins=[ + PlaywrightBrowserPlugin( + browser_type='chromium', + browser_launch_options={ + 'headless': False, + 'channel': 'msedge', + 'slow_mo': 200, + }, + browser_new_context_options={ + 'color_scheme': 'dark', + 'extra_http_headers': { + 'Custom-Header': 'my-header', + 'Accept-Language': 'en', + }, + 'user_agent': 'My-User-Agent', }, - 'user_agent': 'My-User-Agent', - }, - ) - ] + ) + ] + ) ) -) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/handler.py b/docs/guides/code_examples/playwright_crawler_adaptive/handler.py new file mode 100644 index 0000000000..ad88e054cd --- /dev/null +++ b/docs/guides/code_examples/playwright_crawler_adaptive/handler.py @@ -0,0 +1,21 @@ +import asyncio +from datetime import timedelta + +from crawlee.crawlers import AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext + + +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser() + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + # Locate element h2 within 5 seconds + h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000)) + # Do stuff with element found by the selector + context.log.info(h2) + + await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py new file mode 100644 index 0000000000..c0008d3a29 --- /dev/null +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py @@ -0,0 +1,21 @@ +import asyncio + +from crawlee.crawlers import AdaptivePlaywrightCrawler + + +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + # Arguments relevant only for PlaywrightCrawler + playwright_crawler_specific_kwargs={ + 'headless': False, + 'browser_type': 'chromium', + }, + # Common arguments relevant to all crawlers + max_crawl_depth=5, + ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py new file mode 100644 index 0000000000..c220d53be4 --- /dev/null +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py @@ -0,0 +1,21 @@ +import asyncio + +from crawlee.crawlers import AdaptivePlaywrightCrawler + + +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( + # Arguments relevant only for PlaywrightCrawler + playwright_crawler_specific_kwargs={ + 'headless': False, + 'browser_type': 'chromium', + }, + # Common arguments relevant to all crawlers + max_crawl_depth=5, + ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py similarity index 87% rename from docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py rename to docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py index f1d8ec1001..b07b1592ae 100644 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee import Request from crawlee._types import RequestHandlerRunResult from crawlee.crawlers import ( @@ -54,8 +56,15 @@ def result_comparator( ) # For example compare `push_data` calls. -crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( - rendering_type_predictor=CustomRenderingTypePredictor(), - result_checker=result_checker, - result_comparator=result_comparator, -) +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( + rendering_type_predictor=CustomRenderingTypePredictor(), + result_checker=result_checker, + result_comparator=result_comparator, + ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py b/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py new file mode 100644 index 0000000000..bd95bd9f8b --- /dev/null +++ b/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py @@ -0,0 +1,39 @@ +import asyncio + +from playwright.async_api import Route + +from crawlee.crawlers import ( + AdaptivePlaywrightCrawler, + AdaptivePlaywrightPreNavCrawlingContext, +) + + +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser() + + @crawler.pre_navigation_hook + async def hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None: + """Hook executed both in static sub crawler and playwright sub crawler. + + Trying to access `context.page` in this hook would raise `AdaptiveContextError` + for pages crawled without playwright. + """ + context.log.info(f'pre navigation hook for: {context.request.url}') + + @crawler.pre_navigation_hook(playwright_only=True) + async def hook_playwright(context: AdaptivePlaywrightPreNavCrawlingContext) -> None: + """Hook executed only in playwright sub crawler.""" + + async def some_routing_function(route: Route) -> None: + await route.continue_() + + await context.page.route('*/**', some_routing_function) + context.log.info( + f'Playwright only pre navigation hook for: {context.request.url}' + ) + + await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/session_bs_example.py b/docs/guides/code_examples/proxy_management/session_bs_example.py index 2b0acd7f20..1243b0e488 100644 --- a/docs/guides/code_examples/proxy_management/session_bs_example.py +++ b/docs/guides/code_examples/proxy_management/session_bs_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import BeautifulSoupCrawler from crawlee.proxy_configuration import ProxyConfiguration @@ -14,3 +16,9 @@ async def main() -> None: proxy_configuration=proxy_configuration, use_session_pool=True, ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/session_pw_example.py b/docs/guides/code_examples/proxy_management/session_pw_example.py index de44b7911c..68309bda59 100644 --- a/docs/guides/code_examples/proxy_management/session_pw_example.py +++ b/docs/guides/code_examples/proxy_management/session_pw_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler from crawlee.proxy_configuration import ProxyConfiguration @@ -14,3 +16,9 @@ async def main() -> None: proxy_configuration=proxy_configuration, use_session_pool=True, ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/tandem_example.py b/docs/guides/code_examples/request_loaders/tandem_example.py index 0739d483ea..b0e83138ca 100644 --- a/docs/guides/code_examples/request_loaders/tandem_example.py +++ b/docs/guides/code_examples/request_loaders/tandem_example.py @@ -23,4 +23,5 @@ async def handler(context: ParselCrawlingContext) -> None: await crawler.run() -asyncio.run(main()) +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/tandem_example_explicit.py b/docs/guides/code_examples/request_loaders/tandem_example_explicit.py index ff34e1af5d..17ba20c392 100644 --- a/docs/guides/code_examples/request_loaders/tandem_example_explicit.py +++ b/docs/guides/code_examples/request_loaders/tandem_example_explicit.py @@ -26,4 +26,5 @@ async def handler(context: ParselCrawlingContext) -> None: await crawler.run() -asyncio.run(main()) +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py b/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py index 4b2a2e8c4e..cbc1130bc7 100644 --- a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py +++ b/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py @@ -18,6 +18,8 @@ async def main() -> None: concurrency_settings=concurrency_settings, ) + # ... + if __name__ == '__main__': asyncio.run(main()) diff --git a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py b/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py index 2cbac6cfc0..4d491446d0 100644 --- a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py +++ b/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py @@ -20,6 +20,8 @@ async def main() -> None: concurrency_settings=concurrency_settings, ) + # ... + if __name__ == '__main__': asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/session_management_basic.py b/docs/guides/code_examples/session_management/sm_basic.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_basic.py rename to docs/guides/code_examples/session_management/sm_basic.py diff --git a/docs/guides/code_examples/session_management/session_management_beautifulsoup.py b/docs/guides/code_examples/session_management/sm_beautifulsoup.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_beautifulsoup.py rename to docs/guides/code_examples/session_management/sm_beautifulsoup.py diff --git a/docs/guides/code_examples/session_management/session_management_http.py b/docs/guides/code_examples/session_management/sm_http.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_http.py rename to docs/guides/code_examples/session_management/sm_http.py diff --git a/docs/guides/code_examples/session_management/session_management_parsel.py b/docs/guides/code_examples/session_management/sm_parsel.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_parsel.py rename to docs/guides/code_examples/session_management/sm_parsel.py diff --git a/docs/guides/code_examples/session_management/session_management_playwright.py b/docs/guides/code_examples/session_management/sm_playwright.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_playwright.py rename to docs/guides/code_examples/session_management/sm_playwright.py diff --git a/docs/guides/code_examples/session_management/session_management_standalone.py b/docs/guides/code_examples/session_management/sm_standalone.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_standalone.py rename to docs/guides/code_examples/session_management/sm_standalone.py diff --git a/docs/guides/http_clients.mdx b/docs/guides/http_clients.mdx index d6ed89c5b9..2d79dabf8d 100644 --- a/docs/guides/http_clients.mdx +++ b/docs/guides/http_clients.mdx @@ -7,10 +7,10 @@ description: Crawlee supports multiple HTTP clients when making requests. import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BsCurlImpersonateExample from '!!raw-loader!./code_examples/http_clients/curl_impersonate_example.py'; -import BsHttpxExample from '!!raw-loader!./code_examples/http_clients/httpx_example.py'; +import BsCurlImpersonateExample from '!!raw-loader!roa-loader!./code_examples/http_clients/curl_impersonate_example.py'; +import BsHttpxExample from '!!raw-loader!roa-loader!./code_examples/http_clients/httpx_example.py'; HTTP clients are utilized by the HTTP-based crawlers (e.g. `BeautifulSoupCrawler`) to communicate with web servers. They use external HTTP libraries for communication, rather than a browser. Examples of such libraries include [httpx](https://pypi.org/project/httpx/), [aiohttp](https://pypi.org/project/aiohttp/) or [curl-cffi](https://pypi.org/project/curl-cffi/). After retrieving page content, an HTML parsing library is typically used to facilitate data extraction. Examples of such libraries are [beautifulsoup](https://pypi.org/project/beautifulsoup4/), [parsel](https://pypi.org/project/parsel/), [selectolax](https://pypi.org/project/selectolax/) or [pyquery](https://pypi.org/project/pyquery/). These crawlers are faster than browser-based crawlers but they cannot execute client-side JavaScript. @@ -20,14 +20,14 @@ In Crawlee we currently have two HTTP clients: - + {BsHttpxExample} - + - + {BsCurlImpersonateExample} - + diff --git a/docs/guides/http_crawlers.mdx b/docs/guides/http_crawlers.mdx index c8479699ba..42541fe456 100644 --- a/docs/guides/http_crawlers.mdx +++ b/docs/guides/http_crawlers.mdx @@ -7,29 +7,32 @@ description: Crawlee supports multiple HTTP crawlers that can be used to extract import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; Generic class `AbstractHttpCrawler` is parent to `BeautifulSoupCrawler`, `ParselCrawler` and `HttpCrawler` and it could be used as parent for your crawler with custom content parsing requirements. It already includes almost all the functionality to crawl webpages and the only missing part is the parser that should be used to parse HTTP responses, and a context dataclass that defines what context helpers will be available to user handler functions. ## `BeautifulSoupCrawler` + `BeautifulSoupCrawler` uses `BeautifulSoupParser` to parse the HTTP response and makes it available in `BeautifulSoupCrawlingContext` in the `.soup` or `.parsed_content` attribute. ## `ParselCrawler` + `ParselCrawler` uses `ParselParser` to parse the HTTP response and makes it available in `ParselCrawlingContext` in the `.selector` or `.parsed_content` attribute. ## `HttpCrawler` + `HttpCrawler` uses `NoParser` that does not parse the HTTP response at all and is to be used if no parsing is required. -## Creating your own HTTP crawler. +## Creating your own HTTP crawler + ### Why? + In case you want to use some custom parser for parsing HTTP responses, and the rest of the `AbstractHttpCrawler` functionality suit your needs. ### How? + You need to define at least 2 new classes and decide what will be the type returned by the parser's `parse` method. -Parser will inherit from `AbstractHttpParser` and it will need to implement all it's abstract methods. -Crawler will inherit from `AbstractHttpCrawler` and it will need to implement all it's abstract methods. -Newly defined parser is then used in the `parser` argument of `AbstractHttpCrawler.__init__` method. +Parser will inherit from `AbstractHttpParser` and it will need to implement all it's abstract methods. Crawler will inherit from `AbstractHttpCrawler` and it will need to implement all it's abstract methods. Newly defined parser is then used in the `parser` argument of `AbstractHttpCrawler.__init__` method. To get better idea and as an example please see one of our own HTTP-based crawlers mentioned above. diff --git a/docs/guides/playwright_crawler.mdx b/docs/guides/playwright_crawler.mdx index 8fc2ccf40a..124f09e8ad 100644 --- a/docs/guides/playwright_crawler.mdx +++ b/docs/guides/playwright_crawler.mdx @@ -6,11 +6,13 @@ description: How to use the PlaywrightCrawler and its related components. import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import MultipleLaunchExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/multiple_launch_example.py'; +import BrowserConfigurationExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/browser_configuration_example.py'; +import PreNavigationExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/pre_navigation_hook_example.py'; -import MultipleLaunchExample from '!!raw-loader!./code_examples/playwright_crawler/multiple_launch_example.py'; -import BrowserConfigurationExample from '!!raw-loader!./code_examples/playwright_crawler/browser_configuration_example.py'; import PluginBrowserConfigExample from '!!raw-loader!./code_examples/playwright_crawler/plugin_browser_configuration_example.py'; -import PreNavigationExample from '!!raw-loader!./code_examples/playwright_crawler/pre_navigation_hook_example.py'; A `PlaywrightCrawler` is a browser-based crawler. In contrast to HTTP-based crawlers like `ParselCrawler` or `BeautifulSoupCrawler`, it uses a real browser to render pages and extract data. It is built on top of the [Playwright](https://playwright.dev/python/) browser automation library. While browser-based crawlers are typically slower and less efficient than HTTP-based crawlers, they can handle dynamic, client-side rendered sites that standard HTTP-based crawlers cannot manage. @@ -35,17 +37,17 @@ The `PlaywrightCrawler` uses oth The `BrowserPool` allows you to manage multiple browsers. Each browser instance is managed by a separate `PlaywrightBrowserPlugin` and can be configured independently. This is useful for scenarios like testing multiple configurations or implementing browser rotation to help avoid blocks or detect different site behaviors. - + {MultipleLaunchExample} - + ## Browser launch and context configuration The `PlaywrightBrowserPlugin` provides access to all relevant Playwright configuration options for both [browser launches](https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch) and [new browser contexts](https://playwright.dev/python/docs/api/class-browser#browser-new-context). You can specify these options in the constructor of `PlaywrightBrowserPlugin` or `PlaywrightCrawler`: - + {BrowserConfigurationExample} - + You can also configure each plugin used by `BrowserPool`: @@ -59,9 +61,9 @@ For an example of how to implement a custom browser plugin, see the [Camoufox ex In some use cases, you may need to configure the [page](https://playwright.dev/python/docs/api/class-page) before it navigates to the target URL. For instance, you might set navigation timeouts or manipulate other page-level settings. For such cases you can use the `pre_navigation_hook` method of the `PlaywrightCrawler`. This method is called before the page navigates to the target URL and allows you to configure the page instance. - + {PreNavigationExample} - + ## Conclusion diff --git a/docs/guides/playwright_crawler_adaptive.mdx b/docs/guides/playwright_crawler_adaptive.mdx index 583456e9a0..696bc15163 100644 --- a/docs/guides/playwright_crawler_adaptive.mdx +++ b/docs/guides/playwright_crawler_adaptive.mdx @@ -8,14 +8,14 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import AdaptivePlaywrightCrawlerInitBeautifulSoup from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py'; -import AdaptivePlaywrightCrawlerInitParsel from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py'; -import AdaptivePlaywrightCrawlerInitPrediction from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py'; -import AdaptivePlaywrightCrawlerHandler from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py'; -import AdaptivePlaywrightCrawlerPreNavHooks from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py'; - +import AdaptivePlaywrightCrawlerHandler from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_adaptive/handler.py'; +import AdaptivePlaywrightCrawlerPreNavHooks from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_adaptive/pre_nav_hooks.py'; +import AdaptivePlaywrightCrawlerInitBeautifulSoup from '!!raw-loader!./code_examples/playwright_crawler_adaptive/init_beautifulsoup.py'; +import AdaptivePlaywrightCrawlerInitParsel from '!!raw-loader!./code_examples/playwright_crawler_adaptive/init_parsel.py'; +import AdaptivePlaywrightCrawlerInitPrediction from '!!raw-loader!./code_examples/playwright_crawler_adaptive/init_prediction.py'; An `AdaptivePlaywrightCrawler` is a combination of `PlaywrightCrawler` and some implementation of HTTP-based crawler such as `ParselCrawler` or `BeautifulSoupCrawler`. It uses a more limited crawling context interface so that it is able to switch to HTTP-only crawling when it detects that it may bring a performance benefit. @@ -42,30 +42,29 @@ Request handler for `AdaptivePlayw See the following example about how to create request handler and use context helpers: - + {AdaptivePlaywrightCrawlerHandler} - - + ## Crawler configuration + To use `AdaptivePlaywrightCrawler` it is recommended to use one of the prepared factory methods that will create the crawler with specific HTTP-based sub crawler variant: `AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser` or `AdaptivePlaywrightCrawler.with_parsel_static_parser`. `AdaptivePlaywrightCrawler` is internally composed of two sub crawlers and you can do a detailed configuration of both of them. For detailed configuration options of the sub crawlers, please refer to their pages: `PlaywrightCrawler`, `ParselCrawler`, `BeautifulSoupCrawler`. In the following example you can see how to create and configure `AdaptivePlaywrightCrawler` with two different HTTP-based sub crawlers: - - - - {AdaptivePlaywrightCrawlerInitBeautifulSoup} - - - - - {AdaptivePlaywrightCrawlerInitParsel} - - + + + {AdaptivePlaywrightCrawlerInitBeautifulSoup} + + + + + {AdaptivePlaywrightCrawlerInitParsel} + + ### Prediction related arguments @@ -84,13 +83,12 @@ See the following example about how to pass prediction related arguments: {AdaptivePlaywrightCrawlerInitPrediction} - - ## Page configuration with pre-navigation hooks + In some use cases, you may need to configure the [page](https://playwright.dev/python/docs/api/class-page) before it navigates to the target URL. For instance, you might set navigation timeouts or manipulate other page-level settings. For such cases you can use the `pre_navigation_hook` method of the `AdaptivePlaywrightCrawler`. This method is called before the page navigates to the target URL and allows you to configure the page instance. Due to the dynamic nature of `AdaptivePlaywrightCrawler` it is possible that the hook will be executed for HTTP-based sub crawler or playwright-based sub crawler. Using [page](https://playwright.dev/python/docs/api/class-page) object for hook that will be executed on HTTP-based sub crawler will raise an exception. To overcome this you can use optional argument `playwright_only` = `True` when registering the hook. See the following example about how to register the pre navigation hooks: - + {AdaptivePlaywrightCrawlerPreNavHooks} - + diff --git a/docs/guides/proxy_management.mdx b/docs/guides/proxy_management.mdx index 64980f0b97..38385ac950 100644 --- a/docs/guides/proxy_management.mdx +++ b/docs/guides/proxy_management.mdx @@ -8,16 +8,18 @@ import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import QuickStartExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/quick_start_example.py'; +import IntegrationBsExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/integration_bs_example.py'; +import IntegrationPwExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/integration_pw_example.py'; +import TiersBsExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/tiers_bs_example.py'; +import TiersPwExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/tiers_pw_example.py'; +import InspectionBsExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/inspecting_bs_example.py'; +import InspectionPwExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/inspecting_pw_example.py'; -import QuickStartExample from '!!raw-loader!./code_examples/proxy_management/quick_start_example.py'; -import IntegrationBsExample from '!!raw-loader!./code_examples/proxy_management/integration_bs_example.py'; -import IntegrationPwExample from '!!raw-loader!./code_examples/proxy_management/integration_pw_example.py'; import SessionBsExample from '!!raw-loader!./code_examples/proxy_management/session_bs_example.py'; import SessionPwExample from '!!raw-loader!./code_examples/proxy_management/session_pw_example.py'; -import InspectionBsExample from '!!raw-loader!./code_examples/proxy_management/inspecting_bs_example.py'; -import InspectionPwExample from '!!raw-loader!./code_examples/proxy_management/inspecting_pw_example.py'; -import TiersBsExample from '!!raw-loader!./code_examples/proxy_management/tiers_bs_example.py'; -import TiersPwExample from '!!raw-loader!./code_examples/proxy_management/tiers_pw_example.py'; [IP address blocking](https://en.wikipedia.org/wiki/IP_address_blocking) is one of the oldest and most effective ways of preventing access to a website. It is therefore paramount for a good web scraping library to provide easy to use but powerful tools which can work around IP blocking. The most powerful weapon in our anti IP blocking arsenal is a [proxy server](https://en.wikipedia.org/wiki/Proxy_server). @@ -29,9 +31,9 @@ With Crawlee we can use our own proxy servers or proxy servers acquired from thi If you already have proxy URLs of your own, you can start using them immediately in only a few lines of code. - + {QuickStartExample} - + Examples of how to use our proxy URLs with crawlers are shown below in [Crawler integration](#crawler-integration) section. @@ -45,14 +47,14 @@ All our proxy needs are managed by the `P - + {IntegrationBsExample} - + - + {IntegrationPwExample} - + @@ -89,14 +91,14 @@ In an active tier, Crawlee will alternate between proxies in a round-robin fashi - + {TiersBsExample} - + - + {TiersPwExample} - + @@ -106,13 +108,13 @@ The `BeautifulSoupCrawler` an - + {InspectionBsExample} - + - + {InspectionPwExample} - + diff --git a/docs/guides/request_loaders.mdx b/docs/guides/request_loaders.mdx index fc17fbadbc..73fe374a62 100644 --- a/docs/guides/request_loaders.mdx +++ b/docs/guides/request_loaders.mdx @@ -7,12 +7,11 @@ description: How to manage the requests your crawler will go through. import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RlBasicExample from '!!raw-loader!./code_examples/request_loaders/rl_basic_example.py'; - -import TandemExample from '!!raw-loader!./code_examples/request_loaders/tandem_example.py'; -import ExplicitTandemExample from '!!raw-loader!./code_examples/request_loaders/tandem_example_explicit.py'; +import RlBasicExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/rl_basic_example.py'; +import TandemExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/tandem_example.py'; +import ExplicitTandemExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/tandem_example_explicit.py'; The [`request_loaders`](https://github.com/apify/crawlee-python/tree/master/src/crawlee/request_loaders) sub-package extends the functionality of the `RequestQueue`, providing additional tools for managing URLs. If you are new to Crawlee, and you do not know the `RequestQueue`, consider starting with the [Storages](https://crawlee.dev/python/docs/guides/storages) guide first. Request loaders define how requests are fetched and stored, enabling various use cases, such as reading URLs from files, external APIs or combining multiple sources together. @@ -109,9 +108,9 @@ The `RequestList` can accept an asynch Here is a basic example of working with the `RequestList`: - + {RlBasicExample} - + ## Request manager @@ -127,14 +126,14 @@ This sections describes the combination of the ` - + {ExplicitTandemExample} - + - + {TandemExample} - + diff --git a/docs/guides/scaling_crawlers.mdx b/docs/guides/scaling_crawlers.mdx index cbe12ad56e..5dce8ac640 100644 --- a/docs/guides/scaling_crawlers.mdx +++ b/docs/guides/scaling_crawlers.mdx @@ -5,10 +5,10 @@ description: Learn how to scale your crawlers by controlling concurrency and lim --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import MaxTasksPerMinuteExample from '!!raw-loader!./code_examples/scaling_crawlers/max_tasks_per_minute_example.py'; -import MinAndMaxConcurrencyExample from '!!raw-loader!./code_examples/scaling_crawlers/min_and_max_concurrency_example.py'; +import MaxTasksPerMinuteExample from '!!raw-loader!roa-loader!./code_examples/scaling_crawlers/max_tasks_per_minute_example.py'; +import MinAndMaxConcurrencyExample from '!!raw-loader!roa-loader!./code_examples/scaling_crawlers/min_and_max_concurrency_example.py'; As we build our crawler, we may want to control how many tasks it performs at any given time. In other words, how many requests it makes to the web we are trying to scrape. Crawlee offers several options to fine-tune the number of parallel tasks, limit the number of requests per minute, and optimize scaling based on available system resources. @@ -22,9 +22,9 @@ All of these options are available across all crawlers provided by Crawlee. In t The `max_tasks_per_minute` setting in `ConcurrencySettings` controls how many total tasks the crawler can process per minute. It ensures that tasks are spread evenly throughout the minute, preventing a sudden burst at the `max_concurrency` limit followed by idle time. By default, this is set to `Infinity`, meaning the crawler can run at full speed, limited only by `max_concurrency`. Use this option if you want to throttle your crawler to avoid overwhelming the target website with continuous requests. - + {MaxTasksPerMinuteExample} - + ## Minimum and maximum concurrency @@ -40,9 +40,9 @@ If you set `min_concurrency` too high compared to the available system resources The `desired_concurrency` option in the `ConcurrencySettings` specifies the initial number of parallel tasks to start with, assuming sufficient resources are available. It defaults to the same value as `min_concurrency`. - + {MinAndMaxConcurrencyExample} - + ## Autoscaled pool diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdx index 20b4dfd1b2..a3a1385db1 100644 --- a/docs/guides/session_management.mdx +++ b/docs/guides/session_management.mdx @@ -7,16 +7,16 @@ description: How to manage your cookies, proxy IP rotations and more. import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BasicSource from '!!raw-loader!./code_examples/session_management/session_management_basic.py'; -import HttpSource from '!!raw-loader!./code_examples/session_management/session_management_http.py'; -import BeautifulSoupSource from '!!raw-loader!./code_examples/session_management/session_management_beautifulsoup.py'; -import ParselSource from '!!raw-loader!./code_examples/session_management/session_management_parsel.py'; -import PlaywrightSource from '!!raw-loader!./code_examples/session_management/session_management_playwright.py'; -import StandaloneSource from '!!raw-loader!./code_examples/session_management/session_management_standalone.py'; -import OneSession from '!!raw-loader!./code_examples/session_management/one_session_http.py'; -import MultiSessions from '!!raw-loader!./code_examples/session_management/multi_sessions_http.py'; +import BasicSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_basic.py'; +import HttpSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_http.py'; +import BeautifulSoupSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_beautifulsoup.py'; +import ParselSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_parsel.py'; +import PlaywrightSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_playwright.py'; +import StandaloneSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_standalone.py'; +import OneSession from '!!raw-loader!roa-loader!./code_examples/session_management/one_session_http.py'; +import MultiSessions from '!!raw-loader!roa-loader!./code_examples/session_management/multi_sessions_http.py'; The `SessionPool` class provides a robust way to manage the rotation of proxy IP addresses, cookies, and other custom settings in Crawlee. Its primary advantage is the ability to filter out blocked or non-functional proxies, ensuring that your scraper avoids retrying requests through known problematic proxies. @@ -36,34 +36,34 @@ Now, let's explore examples of how to use the `S - + {BasicSource} - + - + {HttpSource} - + - + {BeautifulSoupSource} - + - + {ParselSource} - + - + {PlaywrightSource} - + - + {StandaloneSource} - + @@ -77,9 +77,9 @@ In some cases, you need full control over session usage. For example, when worki When working with a site that requires authentication, we typically don't want multiple sessions with different browser fingerprints or client parameters accessing the site. In this case, we need to configure the `SessionPool` appropriately: - + {OneSession} - + ## Binding requests to specific sessions @@ -89,6 +89,6 @@ In some cases, we need to achieve the same behavior but using multiple sessions To do this, use the `session_id` parameter for the `Request` object to bind a request to a specific session: - + {MultiSessions} - + diff --git a/docs/guides/storages.mdx b/docs/guides/storages.mdx index 4615a2ee83..3be168b683 100644 --- a/docs/guides/storages.mdx +++ b/docs/guides/storages.mdx @@ -7,24 +7,24 @@ description: How to work with storages in Crawlee, how to manage requests and ho import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RqBasicExample from '!!raw-loader!./code_examples/storages/rq_basic_example.py'; -import RqWithCrawlerExample from '!!raw-loader!./code_examples/storages/rq_with_crawler_example.py'; -import RqWithCrawlerExplicitExample from '!!raw-loader!./code_examples/storages/rq_with_crawler_explicit_example.py'; -import RqHelperAddRequestsExample from '!!raw-loader!./code_examples/storages/helper_add_requests_example.py'; -import RqHelperEnqueueLinksExample from '!!raw-loader!./code_examples/storages/helper_enqueue_links_example.py'; +import RqBasicExample from '!!raw-loader!roa-loader!./code_examples/storages/rq_basic_example.py'; +import RqWithCrawlerExample from '!!raw-loader!roa-loader!./code_examples/storages/rq_with_crawler_example.py'; +import RqWithCrawlerExplicitExample from '!!raw-loader!roa-loader!./code_examples/storages/rq_with_crawler_explicit_example.py'; +import RqHelperAddRequestsExample from '!!raw-loader!roa-loader!./code_examples/storages/helper_add_requests_example.py'; +import RqHelperEnqueueLinksExample from '!!raw-loader!roa-loader!./code_examples/storages/helper_enqueue_links_example.py'; -import DatasetBasicExample from '!!raw-loader!./code_examples/storages/dataset_basic_example.py'; -import DatasetWithCrawlerExample from '!!raw-loader!./code_examples/storages/dataset_with_crawler_example.py'; -import DatasetWithCrawerExplicitExample from '!!raw-loader!./code_examples/storages/dataset_with_crawler_explicit_example.py'; +import DatasetBasicExample from '!!raw-loader!roa-loader!./code_examples/storages/dataset_basic_example.py'; +import DatasetWithCrawlerExample from '!!raw-loader!roa-loader!./code_examples/storages/dataset_with_crawler_example.py'; +import DatasetWithCrawerExplicitExample from '!!raw-loader!roa-loader!./code_examples/storages/dataset_with_crawler_explicit_example.py'; -import KvsBasicExample from '!!raw-loader!./code_examples/storages/kvs_basic_example.py'; -import KvsWithCrawlerExample from '!!raw-loader!./code_examples/storages/kvs_with_crawler_example.py'; -import KvsWithCrawlerExplicitExample from '!!raw-loader!./code_examples/storages/kvs_with_crawler_explicit_example.py'; +import KvsBasicExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_basic_example.py'; +import KvsWithCrawlerExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_with_crawler_example.py'; +import KvsWithCrawlerExplicitExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_with_crawler_explicit_example.py'; -import CleaningDoNotPurgeExample from '!!raw-loader!./code_examples/storages/cleaning_do_not_purge_example.py'; -import CleaningPurgeExplicitlyExample from '!!raw-loader!./code_examples/storages/cleaning_purge_explicitly_example.py'; +import CleaningDoNotPurgeExample from '!!raw-loader!roa-loader!./code_examples/storages/cleaning_do_not_purge_example.py'; +import CleaningPurgeExplicitlyExample from '!!raw-loader!roa-loader!./code_examples/storages/cleaning_purge_explicitly_example.py'; Crawlee offers multiple storage types for managing and persisting your crawling data. Request-oriented storages, such as the `RequestQueue`, help you store and deduplicate URLs, while result-oriented storages, like `Dataset` and `KeyValueStore`, focus on storing and retrieving scraping results. This guide helps you choose the storage type that suits your needs. @@ -75,19 +75,19 @@ The following code demonstrates the usage of the - + {RqBasicExample} - + - + {RqWithCrawlerExample} - + - + {RqWithCrawlerExplicitExample} - + @@ -100,14 +100,14 @@ Crawlee provides helper functions to simplify interactions with the - + {RqHelperAddRequestsExample} - + - + {RqHelperEnqueueLinksExample} - + @@ -136,19 +136,19 @@ The following code demonstrates basic operations of the dataset: - + {DatasetBasicExample} - + - + {DatasetWithCrawlerExample} - + - + {DatasetWithCrawerExplicitExample} - + @@ -176,19 +176,19 @@ The following code demonstrates the usage of the - + {KvsBasicExample} - + - + {KvsWithCrawlerExample} - + - + {KvsWithCrawlerExplicitExample} - + @@ -204,17 +204,17 @@ Crawlee provides the following helper function to simplify interactions with the Default storages are purged before the crawler starts, unless explicitly configured otherwise. For that case, see `Configuration.purge_on_start`. This cleanup happens as soon as a storage is accessed, either when you open a storage (e.g. using `RequestQueue.open`, `Dataset.open`, `KeyValueStore.open`) or when interacting with a storage through one of the helper functions (e.g. `push_data`), which implicitly opens the result storage. - + {CleaningDoNotPurgeExample} - + If you do not explicitly interact with storages in your code, the purging will occur automatically when the `BasicCrawler.run` method is invoked. If you need to purge storages earlier, you can call `MemoryStorageClient.purge_on_start` directly if you are using the default storage client. This method triggers the purging process for the underlying storage implementation you are currently using. - + {CleaningPurgeExplicitlyExample} - + ## Conclusion diff --git a/docs/introduction/02_first_crawler.mdx b/docs/introduction/02_first_crawler.mdx index 26dafd8ad2..203ab92146 100644 --- a/docs/introduction/02_first_crawler.mdx +++ b/docs/introduction/02_first_crawler.mdx @@ -4,11 +4,11 @@ title: First crawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RequestQueueExample from '!!raw-loader!./code_examples/02_request_queue.py'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/02_bs.py'; -import BeautifulSoupBetterExample from '!!raw-loader!./code_examples/02_bs_better.py'; +import RequestQueueExample from '!!raw-loader!roa-loader!./code_examples/02_request_queue.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/02_bs.py'; +import BeautifulSoupBetterExample from '!!raw-loader!roa-loader!./code_examples/02_bs_better.py'; Now, you will build your first crawler. But before you do, let's briefly introduce the Crawlee classes involved in the process. @@ -44,9 +44,9 @@ Let's put the theory into practice and start with something easy. Visit a page a Earlier you learned that the crawler uses a queue of requests as its source of URLs to crawl. Let's create it and add the first request. - + {RequestQueueExample} - + The `RequestQueue.add_request` method automatically converts the object with URL string to a `Request` instance. So now you have a `RequestQueue` that holds one request which points to `https://crawlee.dev`. @@ -64,9 +64,9 @@ Unless you have a good reason to start with a different one, you should try buil Let's continue with the earlier `RequestQueue` example. - + {BeautifulSoupExample} - + When you run the example, you will see the title of https://crawlee.dev printed to the log. What really happens is that `BeautifulSoupCrawler` first makes an HTTP request to `https://crawlee.dev`, then parses the received HTML with BeautifulSoup and makes it available as the `context` argument of the request handler. @@ -78,9 +78,9 @@ When you run the example, you will see the title of https://crawlee.dev printed Earlier we mentioned that you'll learn how to use the `BasicCrawler.add_requests` method to skip the request queue initialization. It's simple. Every crawler has an implicit `RequestQueue` instance, and you can add requests to it with the `BasicCrawler.add_requests` method. In fact, you can go even further and just use the first parameter of `crawler.run()`! - + {BeautifulSoupBetterExample} - + When you run this code, you'll see exactly the same output as with the earlier, longer example. The `RequestQueue` is still there, it's just managed by the crawler automatically. diff --git a/docs/introduction/03_adding_more_urls.mdx b/docs/introduction/03_adding_more_urls.mdx index 9e7f7bab58..a9669fb8a3 100644 --- a/docs/introduction/03_adding_more_urls.mdx +++ b/docs/introduction/03_adding_more_urls.mdx @@ -4,19 +4,19 @@ title: Adding more URLs --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import OriginalCodeExample from '!!raw-loader!./code_examples/03_original_code.py'; -import FindingNewLinksExample from '!!raw-loader!./code_examples/03_finding_new_links.py'; -import EnqueueStrategyExample from '!!raw-loader!./code_examples/03_enqueue_strategy.py'; -import GlobsExample from '!!raw-loader!./code_examples/03_globs.py'; -import TransformExample from '!!raw-loader!./code_examples/03_transform_request.py'; +import OriginalCodeExample from '!!raw-loader!roa-loader!./code_examples/03_original_code.py'; +import FindingNewLinksExample from '!!raw-loader!roa-loader!./code_examples/03_finding_new_links.py'; +import EnqueueStrategyExample from '!!raw-loader!roa-loader!./code_examples/03_enqueue_strategy.py'; +import GlobsExample from '!!raw-loader!roa-loader!./code_examples/03_globs.py'; +import TransformExample from '!!raw-loader!roa-loader!./code_examples/03_transform_request.py'; Previously you've built a very simple crawler that downloads HTML of a single page, reads its title and prints it to the console. This is the original source code: - + {OriginalCodeExample} - + Now you'll use the example from the previous section and improve on it. You'll add more URLs to the queue and thanks to that the crawler will keep going, finding new links, enqueuing them into the `RequestQueue` and then scraping them. @@ -58,9 +58,9 @@ There are numerous approaches to finding links to follow when crawling the web. Since this is the most common case, it is also the `enqueue_links` default. - + {FindingNewLinksExample} - + If you need to override the default selection of elements in `enqueue_links`, you can use the `selector` argument. @@ -80,9 +80,9 @@ await context.enqueue_links() The default behavior of `enqueue_links` is to stay on the same hostname. This **does not include subdomains**. To include subdomains in your crawl, use the `strategy` argument. The `strategy` argument is an instance of the `EnqueueStrategy` type alias. - + {EnqueueStrategyExample} - + When you run the code, you will see the crawler log the **title** of the first page, then the **enqueueing** message showing number of URLs, followed by the **title** of the first enqueued page and so on and so on. @@ -103,17 +103,17 @@ await context.enqueue_links(strategy='all') For even more control, you can use the `include` or `exclude` parameters, either as glob patterns or regular expressions, to filter the URLs. Refer to the API documentation for `enqueue_links` for detailed information on these and other available options. - + {GlobsExample} - + ### Transform requests before enqueuing For cases where you need to modify or filter requests before they are enqueued, you can use the `transform_request_function` parameter. This function takes a `Request` object as input and should return either a modified `Request`` object or `None`. If the function returns `None`, the request will be skipped. - + {TransformExample} - + ## Next steps diff --git a/docs/introduction/04_real_world_project.mdx b/docs/introduction/04_real_world_project.mdx index 649f648f89..61f6435980 100644 --- a/docs/introduction/04_real_world_project.mdx +++ b/docs/introduction/04_real_world_project.mdx @@ -4,9 +4,9 @@ title: Real-world project --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import SanityCheckExample from '!!raw-loader!./code_examples/04_sanity_check.py'; +import SanityCheckExample from '!!raw-loader!roa-loader!./code_examples/04_sanity_check.py'; > _Hey, guys, you know, it's cool that we can scrape the `` elements of web pages, but that's not very useful. Can we finally scrape some real data and save it somewhere in a machine-readable format? Because that's why I started reading this tutorial in the first place!_ @@ -110,9 +110,9 @@ Let's check that everything is set up correctly before writing the scraping logi The example below creates a new crawler that visits the start URL and prints the text content of all the categories on that page. When you run the code, you will see the _very badly formatted_ content of the individual category card. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {SanityCheckExample} -</CodeBlock> +</RunnableCodeBlock> If you're wondering how to get that `.collection-block-item` selector. We'll explain it in the next chapter on DevTools. diff --git a/docs/introduction/05_crawling.mdx b/docs/introduction/05_crawling.mdx index 1ced51ff50..7c68662766 100644 --- a/docs/introduction/05_crawling.mdx +++ b/docs/introduction/05_crawling.mdx @@ -4,10 +4,10 @@ title: Crawling --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import CrawlingListingExample from '!!raw-loader!./code_examples/05_crawling_listing.py'; -import CrawlingDetailExample from '!!raw-loader!./code_examples/05_crawling_detail.py'; +import CrawlingListingExample from '!!raw-loader!roa-loader!./code_examples/05_crawling_listing.py'; +import CrawlingDetailExample from '!!raw-loader!roa-loader!./code_examples/05_crawling_detail.py'; To crawl the whole [Warehouse store example](https://warehouse-theme-metal.myshopify.com/collections) and find all the data, you first need to visit all the pages with products - going through all categories available and also all the product detail pages. @@ -21,9 +21,9 @@ await enqueue_links() While useful in that scenario, you need something different now. Instead of finding all the `<a href="..">` elements with links to the same hostname, you need to find only the specific ones that will take your crawler to the next page of results. Otherwise, the crawler will visit a lot of other pages that you're not interested in. Using the power of DevTools and yet another <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> parameter, this becomes fairly easy. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {CrawlingListingExample} -</CodeBlock> +</RunnableCodeBlock> The code should look pretty familiar to you. It's a very simple request handler where we log the currently processed URL to the console and enqueue more links. But there are also a few new, interesting additions. Let's break it down. @@ -39,9 +39,9 @@ You will see `label` used often throughout Crawlee, as it's a convenient way of In a similar fashion, you need to collect all the URLs to the product detail pages, because only from there you can scrape all the data you need. The following code only repeats the concepts you already know for another set of links. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {CrawlingDetailExample} -</CodeBlock> +</RunnableCodeBlock> The crawling code is now complete. When you run the code, you'll see the crawler visit all the listing URLs and all the detail URLs. diff --git a/docs/introduction/06_scraping.mdx b/docs/introduction/06_scraping.mdx index 55393894f1..51c86e5835 100644 --- a/docs/introduction/06_scraping.mdx +++ b/docs/introduction/06_scraping.mdx @@ -4,9 +4,9 @@ title: Scraping --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import ScrapingExample from '!!raw-loader!./code_examples/06_scraping.py'; +import ScrapingExample from '!!raw-loader!roa-loader!./code_examples/06_scraping.py'; In the [Real-world project](./real-world-project#choosing-the-data-you-need) chapter, you've created a list of the information you wanted to collect about the products in the example Warehouse store. Let's review that and figure out ways to access the data. @@ -32,7 +32,8 @@ You can use `request.loaded_url` as well. Remember the difference: `request.url` By splitting the `request.url`, we can extract the manufacturer name directly from the URL. This is done by first splitting the URL to get the product identifier and then splitting that identifier to get the manufacturer name. ```python -# context.request.url: https://warehouse-theme-metal.myshopify.com/products/sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440 +# context.request.url: +# https://warehouse-theme-metal.myshopify.com/products/sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440 # Split the URL and get the last part. url_part = context.request.url.split('/').pop() @@ -132,9 +133,9 @@ For this, all that matters is whether the element exists or not. You can use the You have everything that is needed, so grab your newly created scraping logic, dump it into your original request handler and see the magic happen! -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {ScrapingExample} -</CodeBlock> +</RunnableCodeBlock> When you run the crawler, you will see the crawled URLs and their scraped data printed to the console. The output will look something like this: diff --git a/docs/introduction/07_saving_data.mdx b/docs/introduction/07_saving_data.mdx index 4ba5394861..adddd93af9 100644 --- a/docs/introduction/07_saving_data.mdx +++ b/docs/introduction/07_saving_data.mdx @@ -5,10 +5,12 @@ title: Saving data import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import FinalCodeExample from '!!raw-loader!./code_examples/07_final_code.py'; import FirstCodeExample from '!!raw-loader!./code_examples/07_first_code.py'; +import FinalCodeExample from '!!raw-loader!roa-loader!./code_examples/07_final_code.py'; + A data extraction job would not be complete without saving the data for later use and processing. You've come to the final and most difficult part of this tutorial so make sure to pay attention very carefully! ## Save data to the dataset @@ -20,7 +22,7 @@ Crawlee provides a <ApiLink to="class/Dataset">`Dataset`</ApiLink> class, which Here's an example: -<CodeBlock className="language-python"> +<CodeBlock language="python"> {FirstCodeExample} </CodeBlock> @@ -76,9 +78,9 @@ Instead of importing a new class and manually creating an instance of the datase And that's it. Unlike earlier, we are being serious now. That's it, you're done. The final code looks like this: -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {FinalCodeExample} -</CodeBlock> +</RunnableCodeBlock> ## What `push_data` does? diff --git a/docs/introduction/code_examples/03_enqueue_strategy.py b/docs/introduction/code_examples/03_enqueue_strategy.py index ca68c0823d..6aff8a1fba 100644 --- a/docs/introduction/code_examples/03_enqueue_strategy.py +++ b/docs/introduction/code_examples/03_enqueue_strategy.py @@ -1,8 +1,10 @@ +import asyncio + from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: @@ -17,3 +19,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: ) await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/03_finding_new_links.py b/docs/introduction/code_examples/03_finding_new_links.py index b84a5d32b3..e25af30c13 100644 --- a/docs/introduction/code_examples/03_finding_new_links.py +++ b/docs/introduction/code_examples/03_finding_new_links.py @@ -5,7 +5,7 @@ async def main() -> None: # Let's limit our crawls to make our tests shorter and safer. - crawler = BeautifulSoupCrawler(max_requests_per_crawl=20) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: diff --git a/docs/introduction/code_examples/03_globs.py b/docs/introduction/code_examples/03_globs.py index a8a5c8ae78..c2f2627d95 100644 --- a/docs/introduction/code_examples/03_globs.py +++ b/docs/introduction/code_examples/03_globs.py @@ -1,9 +1,11 @@ +import asyncio + from crawlee import Glob from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: @@ -21,3 +23,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: ) await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/03_transform_request.py b/docs/introduction/code_examples/03_transform_request.py index 18d6d9f0f3..5f11a1cafa 100644 --- a/docs/introduction/code_examples/03_transform_request.py +++ b/docs/introduction/code_examples/03_transform_request.py @@ -1,5 +1,7 @@ from __future__ import annotations +import asyncio + from crawlee import HttpHeaders, RequestOptions, RequestTransformAction from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -27,7 +29,7 @@ def transform_request( async def main() -> None: - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: @@ -41,3 +43,7 @@ async def blog_handler(context: BeautifulSoupCrawlingContext) -> None: context.log.info(f'Blog Processing {context.request.url}.') await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/06_scraping.py b/docs/introduction/code_examples/06_scraping.py index bc07c42690..f1faf1c521 100644 --- a/docs/introduction/code_examples/06_scraping.py +++ b/docs/introduction/code_examples/06_scraping.py @@ -6,7 +6,7 @@ async def main() -> None: crawler = PlaywrightCrawler( # Let's limit our crawls to make our tests shorter and safer. - max_requests_per_crawl=50, + max_requests_per_crawl=10, ) @crawler.router.default_handler diff --git a/docs/introduction/code_examples/07_final_code.py b/docs/introduction/code_examples/07_final_code.py index a6874d06fe..a1a89167b5 100644 --- a/docs/introduction/code_examples/07_final_code.py +++ b/docs/introduction/code_examples/07_final_code.py @@ -6,7 +6,7 @@ async def main() -> None: crawler = PlaywrightCrawler( # Let's limit our crawls to make our tests shorter and safer. - max_requests_per_crawl=50, + max_requests_per_crawl=10, ) @crawler.router.default_handler diff --git a/docs/introduction/code_examples/07_first_code.py b/docs/introduction/code_examples/07_first_code.py index d058692196..89de967684 100644 --- a/docs/introduction/code_examples/07_first_code.py +++ b/docs/introduction/code_examples/07_first_code.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.storages import Dataset @@ -14,3 +16,7 @@ async def main() -> None: async def request_handler(context: PlaywrightCrawlingContext) -> None: ... # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/08_main.py b/docs/introduction/code_examples/08_main.py index 7908c344ce..09f33e3376 100644 --- a/docs/introduction/code_examples/08_main.py +++ b/docs/introduction/code_examples/08_main.py @@ -8,7 +8,7 @@ async def main() -> None: crawler = PlaywrightCrawler( # Let's limit our crawls to make our tests shorter and safer. - max_requests_per_crawl=50, + max_requests_per_crawl=10, # Provide our router instance to the crawler. request_handler=router, ) diff --git a/docs/introduction/code_examples/09_apify_sdk.py b/docs/introduction/code_examples/09_apify_sdk.py index 5da6bd3a0d..fd8ceaffe7 100644 --- a/docs/introduction/code_examples/09_apify_sdk.py +++ b/docs/introduction/code_examples/09_apify_sdk.py @@ -13,7 +13,7 @@ async def main() -> None: async with Actor: crawler = PlaywrightCrawler( # Let's limit our crawls to make our tests shorter and safer. - max_requests_per_crawl=50, + max_requests_per_crawl=10, # Provide our router instance to the crawler. request_handler=router, ) diff --git a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py index 7b0a0d6cd9..2db8874c4b 100644 --- a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py +++ b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py @@ -6,7 +6,7 @@ async def main() -> None: # BeautifulSoupCrawler crawls the web using HTTP requests # and parses HTML using the BeautifulSoup library. - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) # Define a request handler to process each crawled page # and attach it to the crawler using a decorator. diff --git a/docs/quick-start/code_examples/parsel_crawler_example.py b/docs/quick-start/code_examples/parsel_crawler_example.py index 04c4b1ebf6..f8ed2a3e9c 100644 --- a/docs/quick-start/code_examples/parsel_crawler_example.py +++ b/docs/quick-start/code_examples/parsel_crawler_example.py @@ -6,7 +6,7 @@ async def main() -> None: # ParselCrawler crawls the web using HTTP requests # and parses HTML using the Parsel library. - crawler = ParselCrawler(max_requests_per_crawl=50) + crawler = ParselCrawler(max_requests_per_crawl=10) # Define a request handler to process each crawled page # and attach it to the crawler using a decorator. diff --git a/docs/quick-start/code_examples/playwright_crawler_headful_example.py b/docs/quick-start/code_examples/playwright_crawler_headful_example.py index c2d02ae0e7..403c665e51 100644 --- a/docs/quick-start/code_examples/playwright_crawler_headful_example.py +++ b/docs/quick-start/code_examples/playwright_crawler_headful_example.py @@ -1,12 +1,19 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler async def main() -> None: crawler = PlaywrightCrawler( # Run with a visible browser window. + # highlight-next-line headless=False, # Switch to the Firefox browser. browser_type='firefox', ) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/quick-start/index.mdx b/docs/quick-start/index.mdx index ffe7306dbb..6045e39951 100644 --- a/docs/quick-start/index.mdx +++ b/docs/quick-start/index.mdx @@ -7,10 +7,12 @@ import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import BeautifulsoupCrawlerExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_example.py'; +import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler_example.py'; +import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_example.py'; -import BeautifulsoupCrawlerExample from '!!raw-loader!./code_examples/beautifulsoup_crawler_example.py'; -import ParselCrawlerExample from '!!raw-loader!./code_examples/parsel_crawler_example.py'; -import PlaywrightCrawlerExample from '!!raw-loader!./code_examples/playwright_crawler_example.py'; import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/playwright_crawler_headful_example.py'; This short tutorial will help you start scraping with Crawlee in just a minute or two. For an in-depth understanding of how Crawlee works, check out the [Introduction](../introduction/index.mdx) section, which provides a comprehensive step-by-step guide to creating your first scraper. @@ -59,27 +61,27 @@ If you plan to use the <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler` playwright install ``` -For detailed installation instructions see the [Setting up](../introduction/01_setting_up.mdx) documentation page. +For detailed installation instructions, see the [Setting up](../introduction/01_setting_up.mdx) documentation page. ## Crawling Run the following example to perform a recursive crawl of the Crawlee website using the selected crawler. -<Tabs groupId="main"> - <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler"> - <CodeBlock className="language-python"> +<Tabs groupId="quickStart"> + <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler" default> + <RunnableCodeBlock className="language-python" language="python"> {BeautifulsoupCrawlerExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="ParselCrawler" label="ParselCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {ParselCrawlerExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {PlaywrightCrawlerExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -89,9 +91,9 @@ When you run the example, you will see Crawlee automating the data extraction pr ## Running headful browser -By default, browsers controlled by Playwright run in headless mode (without a visible window). However, you can configure the crawler to run in a headful mode, which is useful during development phase to observe the browser's actions. You can alsoswitch from the default Chromium browser to Firefox or WebKit. +By default, browsers controlled by Playwright run in headless mode (without a visible window). However, you can configure the crawler to run in a headful mode, which is useful during the development phase to observe the browser's actions. You can also switch from the default Chromium browser to Firefox or WebKit. -<CodeBlock className="language-python"> +<CodeBlock language="python"> {PlaywrightCrawlerHeadfulExample} </CodeBlock> diff --git a/website/roa-loader/index.js b/website/roa-loader/index.js index 9673ca94e3..d4826ad554 100644 --- a/website/roa-loader/index.js +++ b/website/roa-loader/index.js @@ -1,13 +1,25 @@ -const { inspect } = require('util'); +const { createHash } = require('node:crypto'); +const { inspect } = require('node:util'); const { urlToRequest } = require('loader-utils'); const signingUrl = new URL('https://api.apify.com/v2/tools/encode-and-sign'); signingUrl.searchParams.set('token', process.env.APIFY_SIGNING_TOKEN); const queue = []; +const cache = {}; let working = false; +function hash(source) { + return createHash('sha1').update(source).digest('hex'); +} + async function getHash(source) { + const cacheKey = hash(source); + + if (cache[cacheKey]) { + return cache[cacheKey]; + } + const memory = source.match(/playwright|puppeteer/i) ? 4096 : 1024; const res = await (await fetch(signingUrl, { method: 'POST', @@ -32,13 +44,14 @@ async function getHash(source) { const body = await res.json(); - await new Promise((resolve) => setTimeout(resolve, 100)); - if (!body.data || !body.data.encoded) { console.error(`Signing failed:' ${inspect(body.error) || 'Unknown error'}`, body); return 'invalid-token'; } + cache[cacheKey] = body.data.encoded; + await new Promise((resolve) => setTimeout(resolve, 100)); + return body.data.encoded; } @@ -72,14 +85,11 @@ async function encodeAndSign(source) { } module.exports = async function (code) { - // TODO enable once we have python example runner actor - return { code }; - if (process.env.CRAWLEE_DOCS_FAST) { return { code, hash: 'fast' }; } console.log(`Signing ${urlToRequest(this.resourcePath)}...`, { working, queue: queue.length }); - const hash = await encodeAndSign(code); - return { code, hash }; + const codeHash = await encodeAndSign(code); + return { code, hash: codeHash }; }; diff --git a/website/src/components/RunnableCodeBlock.jsx b/website/src/components/RunnableCodeBlock.jsx index c7b8e2d65a..4af7507033 100644 --- a/website/src/components/RunnableCodeBlock.jsx +++ b/website/src/components/RunnableCodeBlock.jsx @@ -4,13 +4,9 @@ import CodeBlock from '@theme/CodeBlock'; import Link from '@docusaurus/Link'; import styles from './RunnableCodeBlock.module.css'; -const EXAMPLE_RUNNERS = { - playwright: '6i5QsHBMtm3hKph70', - puppeteer: '7tWSD8hrYzuc9Lte7', - cheerio: 'kk67IcZkKSSBTslXI', -}; +const PYTHON_ACTOR_RUNNER = 'HH9rhkFXiZbheuq1V' -const RunnableCodeBlock = ({ children, actor, hash, type, ...props }) => { +const RunnableCodeBlock = ({ children, actor, hash, ...props }) => { hash = hash ?? children.hash; if (!children.code) { @@ -26,7 +22,7 @@ Make sure you are importing the code block contents with the roa-loader.`); ); } - const href = `https://console.apify.com/actors/${actor ?? EXAMPLE_RUNNERS[type ?? 'playwright']}?runConfig=${hash}&asrc=run_on_apify`; + const href = `https://console.apify.com/actors/${actor ?? PYTHON_ACTOR_RUNNER}?runConfig=${hash}&asrc=run_on_apify`; return ( <div className={clsx(styles.container, 'runnable-code-block')}> diff --git a/website/src/pages/home_page_example.py b/website/src/pages/home_page_example.py new file mode 100644 index 0000000000..fe5b80eebc --- /dev/null +++ b/website/src/pages/home_page_example.py @@ -0,0 +1,37 @@ +from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext + + +async def main() -> None: + crawler = PlaywrightCrawler( + max_requests_per_crawl=5, # Limit the crawl to 5 requests at most. + headless=False, # Show the browser window. + browser_type='firefox', # Use the Firefox browser. + ) + + # Define the default request handler, which will be called for every request. + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + + # Extract and enqueue all links found on the page. + await context.enqueue_links() + + # Extract data from the page using Playwright API. + data = { + 'url': context.request.url, + 'title': await context.page.title(), + 'content': (await context.page.content())[:100], + } + + # Push the extracted data to the default dataset. + await context.push_data(data) + + # Run the crawler with the initial list of URLs. + await crawler.run(['https://crawlee.dev']) + + # Export the entire dataset to a JSON file. + await crawler.export_data('results.json') + + # Or work with the data directly. + data = await crawler.get_data() + crawler.log.info(f'Extracted data: {data.items}') diff --git a/website/src/pages/index.js b/website/src/pages/index.js index 17dd7b6eeb..fd53f43f7e 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -14,8 +14,11 @@ import HomepageCtaSection from '../components/Homepage/HomepageCtaSection'; import HomepageHeroSection from '../components/Homepage/HomepageHeroSection'; import LanguageInfoWidget from '../components/Homepage/LanguageInfoWidget'; import RiverSection from '../components/Homepage/RiverSection'; +import RunnableCodeBlock from '../components/RunnableCodeBlock'; import ThreeCardsWithIcon from '../components/Homepage/ThreeCardsWithIcon'; +import HomePageExample from '!!raw-loader!roa-loader!./home_page_example.py'; + function GetStartedSection() { return ( <section className={styles.languageGetStartedSection}> @@ -28,57 +31,14 @@ function GetStartedSection() { ); } -const example = `import asyncio - -from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext - - -async def main() -> None: - crawler = PlaywrightCrawler( - max_requests_per_crawl=5, # Limit the crawl to 5 requests at most. - headless=False, # Show the browser window. - browser_type='firefox', # Use the Firefox browser. - ) - - # Define the default request handler, which will be called for every request. - @crawler.router.default_handler - async def request_handler(context: PlaywrightCrawlingContext) -> None: - context.log.info(f'Processing {context.request.url} ...') - - # Extract and enqueue all links found on the page. - await context.enqueue_links() - - # Extract data from the page using Playwright API. - data = { - 'url': context.request.url, - 'title': await context.page.title(), - 'content': (await context.page.content())[:100], - } - - # Push the extracted data to the default dataset. - await context.push_data(data) - - # Run the crawler with the initial list of URLs. - await crawler.run(['https://crawlee.dev']) - - # Export the entire dataset to a JSON file. - await crawler.export_data('results.json') - - # Or work with the data directly. - data = await crawler.get_data() - crawler.log.info(f'Extracted data: {data.items}') - - -if __name__ == '__main__': - asyncio.run(main()) -`; - function CodeExampleSection() { return ( <section className={styles.codeExampleSection}> <div className={styles.decorativeRow} /> <div className={styles.codeBlockContainer}> - <CodeBlock language="python">{example}</CodeBlock> + <RunnableCodeBlock className="language-python" language="python"> + {HomePageExample} + </RunnableCodeBlock> </div> <div className={styles.dashedSeparator} /> <div className={styles.decorativeRow} />