From a3710c07cd038ea0adf1a61d6173bb00ba1f81db Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 31 Mar 2025 15:49:55 +0200 Subject: [PATCH 01/16] rm asyncio run --- docs/examples/code_examples/adaptive_playwright_crawler.py | 5 ----- docs/examples/code_examples/add_data_to_dataset_bs.py | 6 ------ docs/examples/code_examples/add_data_to_dataset_pw.py | 6 ------ docs/examples/code_examples/beautifulsoup_crawler.py | 5 ----- .../code_examples/beautifulsoup_crawler_keep_alive.py | 4 ---- docs/examples/code_examples/beautifulsoup_crawler_stop.py | 6 ------ .../code_examples/capture_screenshot_using_playwright.py | 6 ------ docs/examples/code_examples/configure_json_logging.py | 5 ----- .../examples/code_examples/crawl_all_links_on_website_bs.py | 6 ------ .../examples/code_examples/crawl_all_links_on_website_pw.py | 6 ------ docs/examples/code_examples/crawl_multiple_urls_bs.py | 6 ------ docs/examples/code_examples/crawl_multiple_urls_pw.py | 6 ------ .../code_examples/crawl_specific_links_on_website_bs.py | 6 ------ .../code_examples/crawl_specific_links_on_website_pw.py | 6 ------ .../crawl_website_with_relative_links_all_links.py | 6 ------ .../crawl_website_with_relative_links_same_domain.py | 6 ------ .../crawl_website_with_relative_links_same_hostname.py | 6 ------ .../crawl_website_with_relative_links_same_origin.py | 6 ------ .../code_examples/export_entire_dataset_to_file_csv.py | 6 ------ .../code_examples/export_entire_dataset_to_file_json.py | 6 ------ .../code_examples/fill_and_submit_web_form_crawler.py | 5 ----- docs/examples/code_examples/parsel_crawler.py | 6 ------ docs/examples/code_examples/playwright_block_requests.py | 6 ------ docs/examples/code_examples/playwright_crawler.py | 6 ------ .../code_examples/playwright_crawler_with_camoufox.py | 6 ------ .../playwright_crawler_with_fingerprint_generator.py | 6 ------ .../playwright_with_fingerprint_generator.py | 6 ------ .../code_examples/http_clients/curl_impersonate_example.py | 6 ------ docs/guides/code_examples/http_clients/httpx_example.py | 6 ------ .../playwright_crawler/browser_configuration_example.py | 6 ------ .../playwright_crawler/multiple_launch_example.py | 6 ------ .../playwright_crawler/pre_navigation_hook_example.py | 6 ------ .../code_examples/proxy_management/inspecting_bs_example.py | 6 ------ .../code_examples/proxy_management/inspecting_pw_example.py | 6 ------ .../proxy_management/integration_bs_example.py | 6 ------ .../proxy_management/integration_pw_example.py | 6 ------ .../code_examples/proxy_management/quick_start_example.py | 6 ------ .../code_examples/proxy_management/tiers_bs_example.py | 6 ------ .../code_examples/proxy_management/tiers_pw_example.py | 6 ------ .../code_examples/request_loaders/rl_basic_example.py | 6 ------ docs/guides/code_examples/request_loaders/tandem_example.py | 5 ----- .../request_loaders/tandem_example_explicit.py | 5 ----- .../scaling_crawlers/max_tasks_per_minute_example.py | 6 ------ .../scaling_crawlers/min_and_max_concurrency_example.py | 6 ------ .../code_examples/session_management/multi_sessions_http.py | 5 ----- .../code_examples/session_management/one_session_http.py | 5 ----- .../session_management/session_management_basic.py | 5 ----- .../session_management/session_management_beautifulsoup.py | 6 ------ .../session_management/session_management_http.py | 5 ----- .../session_management/session_management_parsel.py | 6 ------ .../session_management/session_management_playwright.py | 6 ------ .../session_management/session_management_standalone.py | 6 ------ .../code_examples/storages/cleaning_do_not_purge_example.py | 6 ------ .../storages/cleaning_purge_explicitly_example.py | 6 ------ docs/guides/code_examples/storages/dataset_basic_example.py | 6 ------ .../code_examples/storages/dataset_with_crawler_example.py | 6 ------ .../storages/dataset_with_crawler_explicit_example.py | 6 ------ .../code_examples/storages/helper_add_requests_example.py | 6 ------ .../code_examples/storages/helper_enqueue_links_example.py | 6 ------ docs/guides/code_examples/storages/kvs_basic_example.py | 6 ------ .../code_examples/storages/kvs_with_crawler_example.py | 6 ------ .../storages/kvs_with_crawler_explicit_example.py | 6 ------ docs/guides/code_examples/storages/rq_basic_example.py | 6 ------ .../code_examples/storages/rq_with_crawler_example.py | 6 ------ .../storages/rq_with_crawler_explicit_example.py | 6 ------ docs/introduction/code_examples/02_bs.py | 6 ------ docs/introduction/code_examples/02_bs_better.py | 6 ------ docs/introduction/code_examples/02_request_queue.py | 6 ------ docs/introduction/code_examples/03_finding_new_links.py | 6 ------ docs/introduction/code_examples/03_original_code.py | 6 ------ docs/introduction/code_examples/04_sanity_check.py | 6 ------ docs/introduction/code_examples/05_crawling_detail.py | 6 ------ docs/introduction/code_examples/05_crawling_listing.py | 6 ------ docs/introduction/code_examples/06_scraping.py | 6 ------ docs/introduction/code_examples/07_final_code.py | 6 ------ docs/introduction/code_examples/08_main.py | 6 ------ docs/introduction/code_examples/09_apify_sdk.py | 6 ------ .../code_examples/beautifulsoup_crawler_example.py | 6 ------ docs/quick-start/code_examples/parsel_crawler_example.py | 6 ------ .../quick-start/code_examples/playwright_crawler_example.py | 6 ------ 80 files changed, 468 deletions(-) diff --git a/docs/examples/code_examples/adaptive_playwright_crawler.py b/docs/examples/code_examples/adaptive_playwright_crawler.py index add09041b2..7513cbba6d 100644 --- a/docs/examples/code_examples/adaptive_playwright_crawler.py +++ b/docs/examples/code_examples/adaptive_playwright_crawler.py @@ -1,4 +1,3 @@ -import asyncio from datetime import timedelta from playwright.async_api import Route @@ -59,7 +58,3 @@ async def some_routing_function(route: Route) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://warehouse-theme-metal.myshopify.com/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/add_data_to_dataset_bs.py b/docs/examples/code_examples/add_data_to_dataset_bs.py index 4318cbe0d4..5044a35bb0 100644 --- a/docs/examples/code_examples/add_data_to_dataset_bs.py +++ b/docs/examples/code_examples/add_data_to_dataset_bs.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -29,7 +27,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: 'https://example.com', ] ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/add_data_to_dataset_pw.py b/docs/examples/code_examples/add_data_to_dataset_pw.py index 8eb714aef3..9b07ac90db 100644 --- a/docs/examples/code_examples/add_data_to_dataset_pw.py +++ b/docs/examples/code_examples/add_data_to_dataset_pw.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -29,7 +27,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: 'https://example.com', ] ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/beautifulsoup_crawler.py b/docs/examples/code_examples/beautifulsoup_crawler.py index 5e9701d7cb..73634ed4f0 100644 --- a/docs/examples/code_examples/beautifulsoup_crawler.py +++ b/docs/examples/code_examples/beautifulsoup_crawler.py @@ -1,4 +1,3 @@ -import asyncio from datetime import timedelta from crawlee.crawlers import ( @@ -51,7 +50,3 @@ async def some_hook(context: BasicCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py b/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py index 38e5623939..2f10cd148d 100644 --- a/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py +++ b/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py @@ -50,7 +50,3 @@ async def request_handler(context: BasicCrawlingContext) -> None: await crawler.run() await asyncio.gather(add_request_later_task1, add_request_later_task2) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/beautifulsoup_crawler_stop.py b/docs/examples/code_examples/beautifulsoup_crawler_stop.py index 2069bd6ecb..21904533a7 100644 --- a/docs/examples/code_examples/beautifulsoup_crawler_stop.py +++ b/docs/examples/code_examples/beautifulsoup_crawler_stop.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -35,7 +33,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/capture_screenshot_using_playwright.py b/docs/examples/code_examples/capture_screenshot_using_playwright.py index e4b4c1ec22..4a02f51dc6 100644 --- a/docs/examples/code_examples/capture_screenshot_using_playwright.py +++ b/docs/examples/code_examples/capture_screenshot_using_playwright.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.storages import KeyValueStore @@ -41,7 +39,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: 'https://example.com', ] ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/configure_json_logging.py b/docs/examples/code_examples/configure_json_logging.py index 25cb37c745..1831cc5fd6 100644 --- a/docs/examples/code_examples/configure_json_logging.py +++ b/docs/examples/code_examples/configure_json_logging.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import inspect import logging import sys @@ -85,7 +84,3 @@ async def request_handler(context: HttpCrawlingContext) -> None: # Run the crawler await crawler.run(['https://www.crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_all_links_on_website_bs.py b/docs/examples/code_examples/crawl_all_links_on_website_bs.py index ad5ef62f54..1e1a219718 100644 --- a/docs/examples/code_examples/crawl_all_links_on_website_bs.py +++ b/docs/examples/code_examples/crawl_all_links_on_website_bs.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -19,7 +17,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_all_links_on_website_pw.py b/docs/examples/code_examples/crawl_all_links_on_website_pw.py index 4a6fb6e616..83e7659e5b 100644 --- a/docs/examples/code_examples/crawl_all_links_on_website_pw.py +++ b/docs/examples/code_examples/crawl_all_links_on_website_pw.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -19,7 +17,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_multiple_urls_bs.py b/docs/examples/code_examples/crawl_multiple_urls_bs.py index e8cf82f2bc..e80db58f81 100644 --- a/docs/examples/code_examples/crawl_multiple_urls_bs.py +++ b/docs/examples/code_examples/crawl_multiple_urls_bs.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -19,7 +17,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: 'https://example.com', ] ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_multiple_urls_pw.py b/docs/examples/code_examples/crawl_multiple_urls_pw.py index b18d04c8ad..ef4eccaf01 100644 --- a/docs/examples/code_examples/crawl_multiple_urls_pw.py +++ b/docs/examples/code_examples/crawl_multiple_urls_pw.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -19,7 +17,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: 'https://example.com', ] ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py b/docs/examples/code_examples/crawl_specific_links_on_website_bs.py index 8dfc1bdf85..65feb175aa 100644 --- a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py +++ b/docs/examples/code_examples/crawl_specific_links_on_website_bs.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee import Glob from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -23,7 +21,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_specific_links_on_website_pw.py b/docs/examples/code_examples/crawl_specific_links_on_website_pw.py index 98a2f0435b..144c55e1b0 100644 --- a/docs/examples/code_examples/crawl_specific_links_on_website_pw.py +++ b/docs/examples/code_examples/crawl_specific_links_on_website_pw.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee import Glob from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -23,7 +21,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py b/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py index b253a9566f..439e70e1a2 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -20,7 +18,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py index 0fa264ef20..6a2ee32b71 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -20,7 +18,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py index 0259cafe67..f85061da4a 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -20,7 +18,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py index 46e9f32759..6b4fd29699 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -20,7 +18,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/export_entire_dataset_to_file_csv.py b/docs/examples/code_examples/export_entire_dataset_to_file_csv.py index 115474fc61..45f3b9c666 100644 --- a/docs/examples/code_examples/export_entire_dataset_to_file_csv.py +++ b/docs/examples/code_examples/export_entire_dataset_to_file_csv.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -31,7 +29,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Export the entire dataset to a CSV file. await crawler.export_data_csv(path='results.csv') - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/export_entire_dataset_to_file_json.py b/docs/examples/code_examples/export_entire_dataset_to_file_json.py index 5c871fb228..4f116a5df9 100644 --- a/docs/examples/code_examples/export_entire_dataset_to_file_json.py +++ b/docs/examples/code_examples/export_entire_dataset_to_file_json.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -31,7 +29,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Export the entire dataset to a JSON file. await crawler.export_data_json(path='results.json') - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/fill_and_submit_web_form_crawler.py b/docs/examples/code_examples/fill_and_submit_web_form_crawler.py index c00a784411..e27c74e9b3 100644 --- a/docs/examples/code_examples/fill_and_submit_web_form_crawler.py +++ b/docs/examples/code_examples/fill_and_submit_web_form_crawler.py @@ -1,4 +1,3 @@ -import asyncio from urllib.parse import urlencode from crawlee import Request @@ -35,7 +34,3 @@ async def request_handler(context: HttpCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run([request]) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/parsel_crawler.py b/docs/examples/code_examples/parsel_crawler.py index 61ddb7484e..52b6e98c86 100644 --- a/docs/examples/code_examples/parsel_crawler.py +++ b/docs/examples/code_examples/parsel_crawler.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext # Regex for identifying email addresses on a webpage. @@ -41,7 +39,3 @@ async def some_hook(context: BasicCrawlingContext) -> None: # Export the entire dataset to a JSON file. await crawler.export_data_json(path='results.json') - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_block_requests.py b/docs/examples/code_examples/playwright_block_requests.py index 991a67aede..9a6af4c65f 100644 --- a/docs/examples/code_examples/playwright_block_requests.py +++ b/docs/examples/code_examples/playwright_block_requests.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import ( PlaywrightCrawler, PlaywrightCrawlingContext, @@ -30,7 +28,3 @@ async def navigation_hook(context: PlaywrightPreNavCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_crawler.py b/docs/examples/code_examples/playwright_crawler.py index f35332b063..e9cebfc028 100644 --- a/docs/examples/code_examples/playwright_crawler.py +++ b/docs/examples/code_examples/playwright_crawler.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import ( PlaywrightCrawler, PlaywrightCrawlingContext, @@ -61,7 +59,3 @@ async def log_navigation_url(context: PlaywrightPreNavCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://news.ycombinator.com/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_crawler_with_camoufox.py b/docs/examples/code_examples/playwright_crawler_with_camoufox.py index 691197da55..1a03578540 100644 --- a/docs/examples/code_examples/playwright_crawler_with_camoufox.py +++ b/docs/examples/code_examples/playwright_crawler_with_camoufox.py @@ -1,5 +1,3 @@ -import asyncio - # Camoufox is external package and needs to be installed. It is not included in crawlee. from camoufox import AsyncNewBrowser from typing_extensions import override @@ -63,7 +61,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://news.ycombinator.com/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py b/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py index a5d80797fb..6e1c363e3f 100644 --- a/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py +++ b/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.fingerprint_suite import ( DefaultFingerprintGenerator, @@ -38,7 +36,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://news.ycombinator.com/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py b/docs/guides/code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py index 5e1c8d2668..f0ec2db446 100644 --- a/docs/guides/code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py +++ b/docs/guides/code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -17,7 +15,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://news.ycombinator.com/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/http_clients/curl_impersonate_example.py b/docs/guides/code_examples/http_clients/curl_impersonate_example.py index 28813a2f46..8f3cb49b19 100644 --- a/docs/guides/code_examples/http_clients/curl_impersonate_example.py +++ b/docs/guides/code_examples/http_clients/curl_impersonate_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.http_clients import CurlImpersonateHttpClient @@ -36,7 +34,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/http_clients/httpx_example.py b/docs/guides/code_examples/http_clients/httpx_example.py index 5c3c4883cb..a113445fba 100644 --- a/docs/guides/code_examples/http_clients/httpx_example.py +++ b/docs/guides/code_examples/http_clients/httpx_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.http_clients import HttpxHttpClient @@ -36,7 +34,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py b/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py index 10ff84eba0..977c56a8b1 100644 --- a/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py +++ b/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -37,7 +35,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py b/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py index 59219b89c7..8f228cf326 100644 --- a/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py +++ b/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -32,7 +30,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev', 'https://apify.com/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler/pre_navigation_hook_example.py b/docs/guides/code_examples/playwright_crawler/pre_navigation_hook_example.py index 5a36456679..bf6a919d22 100644 --- a/docs/guides/code_examples/playwright_crawler/pre_navigation_hook_example.py +++ b/docs/guides/code_examples/playwright_crawler/pre_navigation_hook_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import ( PlaywrightCrawler, PlaywrightCrawlingContext, @@ -28,7 +26,3 @@ async def log_navigation_url(context: PlaywrightPreNavCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/inspecting_bs_example.py b/docs/guides/code_examples/proxy_management/inspecting_bs_example.py index b6035097d6..a25af60f5b 100644 --- a/docs/guides/code_examples/proxy_management/inspecting_bs_example.py +++ b/docs/guides/code_examples/proxy_management/inspecting_bs_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -22,7 +20,3 @@ async def default_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/inspecting_pw_example.py b/docs/guides/code_examples/proxy_management/inspecting_pw_example.py index e193972399..f82c04cd9b 100644 --- a/docs/guides/code_examples/proxy_management/inspecting_pw_example.py +++ b/docs/guides/code_examples/proxy_management/inspecting_pw_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -22,7 +20,3 @@ async def default_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/integration_bs_example.py b/docs/guides/code_examples/proxy_management/integration_bs_example.py index 63a2e703e3..c47f0a76a9 100644 --- a/docs/guides/code_examples/proxy_management/integration_bs_example.py +++ b/docs/guides/code_examples/proxy_management/integration_bs_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -26,7 +24,3 @@ async def default_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/integration_pw_example.py b/docs/guides/code_examples/proxy_management/integration_pw_example.py index e8e0ea8821..394551d706 100644 --- a/docs/guides/code_examples/proxy_management/integration_pw_example.py +++ b/docs/guides/code_examples/proxy_management/integration_pw_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -26,7 +24,3 @@ async def default_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/quick_start_example.py b/docs/guides/code_examples/proxy_management/quick_start_example.py index 3b43a48312..3941f1440e 100644 --- a/docs/guides/code_examples/proxy_management/quick_start_example.py +++ b/docs/guides/code_examples/proxy_management/quick_start_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.proxy_configuration import ProxyConfiguration @@ -15,7 +13,3 @@ async def main() -> None: proxy_url_1 = await proxy_configuration.new_url() # http://proxy-1.com/ proxy_url_2 = await proxy_configuration.new_url() # http://proxy-2.com/ proxy_url_3 = await proxy_configuration.new_url() # http://proxy-1.com/ - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/tiers_bs_example.py b/docs/guides/code_examples/proxy_management/tiers_bs_example.py index 37f69e6419..66183cd8d6 100644 --- a/docs/guides/code_examples/proxy_management/tiers_bs_example.py +++ b/docs/guides/code_examples/proxy_management/tiers_bs_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -33,7 +31,3 @@ async def default_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/tiers_pw_example.py b/docs/guides/code_examples/proxy_management/tiers_pw_example.py index 2dcb5ad3bd..53500e3786 100644 --- a/docs/guides/code_examples/proxy_management/tiers_pw_example.py +++ b/docs/guides/code_examples/proxy_management/tiers_pw_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -33,7 +31,3 @@ async def default_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/rl_basic_example.py b/docs/guides/code_examples/request_loaders/rl_basic_example.py index abe4d55584..ec1e347a7a 100644 --- a/docs/guides/code_examples/request_loaders/rl_basic_example.py +++ b/docs/guides/code_examples/request_loaders/rl_basic_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.request_loaders import RequestList @@ -21,7 +19,3 @@ async def main() -> None: # And mark it as handled. await request_list.mark_request_as_handled(request) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/tandem_example.py b/docs/guides/code_examples/request_loaders/tandem_example.py index 0739d483ea..338590d6af 100644 --- a/docs/guides/code_examples/request_loaders/tandem_example.py +++ b/docs/guides/code_examples/request_loaders/tandem_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import ParselCrawler, ParselCrawlingContext from crawlee.request_loaders import RequestList @@ -21,6 +19,3 @@ async def handler(context: ParselCrawlingContext) -> None: await context.enqueue_links() await crawler.run() - - -asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/tandem_example_explicit.py b/docs/guides/code_examples/request_loaders/tandem_example_explicit.py index ff34e1af5d..11a6008e1b 100644 --- a/docs/guides/code_examples/request_loaders/tandem_example_explicit.py +++ b/docs/guides/code_examples/request_loaders/tandem_example_explicit.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import ParselCrawler, ParselCrawlingContext from crawlee.request_loaders import RequestList, RequestManagerTandem from crawlee.storages import RequestQueue @@ -24,6 +22,3 @@ async def handler(context: ParselCrawlingContext) -> None: await context.enqueue_links() await crawler.run() - - -asyncio.run(main()) diff --git a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py b/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py index 4b2a2e8c4e..e375198698 100644 --- a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py +++ b/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee import ConcurrencySettings from crawlee.crawlers import BeautifulSoupCrawler @@ -17,7 +15,3 @@ async def main() -> None: # Apply the defined concurrency settings to the crawler. concurrency_settings=concurrency_settings, ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py b/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py index 2cbac6cfc0..3ec02aead6 100644 --- a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py +++ b/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee import ConcurrencySettings from crawlee.crawlers import BeautifulSoupCrawler @@ -19,7 +17,3 @@ async def main() -> None: # Use the configured concurrency settings for the crawler. concurrency_settings=concurrency_settings, ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/multi_sessions_http.py b/docs/guides/code_examples/session_management/multi_sessions_http.py index aba4d5e9d0..2ad8ff8802 100644 --- a/docs/guides/code_examples/session_management/multi_sessions_http.py +++ b/docs/guides/code_examples/session_management/multi_sessions_http.py @@ -1,4 +1,3 @@ -import asyncio from datetime import timedelta from itertools import count from typing import Callable @@ -79,7 +78,3 @@ async def error_processing(context: BasicCrawlingContext, error: Exception) -> N ] await crawler.run(init_requests) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/one_session_http.py b/docs/guides/code_examples/session_management/one_session_http.py index 28cec44b63..58dc6815bb 100644 --- a/docs/guides/code_examples/session_management/one_session_http.py +++ b/docs/guides/code_examples/session_management/one_session_http.py @@ -1,4 +1,3 @@ -import asyncio from datetime import timedelta from crawlee import ConcurrencySettings, Request @@ -50,7 +49,3 @@ async def error_processing(context: BasicCrawlingContext, error: Exception) -> N crawler.stop() await crawler.run([Request.from_url('https://example.org/', label='session_init')]) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/session_management_basic.py b/docs/guides/code_examples/session_management/session_management_basic.py index 958ad5a665..9241f184be 100644 --- a/docs/guides/code_examples/session_management/session_management_basic.py +++ b/docs/guides/code_examples/session_management/session_management_basic.py @@ -1,4 +1,3 @@ -import asyncio import re from crawlee.crawlers import BasicCrawler, BasicCrawlingContext @@ -42,7 +41,3 @@ async def default_handler(context: BasicCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/session_management_beautifulsoup.py b/docs/guides/code_examples/session_management/session_management_beautifulsoup.py index a54fd8425f..6077502a4b 100644 --- a/docs/guides/code_examples/session_management/session_management_beautifulsoup.py +++ b/docs/guides/code_examples/session_management/session_management_beautifulsoup.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration from crawlee.sessions import SessionPool @@ -36,7 +34,3 @@ async def default_handler(context: BeautifulSoupCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/session_management_http.py b/docs/guides/code_examples/session_management/session_management_http.py index cd12d04bdf..275b0a9fe9 100644 --- a/docs/guides/code_examples/session_management/session_management_http.py +++ b/docs/guides/code_examples/session_management/session_management_http.py @@ -1,4 +1,3 @@ -import asyncio import re from crawlee.crawlers import HttpCrawler, HttpCrawlingContext @@ -38,7 +37,3 @@ async def default_handler(context: HttpCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/session_management_parsel.py b/docs/guides/code_examples/session_management/session_management_parsel.py index 66752a63c3..583f1e3a06 100644 --- a/docs/guides/code_examples/session_management/session_management_parsel.py +++ b/docs/guides/code_examples/session_management/session_management_parsel.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import ParselCrawler, ParselCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration from crawlee.sessions import SessionPool @@ -36,7 +34,3 @@ async def default_handler(context: ParselCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/session_management_playwright.py b/docs/guides/code_examples/session_management/session_management_playwright.py index 46a4c4f096..bcc1ea7aa0 100644 --- a/docs/guides/code_examples/session_management/session_management_playwright.py +++ b/docs/guides/code_examples/session_management/session_management_playwright.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration from crawlee.sessions import SessionPool @@ -36,7 +34,3 @@ async def default_handler(context: PlaywrightCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/session_management_standalone.py b/docs/guides/code_examples/session_management/session_management_standalone.py index 32989dc7e0..ae0d542273 100644 --- a/docs/guides/code_examples/session_management/session_management_standalone.py +++ b/docs/guides/code_examples/session_management/session_management_standalone.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.sessions import SessionPool @@ -19,7 +17,3 @@ async def main() -> None: # Lower the error_score and mark the session good. session.mark_good() - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py b/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py index 6514863555..dceab42d16 100644 --- a/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py +++ b/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.configuration import Configuration from crawlee.crawlers import HttpCrawler, HttpCrawlingContext @@ -17,7 +15,3 @@ async def request_handler(context: HttpCrawlingContext) -> None: context.log.info(f'Processing {context.request.url} ...') await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py b/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py index 15435da7bf..3e3c400a4e 100644 --- a/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py +++ b/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import HttpCrawler from crawlee.storage_clients import MemoryStorageClient @@ -15,7 +13,3 @@ async def main() -> None: crawler = HttpCrawler(storage_client=storage_client) # ... - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/dataset_basic_example.py b/docs/guides/code_examples/storages/dataset_basic_example.py index 9b67f36eb0..26bff30518 100644 --- a/docs/guides/code_examples/storages/dataset_basic_example.py +++ b/docs/guides/code_examples/storages/dataset_basic_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.storages import Dataset @@ -20,7 +18,3 @@ async def main() -> None: # Remove the dataset. await dataset.drop() - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/dataset_with_crawler_example.py b/docs/guides/code_examples/storages/dataset_with_crawler_example.py index 7e40824166..41b8258260 100644 --- a/docs/guides/code_examples/storages/dataset_with_crawler_example.py +++ b/docs/guides/code_examples/storages/dataset_with_crawler_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -26,7 +24,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Export the dataset to a file. await crawler.export_data(path='dataset.csv') - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py b/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py index 7c6a613b8f..a88282e75e 100644 --- a/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py +++ b/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.storages import Dataset @@ -31,7 +29,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Export the dataset to the key-value store. await dataset.export_to(key='dataset', content_type='csv') - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/helper_add_requests_example.py b/docs/guides/code_examples/storages/helper_add_requests_example.py index 15104cf6fc..db344310c5 100644 --- a/docs/guides/code_examples/storages/helper_add_requests_example.py +++ b/docs/guides/code_examples/storages/helper_add_requests_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -13,7 +11,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await context.add_requests(['https://apify.com/']) await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/helper_enqueue_links_example.py b/docs/guides/code_examples/storages/helper_enqueue_links_example.py index 6c7392bc3b..72862cbf15 100644 --- a/docs/guides/code_examples/storages/helper_enqueue_links_example.py +++ b/docs/guides/code_examples/storages/helper_enqueue_links_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -13,7 +11,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await context.enqueue_links() await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/kvs_basic_example.py b/docs/guides/code_examples/storages/kvs_basic_example.py index 7821fa75de..aa18cf88a5 100644 --- a/docs/guides/code_examples/storages/kvs_basic_example.py +++ b/docs/guides/code_examples/storages/kvs_basic_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.storages import KeyValueStore @@ -20,7 +18,3 @@ async def main() -> None: # Remove the key-value store. await kvs.drop() - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/kvs_with_crawler_example.py b/docs/guides/code_examples/storages/kvs_with_crawler_example.py index 732ee41f76..781a7c18ec 100644 --- a/docs/guides/code_examples/storages/kvs_with_crawler_example.py +++ b/docs/guides/code_examples/storages/kvs_with_crawler_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -29,7 +27,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial URLs. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py b/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py index 66a921bd04..19ad386415 100644 --- a/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py +++ b/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.storages import KeyValueStore @@ -30,7 +28,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial URLs. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/rq_basic_example.py b/docs/guides/code_examples/storages/rq_basic_example.py index 9e983bb9fe..9611d1ce03 100644 --- a/docs/guides/code_examples/storages/rq_basic_example.py +++ b/docs/guides/code_examples/storages/rq_basic_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.storages import RequestQueue @@ -25,7 +23,3 @@ async def main() -> None: # Remove the request queue. await request_queue.drop() - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/rq_with_crawler_example.py b/docs/guides/code_examples/storages/rq_with_crawler_example.py index ce6a34cb59..1f3fd8ea0a 100644 --- a/docs/guides/code_examples/storages/rq_with_crawler_example.py +++ b/docs/guides/code_examples/storages/rq_with_crawler_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import HttpCrawler, HttpCrawlingContext @@ -21,7 +19,3 @@ async def request_handler(context: HttpCrawlingContext) -> None: # Run the crawler. You can optionally pass the list of initial requests. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py b/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py index 21bedad0b9..c63fcccfc5 100644 --- a/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py +++ b/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import HttpCrawler, HttpCrawlingContext from crawlee.storages import RequestQueue @@ -25,7 +23,3 @@ async def request_handler(context: HttpCrawlingContext) -> None: # And execute the crawler. await crawler.run() - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/02_bs.py b/docs/introduction/code_examples/02_bs.py index 6e5ee30069..7da7389f3c 100644 --- a/docs/introduction/code_examples/02_bs.py +++ b/docs/introduction/code_examples/02_bs.py @@ -1,5 +1,3 @@ -import asyncio - # Add import of crawler and crawling context. from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.storages import RequestQueue @@ -24,7 +22,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: context.log.info(f'The title of {url} is: {title}.') await crawler.run() - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/02_bs_better.py b/docs/introduction/code_examples/02_bs_better.py index 1a985722b6..dddfb79181 100644 --- a/docs/introduction/code_examples/02_bs_better.py +++ b/docs/introduction/code_examples/02_bs_better.py @@ -1,5 +1,3 @@ -import asyncio - # You don't need to import RequestQueue anymore. from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -15,7 +13,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Start the crawler with the provided URLs. await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/02_request_queue.py b/docs/introduction/code_examples/02_request_queue.py index e6cc5eb8c3..4e11ad4aa0 100644 --- a/docs/introduction/code_examples/02_request_queue.py +++ b/docs/introduction/code_examples/02_request_queue.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.storages import RequestQueue @@ -9,7 +7,3 @@ async def main() -> None: # And then you add one or more requests to it. await rq.add_request('https://crawlee.dev') - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/03_finding_new_links.py b/docs/introduction/code_examples/03_finding_new_links.py index b84a5d32b3..d30a0db3e4 100644 --- a/docs/introduction/code_examples/03_finding_new_links.py +++ b/docs/introduction/code_examples/03_finding_new_links.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -18,7 +16,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await context.enqueue_links() await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/03_original_code.py b/docs/introduction/code_examples/03_original_code.py index 976e84d562..bf19fd2ad8 100644 --- a/docs/introduction/code_examples/03_original_code.py +++ b/docs/introduction/code_examples/03_original_code.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -13,7 +11,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: context.log.info(f'The title of {url} is: {title}.') await crawler.run(['https://crawlee.dev/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/04_sanity_check.py b/docs/introduction/code_examples/04_sanity_check.py index 5bfbccd27e..b83a021b6e 100644 --- a/docs/introduction/code_examples/04_sanity_check.py +++ b/docs/introduction/code_examples/04_sanity_check.py @@ -1,5 +1,3 @@ -import asyncio - # Instead of BeautifulSoupCrawler let's use Playwright to be able to render JavaScript. from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -26,7 +24,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: context.log.info(f'CATEGORY_{i + 1}: {text}') await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/05_crawling_detail.py b/docs/introduction/code_examples/05_crawling_detail.py index a6845f23b0..6a4ab84963 100644 --- a/docs/introduction/code_examples/05_crawling_detail.py +++ b/docs/introduction/code_examples/05_crawling_detail.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -51,7 +49,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/05_crawling_listing.py b/docs/introduction/code_examples/05_crawling_listing.py index c9c47f57d8..fb481058da 100644 --- a/docs/introduction/code_examples/05_crawling_listing.py +++ b/docs/introduction/code_examples/05_crawling_listing.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -22,7 +20,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/06_scraping.py b/docs/introduction/code_examples/06_scraping.py index bc07c42690..9a250a8c1a 100644 --- a/docs/introduction/code_examples/06_scraping.py +++ b/docs/introduction/code_examples/06_scraping.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -91,7 +89,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/07_final_code.py b/docs/introduction/code_examples/07_final_code.py index a6874d06fe..333114f9e0 100644 --- a/docs/introduction/code_examples/07_final_code.py +++ b/docs/introduction/code_examples/07_final_code.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -91,7 +89,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/08_main.py b/docs/introduction/code_examples/08_main.py index 7908c344ce..9263c18123 100644 --- a/docs/introduction/code_examples/08_main.py +++ b/docs/introduction/code_examples/08_main.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler from .routes import router @@ -14,7 +12,3 @@ async def main() -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/introduction/code_examples/09_apify_sdk.py b/docs/introduction/code_examples/09_apify_sdk.py index 5da6bd3a0d..e2f34b2979 100644 --- a/docs/introduction/code_examples/09_apify_sdk.py +++ b/docs/introduction/code_examples/09_apify_sdk.py @@ -1,5 +1,3 @@ -import asyncio - # highlight-next-line from apify import Actor @@ -19,7 +17,3 @@ async def main() -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py index 7b0a0d6cd9..ef5ad367d8 100644 --- a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py +++ b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext @@ -25,7 +23,3 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Add first URL to the queue and start the crawl. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/quick-start/code_examples/parsel_crawler_example.py b/docs/quick-start/code_examples/parsel_crawler_example.py index 04c4b1ebf6..5ccdf8ff4e 100644 --- a/docs/quick-start/code_examples/parsel_crawler_example.py +++ b/docs/quick-start/code_examples/parsel_crawler_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import ParselCrawler, ParselCrawlingContext @@ -25,7 +23,3 @@ async def request_handler(context: ParselCrawlingContext) -> None: # Add first URL to the queue and start the crawl. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/quick-start/code_examples/playwright_crawler_example.py b/docs/quick-start/code_examples/playwright_crawler_example.py index 1bc30ae320..fa6d8a6de4 100644 --- a/docs/quick-start/code_examples/playwright_crawler_example.py +++ b/docs/quick-start/code_examples/playwright_crawler_example.py @@ -1,5 +1,3 @@ -import asyncio - from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -25,7 +23,3 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Add first URL to the queue and start the crawl. await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) From 8dc3c2dbf25b92b6c7578814b7ce622801a1b7ad Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 1 Apr 2025 09:44:37 +0200 Subject: [PATCH 02/16] add python actor id --- website/src/components/RunnableCodeBlock.jsx | 1 + 1 file changed, 1 insertion(+) diff --git a/website/src/components/RunnableCodeBlock.jsx b/website/src/components/RunnableCodeBlock.jsx index c7b8e2d65a..31b5af5ebe 100644 --- a/website/src/components/RunnableCodeBlock.jsx +++ b/website/src/components/RunnableCodeBlock.jsx @@ -5,6 +5,7 @@ import Link from '@docusaurus/Link'; import styles from './RunnableCodeBlock.module.css'; const EXAMPLE_RUNNERS = { + python: 'HH9rhkFXiZbheuq1V', playwright: '6i5QsHBMtm3hKph70', puppeteer: '7tWSD8hrYzuc9Lte7', cheerio: 'kk67IcZkKSSBTslXI', From 796a517fc62865bc5ee3e480cf3cfbe698596d36 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 1 Apr 2025 09:55:31 +0200 Subject: [PATCH 03/16] code example back to separate file --- website/src/pages/home_page_example.py | 37 +++++++++++++++++++ website/src/pages/index.js | 49 ++------------------------ 2 files changed, 40 insertions(+), 46 deletions(-) create mode 100644 website/src/pages/home_page_example.py diff --git a/website/src/pages/home_page_example.py b/website/src/pages/home_page_example.py new file mode 100644 index 0000000000..fe5b80eebc --- /dev/null +++ b/website/src/pages/home_page_example.py @@ -0,0 +1,37 @@ +from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext + + +async def main() -> None: + crawler = PlaywrightCrawler( + max_requests_per_crawl=5, # Limit the crawl to 5 requests at most. + headless=False, # Show the browser window. + browser_type='firefox', # Use the Firefox browser. + ) + + # Define the default request handler, which will be called for every request. + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + + # Extract and enqueue all links found on the page. + await context.enqueue_links() + + # Extract data from the page using Playwright API. + data = { + 'url': context.request.url, + 'title': await context.page.title(), + 'content': (await context.page.content())[:100], + } + + # Push the extracted data to the default dataset. + await context.push_data(data) + + # Run the crawler with the initial list of URLs. + await crawler.run(['https://crawlee.dev']) + + # Export the entire dataset to a JSON file. + await crawler.export_data('results.json') + + # Or work with the data directly. + data = await crawler.get_data() + crawler.log.info(f'Extracted data: {data.items}') diff --git a/website/src/pages/index.js b/website/src/pages/index.js index 17dd7b6eeb..272e1738ad 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -16,6 +16,8 @@ import LanguageInfoWidget from '../components/Homepage/LanguageInfoWidget'; import RiverSection from '../components/Homepage/RiverSection'; import ThreeCardsWithIcon from '../components/Homepage/ThreeCardsWithIcon'; +import HomePageExample from '!!raw-loader!./home_page_example.py'; + function GetStartedSection() { return (
@@ -28,57 +30,12 @@ function GetStartedSection() { ); } -const example = `import asyncio - -from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext - - -async def main() -> None: - crawler = PlaywrightCrawler( - max_requests_per_crawl=5, # Limit the crawl to 5 requests at most. - headless=False, # Show the browser window. - browser_type='firefox', # Use the Firefox browser. - ) - - # Define the default request handler, which will be called for every request. - @crawler.router.default_handler - async def request_handler(context: PlaywrightCrawlingContext) -> None: - context.log.info(f'Processing {context.request.url} ...') - - # Extract and enqueue all links found on the page. - await context.enqueue_links() - - # Extract data from the page using Playwright API. - data = { - 'url': context.request.url, - 'title': await context.page.title(), - 'content': (await context.page.content())[:100], - } - - # Push the extracted data to the default dataset. - await context.push_data(data) - - # Run the crawler with the initial list of URLs. - await crawler.run(['https://crawlee.dev']) - - # Export the entire dataset to a JSON file. - await crawler.export_data('results.json') - - # Or work with the data directly. - data = await crawler.get_data() - crawler.log.info(f'Extracted data: {data.items}') - - -if __name__ == '__main__': - asyncio.run(main()) -`; - function CodeExampleSection() { return (
- {example} + {HomePageExample}
From 3c46049ffab749f4bd057975b1df9a040ce593fd Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 1 Apr 2025 11:08:01 +0200 Subject: [PATCH 04/16] integrating runnable code example --- website/roa-loader/index.js | 26 ++++++++++++++------ website/src/components/RunnableCodeBlock.jsx | 12 +++------ website/src/pages/index.js | 7 ++++-- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/website/roa-loader/index.js b/website/roa-loader/index.js index 9673ca94e3..d4826ad554 100644 --- a/website/roa-loader/index.js +++ b/website/roa-loader/index.js @@ -1,13 +1,25 @@ -const { inspect } = require('util'); +const { createHash } = require('node:crypto'); +const { inspect } = require('node:util'); const { urlToRequest } = require('loader-utils'); const signingUrl = new URL('https://api.apify.com/v2/tools/encode-and-sign'); signingUrl.searchParams.set('token', process.env.APIFY_SIGNING_TOKEN); const queue = []; +const cache = {}; let working = false; +function hash(source) { + return createHash('sha1').update(source).digest('hex'); +} + async function getHash(source) { + const cacheKey = hash(source); + + if (cache[cacheKey]) { + return cache[cacheKey]; + } + const memory = source.match(/playwright|puppeteer/i) ? 4096 : 1024; const res = await (await fetch(signingUrl, { method: 'POST', @@ -32,13 +44,14 @@ async function getHash(source) { const body = await res.json(); - await new Promise((resolve) => setTimeout(resolve, 100)); - if (!body.data || !body.data.encoded) { console.error(`Signing failed:' ${inspect(body.error) || 'Unknown error'}`, body); return 'invalid-token'; } + cache[cacheKey] = body.data.encoded; + await new Promise((resolve) => setTimeout(resolve, 100)); + return body.data.encoded; } @@ -72,14 +85,11 @@ async function encodeAndSign(source) { } module.exports = async function (code) { - // TODO enable once we have python example runner actor - return { code }; - if (process.env.CRAWLEE_DOCS_FAST) { return { code, hash: 'fast' }; } console.log(`Signing ${urlToRequest(this.resourcePath)}...`, { working, queue: queue.length }); - const hash = await encodeAndSign(code); - return { code, hash }; + const codeHash = await encodeAndSign(code); + return { code, hash: codeHash }; }; diff --git a/website/src/components/RunnableCodeBlock.jsx b/website/src/components/RunnableCodeBlock.jsx index 31b5af5ebe..171175b73d 100644 --- a/website/src/components/RunnableCodeBlock.jsx +++ b/website/src/components/RunnableCodeBlock.jsx @@ -1,17 +1,13 @@ + import React from 'react'; import clsx from 'clsx'; import CodeBlock from '@theme/CodeBlock'; import Link from '@docusaurus/Link'; import styles from './RunnableCodeBlock.module.css'; -const EXAMPLE_RUNNERS = { - python: 'HH9rhkFXiZbheuq1V', - playwright: '6i5QsHBMtm3hKph70', - puppeteer: '7tWSD8hrYzuc9Lte7', - cheerio: 'kk67IcZkKSSBTslXI', -}; +const PYTHON_ACTOR_RUNNER = 'HH9rhkFXiZbheuq1V' -const RunnableCodeBlock = ({ children, actor, hash, type, ...props }) => { +const RunnableCodeBlock = ({ children, actor, hash, ...props }) => { hash = hash ?? children.hash; if (!children.code) { @@ -27,7 +23,7 @@ Make sure you are importing the code block contents with the roa-loader.`); ); } - const href = `https://console.apify.com/actors/${actor ?? EXAMPLE_RUNNERS[type ?? 'playwright']}?runConfig=${hash}&asrc=run_on_apify`; + const href = `https://console.apify.com/actors/${actor ?? PYTHON_ACTOR_RUNNER}?runConfig=${hash}&asrc=run_on_apify`; return (
diff --git a/website/src/pages/index.js b/website/src/pages/index.js index 272e1738ad..aa4aec890d 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -14,9 +14,10 @@ import HomepageCtaSection from '../components/Homepage/HomepageCtaSection'; import HomepageHeroSection from '../components/Homepage/HomepageHeroSection'; import LanguageInfoWidget from '../components/Homepage/LanguageInfoWidget'; import RiverSection from '../components/Homepage/RiverSection'; +import RunnableCodeBlock from '../components/RunnableCodeBlock'; import ThreeCardsWithIcon from '../components/Homepage/ThreeCardsWithIcon'; -import HomePageExample from '!!raw-loader!./home_page_example.py'; +import HomePageExample from '!!raw-loader!roa-loader!./home_page_example.py'; function GetStartedSection() { return ( @@ -35,7 +36,9 @@ function CodeExampleSection() {
- {HomePageExample} + + {HomePageExample} +
From e01669a1c00d86db6d54ed352f3c369cf092364a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 1 Apr 2025 11:56:48 +0200 Subject: [PATCH 05/16] tabs do not work --- docs/quick-start/index.mdx | 25 +++++++++++++------------ website/src/pages/index.js | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/quick-start/index.mdx b/docs/quick-start/index.mdx index ffe7306dbb..b3f376644e 100644 --- a/docs/quick-start/index.mdx +++ b/docs/quick-start/index.mdx @@ -7,11 +7,12 @@ import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulsoupCrawlerExample from '!!raw-loader!./code_examples/beautifulsoup_crawler_example.py'; -import ParselCrawlerExample from '!!raw-loader!./code_examples/parsel_crawler_example.py'; -import PlaywrightCrawlerExample from '!!raw-loader!./code_examples/playwright_crawler_example.py'; -import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/playwright_crawler_headful_example.py'; +import BeautifulsoupCrawlerExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_example.py'; +import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler_example.py'; +import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_example.py'; +import PlaywrightCrawlerHeadfulExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_headful_example.py'; This short tutorial will help you start scraping with Crawlee in just a minute or two. For an in-depth understanding of how Crawlee works, check out the [Introduction](../introduction/index.mdx) section, which provides a comprehensive step-by-step guide to creating your first scraper. @@ -65,21 +66,21 @@ For detailed installation instructions see the [Setting up](../introduction/01_s Run the following example to perform a recursive crawl of the Crawlee website using the selected crawler. - - - + + + {BeautifulsoupCrawlerExample} - + - + {ParselCrawlerExample} - + - + {PlaywrightCrawlerExample} - + diff --git a/website/src/pages/index.js b/website/src/pages/index.js index aa4aec890d..fd53f43f7e 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -36,7 +36,7 @@ function CodeExampleSection() {
- + {HomePageExample}
From db6f16fd6e45003fec4a44ac36c82967bf909b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Tue, 1 Apr 2025 16:02:38 +0200 Subject: [PATCH 06/16] fix quick start --- docs/quick-start/index.mdx | 8 ++++---- website/src/components/RunnableCodeBlock.jsx | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/quick-start/index.mdx b/docs/quick-start/index.mdx index b3f376644e..7314e82efa 100644 --- a/docs/quick-start/index.mdx +++ b/docs/quick-start/index.mdx @@ -60,7 +60,7 @@ If you plan to use the `PlaywrightCrawler` playwright install ``` -For detailed installation instructions see the [Setting up](../introduction/01_setting_up.mdx) documentation page. +For detailed installation instructions, see the [Setting up](../introduction/01_setting_up.mdx) documentation page. ## Crawling @@ -90,11 +90,11 @@ When you run the example, you will see Crawlee automating the data extraction pr ## Running headful browser -By default, browsers controlled by Playwright run in headless mode (without a visible window). However, you can configure the crawler to run in a headful mode, which is useful during development phase to observe the browser's actions. You can alsoswitch from the default Chromium browser to Firefox or WebKit. +By default, browsers controlled by Playwright run in headless mode (without a visible window). However, you can configure the crawler to run in a headful mode, which is useful during the development phase to observe the browser's actions. You can also switch from the default Chromium browser to Firefox or WebKit. - + {PlaywrightCrawlerHeadfulExample} - + When you run the example code, you'll see an automated browser navigating through the Crawlee website. diff --git a/website/src/components/RunnableCodeBlock.jsx b/website/src/components/RunnableCodeBlock.jsx index 171175b73d..4af7507033 100644 --- a/website/src/components/RunnableCodeBlock.jsx +++ b/website/src/components/RunnableCodeBlock.jsx @@ -1,4 +1,3 @@ - import React from 'react'; import clsx from 'clsx'; import CodeBlock from '@theme/CodeBlock'; From 20607a03b928b169137adbfcab87a81d557dd5e0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 1 Apr 2025 17:54:41 +0200 Subject: [PATCH 07/16] update contributing guide --- CONTRIBUTING.md | 38 +++----------------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e22fa1b791..67d558f5a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,43 +82,11 @@ TODO: ## Documentation -We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). +We follow the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for code documentation. All user-facing classes and functions must be documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). -Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages. +Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with custom post-processing. Additional content is provided through markdown files in the `docs/` directory. The final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub pages. -To run the documentation locally, you need to have Node.js version 20 or higher installed. Once you have the correct version of Node.js, follow these steps: - -Navigate to the `website/` directory: - -```sh -cd website/ -``` - -Enable Corepack, which installs Yarn automatically: - -```sh -corepack enable -``` - -Build the API reference: - -```sh -./build_api_reference.sh -``` - -Install the necessary dependencies: - -```sh -yarn -``` - -Start the project in development mode with Hot Module Replacement (HMR): - -```sh -yarn start -``` - -Or using `make`: +To run the documentation locally, ensure you have `Node.js` 20+ installed, then run: ```sh make run-docs From dd010f03e139d1dd7a8014a5dd0751d428720574 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 1 Apr 2025 19:52:29 +0200 Subject: [PATCH 08/16] quick start & introduction --- .../apify/crawler_as_actor_example.py | 2 +- .../adaptive_playwright_crawler.py | 2 +- docs/introduction/02_first_crawler.mdx | 20 ++++++------ docs/introduction/03_adding_more_urls.mdx | 32 +++++++++---------- docs/introduction/04_real_world_project.mdx | 8 ++--- docs/introduction/05_crawling.mdx | 14 ++++---- docs/introduction/06_scraping.mdx | 12 ++++--- docs/introduction/07_saving_data.mdx | 10 +++--- .../code_examples/03_enqueue_strategy.py | 2 +- .../code_examples/03_finding_new_links.py | 2 +- docs/introduction/code_examples/03_globs.py | 2 +- .../code_examples/03_transform_request.py | 2 +- .../introduction/code_examples/06_scraping.py | 2 +- .../code_examples/07_final_code.py | 2 +- docs/introduction/code_examples/08_main.py | 2 +- .../code_examples/09_apify_sdk.py | 2 +- .../beautifulsoup_crawler_example.py | 2 +- .../code_examples/parsel_crawler_example.py | 2 +- .../playwright_crawler_headful_example.py | 1 + docs/quick-start/index.mdx | 7 ++-- 20 files changed, 67 insertions(+), 61 deletions(-) diff --git a/docs/deployment/code_examples/apify/crawler_as_actor_example.py b/docs/deployment/code_examples/apify/crawler_as_actor_example.py index d14356623d..8637340557 100644 --- a/docs/deployment/code_examples/apify/crawler_as_actor_example.py +++ b/docs/deployment/code_examples/apify/crawler_as_actor_example.py @@ -6,7 +6,7 @@ async def main() -> None: # Wrap the crawler code in an Actor context manager. async with Actor: - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: diff --git a/docs/examples/code_examples/adaptive_playwright_crawler.py b/docs/examples/code_examples/adaptive_playwright_crawler.py index 7513cbba6d..1827a5ef13 100644 --- a/docs/examples/code_examples/adaptive_playwright_crawler.py +++ b/docs/examples/code_examples/adaptive_playwright_crawler.py @@ -13,7 +13,7 @@ async def main() -> None: # Crawler created by following factory method will use `beautifulsoup` # for parsing static content. crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( - max_requests_per_crawl=5, playwright_crawler_specific_kwargs={'headless': False} + max_requests_per_crawl=10, playwright_crawler_specific_kwargs={'headless': False} ) @crawler.router.default_handler diff --git a/docs/introduction/02_first_crawler.mdx b/docs/introduction/02_first_crawler.mdx index 26dafd8ad2..203ab92146 100644 --- a/docs/introduction/02_first_crawler.mdx +++ b/docs/introduction/02_first_crawler.mdx @@ -4,11 +4,11 @@ title: First crawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RequestQueueExample from '!!raw-loader!./code_examples/02_request_queue.py'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/02_bs.py'; -import BeautifulSoupBetterExample from '!!raw-loader!./code_examples/02_bs_better.py'; +import RequestQueueExample from '!!raw-loader!roa-loader!./code_examples/02_request_queue.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/02_bs.py'; +import BeautifulSoupBetterExample from '!!raw-loader!roa-loader!./code_examples/02_bs_better.py'; Now, you will build your first crawler. But before you do, let's briefly introduce the Crawlee classes involved in the process. @@ -44,9 +44,9 @@ Let's put the theory into practice and start with something easy. Visit a page a Earlier you learned that the crawler uses a queue of requests as its source of URLs to crawl. Let's create it and add the first request. - + {RequestQueueExample} - + The `RequestQueue.add_request` method automatically converts the object with URL string to a `Request` instance. So now you have a `RequestQueue` that holds one request which points to `https://crawlee.dev`. @@ -64,9 +64,9 @@ Unless you have a good reason to start with a different one, you should try buil Let's continue with the earlier `RequestQueue` example. - + {BeautifulSoupExample} - + When you run the example, you will see the title of https://crawlee.dev printed to the log. What really happens is that `BeautifulSoupCrawler` first makes an HTTP request to `https://crawlee.dev`, then parses the received HTML with BeautifulSoup and makes it available as the `context` argument of the request handler. @@ -78,9 +78,9 @@ When you run the example, you will see the title of https://crawlee.dev printed Earlier we mentioned that you'll learn how to use the `BasicCrawler.add_requests` method to skip the request queue initialization. It's simple. Every crawler has an implicit `RequestQueue` instance, and you can add requests to it with the `BasicCrawler.add_requests` method. In fact, you can go even further and just use the first parameter of `crawler.run()`! - + {BeautifulSoupBetterExample} - + When you run this code, you'll see exactly the same output as with the earlier, longer example. The `RequestQueue` is still there, it's just managed by the crawler automatically. diff --git a/docs/introduction/03_adding_more_urls.mdx b/docs/introduction/03_adding_more_urls.mdx index 9e7f7bab58..a9669fb8a3 100644 --- a/docs/introduction/03_adding_more_urls.mdx +++ b/docs/introduction/03_adding_more_urls.mdx @@ -4,19 +4,19 @@ title: Adding more URLs --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import OriginalCodeExample from '!!raw-loader!./code_examples/03_original_code.py'; -import FindingNewLinksExample from '!!raw-loader!./code_examples/03_finding_new_links.py'; -import EnqueueStrategyExample from '!!raw-loader!./code_examples/03_enqueue_strategy.py'; -import GlobsExample from '!!raw-loader!./code_examples/03_globs.py'; -import TransformExample from '!!raw-loader!./code_examples/03_transform_request.py'; +import OriginalCodeExample from '!!raw-loader!roa-loader!./code_examples/03_original_code.py'; +import FindingNewLinksExample from '!!raw-loader!roa-loader!./code_examples/03_finding_new_links.py'; +import EnqueueStrategyExample from '!!raw-loader!roa-loader!./code_examples/03_enqueue_strategy.py'; +import GlobsExample from '!!raw-loader!roa-loader!./code_examples/03_globs.py'; +import TransformExample from '!!raw-loader!roa-loader!./code_examples/03_transform_request.py'; Previously you've built a very simple crawler that downloads HTML of a single page, reads its title and prints it to the console. This is the original source code: - + {OriginalCodeExample} - + Now you'll use the example from the previous section and improve on it. You'll add more URLs to the queue and thanks to that the crawler will keep going, finding new links, enqueuing them into the `RequestQueue` and then scraping them. @@ -58,9 +58,9 @@ There are numerous approaches to finding links to follow when crawling the web. Since this is the most common case, it is also the `enqueue_links` default. - + {FindingNewLinksExample} - + If you need to override the default selection of elements in `enqueue_links`, you can use the `selector` argument. @@ -80,9 +80,9 @@ await context.enqueue_links() The default behavior of `enqueue_links` is to stay on the same hostname. This **does not include subdomains**. To include subdomains in your crawl, use the `strategy` argument. The `strategy` argument is an instance of the `EnqueueStrategy` type alias. - + {EnqueueStrategyExample} - + When you run the code, you will see the crawler log the **title** of the first page, then the **enqueueing** message showing number of URLs, followed by the **title** of the first enqueued page and so on and so on. @@ -103,17 +103,17 @@ await context.enqueue_links(strategy='all') For even more control, you can use the `include` or `exclude` parameters, either as glob patterns or regular expressions, to filter the URLs. Refer to the API documentation for `enqueue_links` for detailed information on these and other available options. - + {GlobsExample} - + ### Transform requests before enqueuing For cases where you need to modify or filter requests before they are enqueued, you can use the `transform_request_function` parameter. This function takes a `Request` object as input and should return either a modified `Request`` object or `None`. If the function returns `None`, the request will be skipped. - + {TransformExample} - + ## Next steps diff --git a/docs/introduction/04_real_world_project.mdx b/docs/introduction/04_real_world_project.mdx index 649f648f89..61f6435980 100644 --- a/docs/introduction/04_real_world_project.mdx +++ b/docs/introduction/04_real_world_project.mdx @@ -4,9 +4,9 @@ title: Real-world project --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import SanityCheckExample from '!!raw-loader!./code_examples/04_sanity_check.py'; +import SanityCheckExample from '!!raw-loader!roa-loader!./code_examples/04_sanity_check.py'; > _Hey, guys, you know, it's cool that we can scrape the `` elements of web pages, but that's not very useful. Can we finally scrape some real data and save it somewhere in a machine-readable format? Because that's why I started reading this tutorial in the first place!_ @@ -110,9 +110,9 @@ Let's check that everything is set up correctly before writing the scraping logi The example below creates a new crawler that visits the start URL and prints the text content of all the categories on that page. When you run the code, you will see the _very badly formatted_ content of the individual category card. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {SanityCheckExample} -</CodeBlock> +</RunnableCodeBlock> If you're wondering how to get that `.collection-block-item` selector. We'll explain it in the next chapter on DevTools. diff --git a/docs/introduction/05_crawling.mdx b/docs/introduction/05_crawling.mdx index 1ced51ff50..7c68662766 100644 --- a/docs/introduction/05_crawling.mdx +++ b/docs/introduction/05_crawling.mdx @@ -4,10 +4,10 @@ title: Crawling --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import CrawlingListingExample from '!!raw-loader!./code_examples/05_crawling_listing.py'; -import CrawlingDetailExample from '!!raw-loader!./code_examples/05_crawling_detail.py'; +import CrawlingListingExample from '!!raw-loader!roa-loader!./code_examples/05_crawling_listing.py'; +import CrawlingDetailExample from '!!raw-loader!roa-loader!./code_examples/05_crawling_detail.py'; To crawl the whole [Warehouse store example](https://warehouse-theme-metal.myshopify.com/collections) and find all the data, you first need to visit all the pages with products - going through all categories available and also all the product detail pages. @@ -21,9 +21,9 @@ await enqueue_links() While useful in that scenario, you need something different now. Instead of finding all the `<a href="..">` elements with links to the same hostname, you need to find only the specific ones that will take your crawler to the next page of results. Otherwise, the crawler will visit a lot of other pages that you're not interested in. Using the power of DevTools and yet another <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> parameter, this becomes fairly easy. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {CrawlingListingExample} -</CodeBlock> +</RunnableCodeBlock> The code should look pretty familiar to you. It's a very simple request handler where we log the currently processed URL to the console and enqueue more links. But there are also a few new, interesting additions. Let's break it down. @@ -39,9 +39,9 @@ You will see `label` used often throughout Crawlee, as it's a convenient way of In a similar fashion, you need to collect all the URLs to the product detail pages, because only from there you can scrape all the data you need. The following code only repeats the concepts you already know for another set of links. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {CrawlingDetailExample} -</CodeBlock> +</RunnableCodeBlock> The crawling code is now complete. When you run the code, you'll see the crawler visit all the listing URLs and all the detail URLs. diff --git a/docs/introduction/06_scraping.mdx b/docs/introduction/06_scraping.mdx index 55393894f1..443819a3de 100644 --- a/docs/introduction/06_scraping.mdx +++ b/docs/introduction/06_scraping.mdx @@ -4,9 +4,9 @@ title: Scraping --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import ScrapingExample from '!!raw-loader!./code_examples/06_scraping.py'; +import ScrapingExample from '!!raw-loader!roa-loader!./code_examples/06_scraping.py'; In the [Real-world project](./real-world-project#choosing-the-data-you-need) chapter, you've created a list of the information you wanted to collect about the products in the example Warehouse store. Let's review that and figure out ways to access the data. @@ -32,7 +32,9 @@ You can use `request.loaded_url` as well. Remember the difference: `request.url` By splitting the `request.url`, we can extract the manufacturer name directly from the URL. This is done by first splitting the URL to get the product identifier and then splitting that identifier to get the manufacturer name. ```python -# context.request.url: https://warehouse-theme-metal.myshopify.com/products/sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440 +# context.request.url: +# https://warehouse-theme-metal.myshopify.com/products/ +# sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440 # Split the URL and get the last part. url_part = context.request.url.split('/').pop() @@ -132,9 +134,9 @@ For this, all that matters is whether the element exists or not. You can use the You have everything that is needed, so grab your newly created scraping logic, dump it into your original request handler and see the magic happen! -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {ScrapingExample} -</CodeBlock> +</RunnableCodeBlock> When you run the crawler, you will see the crawled URLs and their scraped data printed to the console. The output will look something like this: diff --git a/docs/introduction/07_saving_data.mdx b/docs/introduction/07_saving_data.mdx index 4ba5394861..adddd93af9 100644 --- a/docs/introduction/07_saving_data.mdx +++ b/docs/introduction/07_saving_data.mdx @@ -5,10 +5,12 @@ title: Saving data import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import FinalCodeExample from '!!raw-loader!./code_examples/07_final_code.py'; import FirstCodeExample from '!!raw-loader!./code_examples/07_first_code.py'; +import FinalCodeExample from '!!raw-loader!roa-loader!./code_examples/07_final_code.py'; + A data extraction job would not be complete without saving the data for later use and processing. You've come to the final and most difficult part of this tutorial so make sure to pay attention very carefully! ## Save data to the dataset @@ -20,7 +22,7 @@ Crawlee provides a <ApiLink to="class/Dataset">`Dataset`</ApiLink> class, which Here's an example: -<CodeBlock className="language-python"> +<CodeBlock language="python"> {FirstCodeExample} </CodeBlock> @@ -76,9 +78,9 @@ Instead of importing a new class and manually creating an instance of the datase And that's it. Unlike earlier, we are being serious now. That's it, you're done. The final code looks like this: -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {FinalCodeExample} -</CodeBlock> +</RunnableCodeBlock> ## What `push_data` does? diff --git a/docs/introduction/code_examples/03_enqueue_strategy.py b/docs/introduction/code_examples/03_enqueue_strategy.py index ca68c0823d..322769adb3 100644 --- a/docs/introduction/code_examples/03_enqueue_strategy.py +++ b/docs/introduction/code_examples/03_enqueue_strategy.py @@ -2,7 +2,7 @@ async def main() -> None: - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: diff --git a/docs/introduction/code_examples/03_finding_new_links.py b/docs/introduction/code_examples/03_finding_new_links.py index d30a0db3e4..67a935285c 100644 --- a/docs/introduction/code_examples/03_finding_new_links.py +++ b/docs/introduction/code_examples/03_finding_new_links.py @@ -3,7 +3,7 @@ async def main() -> None: # Let's limit our crawls to make our tests shorter and safer. - crawler = BeautifulSoupCrawler(max_requests_per_crawl=20) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: diff --git a/docs/introduction/code_examples/03_globs.py b/docs/introduction/code_examples/03_globs.py index a8a5c8ae78..c605479472 100644 --- a/docs/introduction/code_examples/03_globs.py +++ b/docs/introduction/code_examples/03_globs.py @@ -3,7 +3,7 @@ async def main() -> None: - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: diff --git a/docs/introduction/code_examples/03_transform_request.py b/docs/introduction/code_examples/03_transform_request.py index 18d6d9f0f3..fd2be7a20a 100644 --- a/docs/introduction/code_examples/03_transform_request.py +++ b/docs/introduction/code_examples/03_transform_request.py @@ -27,7 +27,7 @@ def transform_request( async def main() -> None: - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) @crawler.router.default_handler async def request_handler(context: BeautifulSoupCrawlingContext) -> None: diff --git a/docs/introduction/code_examples/06_scraping.py b/docs/introduction/code_examples/06_scraping.py index 9a250a8c1a..9561eb3522 100644 --- a/docs/introduction/code_examples/06_scraping.py +++ b/docs/introduction/code_examples/06_scraping.py @@ -4,7 +4,7 @@ async def main() -> None: crawler = PlaywrightCrawler( # Let's limit our crawls to make our tests shorter and safer. - max_requests_per_crawl=50, + max_requests_per_crawl=10, ) @crawler.router.default_handler diff --git a/docs/introduction/code_examples/07_final_code.py b/docs/introduction/code_examples/07_final_code.py index 333114f9e0..b7379eca8e 100644 --- a/docs/introduction/code_examples/07_final_code.py +++ b/docs/introduction/code_examples/07_final_code.py @@ -4,7 +4,7 @@ async def main() -> None: crawler = PlaywrightCrawler( # Let's limit our crawls to make our tests shorter and safer. - max_requests_per_crawl=50, + max_requests_per_crawl=10, ) @crawler.router.default_handler diff --git a/docs/introduction/code_examples/08_main.py b/docs/introduction/code_examples/08_main.py index 9263c18123..f877cc2e1d 100644 --- a/docs/introduction/code_examples/08_main.py +++ b/docs/introduction/code_examples/08_main.py @@ -6,7 +6,7 @@ async def main() -> None: crawler = PlaywrightCrawler( # Let's limit our crawls to make our tests shorter and safer. - max_requests_per_crawl=50, + max_requests_per_crawl=10, # Provide our router instance to the crawler. request_handler=router, ) diff --git a/docs/introduction/code_examples/09_apify_sdk.py b/docs/introduction/code_examples/09_apify_sdk.py index e2f34b2979..f7a5acd839 100644 --- a/docs/introduction/code_examples/09_apify_sdk.py +++ b/docs/introduction/code_examples/09_apify_sdk.py @@ -11,7 +11,7 @@ async def main() -> None: async with Actor: crawler = PlaywrightCrawler( # Let's limit our crawls to make our tests shorter and safer. - max_requests_per_crawl=50, + max_requests_per_crawl=10, # Provide our router instance to the crawler. request_handler=router, ) diff --git a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py index ef5ad367d8..c69aeea8b7 100644 --- a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py +++ b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py @@ -4,7 +4,7 @@ async def main() -> None: # BeautifulSoupCrawler crawls the web using HTTP requests # and parses HTML using the BeautifulSoup library. - crawler = BeautifulSoupCrawler(max_requests_per_crawl=50) + crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) # Define a request handler to process each crawled page # and attach it to the crawler using a decorator. diff --git a/docs/quick-start/code_examples/parsel_crawler_example.py b/docs/quick-start/code_examples/parsel_crawler_example.py index 5ccdf8ff4e..297abd2b84 100644 --- a/docs/quick-start/code_examples/parsel_crawler_example.py +++ b/docs/quick-start/code_examples/parsel_crawler_example.py @@ -4,7 +4,7 @@ async def main() -> None: # ParselCrawler crawls the web using HTTP requests # and parses HTML using the Parsel library. - crawler = ParselCrawler(max_requests_per_crawl=50) + crawler = ParselCrawler(max_requests_per_crawl=10) # Define a request handler to process each crawled page # and attach it to the crawler using a decorator. diff --git a/docs/quick-start/code_examples/playwright_crawler_headful_example.py b/docs/quick-start/code_examples/playwright_crawler_headful_example.py index c2d02ae0e7..f416f26277 100644 --- a/docs/quick-start/code_examples/playwright_crawler_headful_example.py +++ b/docs/quick-start/code_examples/playwright_crawler_headful_example.py @@ -4,6 +4,7 @@ async def main() -> None: crawler = PlaywrightCrawler( # Run with a visible browser window. + # highlight-next-line headless=False, # Switch to the Firefox browser. browser_type='firefox', diff --git a/docs/quick-start/index.mdx b/docs/quick-start/index.mdx index 7314e82efa..6045e39951 100644 --- a/docs/quick-start/index.mdx +++ b/docs/quick-start/index.mdx @@ -12,7 +12,8 @@ import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import BeautifulsoupCrawlerExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_example.py'; import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler_example.py'; import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_example.py'; -import PlaywrightCrawlerHeadfulExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_headful_example.py'; + +import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/playwright_crawler_headful_example.py'; This short tutorial will help you start scraping with Crawlee in just a minute or two. For an in-depth understanding of how Crawlee works, check out the [Introduction](../introduction/index.mdx) section, which provides a comprehensive step-by-step guide to creating your first scraper. @@ -92,9 +93,9 @@ When you run the example, you will see Crawlee automating the data extraction pr By default, browsers controlled by Playwright run in headless mode (without a visible window). However, you can configure the crawler to run in a headful mode, which is useful during the development phase to observe the browser's actions. You can also switch from the default Chromium browser to Firefox or WebKit. -<RunnableCodeBlock className="language-python" language="python"> +<CodeBlock language="python"> {PlaywrightCrawlerHeadfulExample} -</RunnableCodeBlock> +</CodeBlock> When you run the example code, you'll see an automated browser navigating through the Crawlee website. From fb86880d5c63ae052d6bac606c59fd146db05406 Mon Sep 17 00:00:00 2001 From: Vlada Dusek <v.dusek96@gmail.com> Date: Tue, 1 Apr 2025 21:04:17 +0200 Subject: [PATCH 09/16] guides --- docs/guides/avoid_blocking.mdx | 16 ++-- .../adaptive_playwright_crawler_handler.py | 13 --- ...e_playwright_crawler_init_beautifulsoup.py | 8 -- ...adaptive_playwright_crawler_init_parsel.py | 8 -- ...aptive_playwright_crawler_pre_nav_hooks.py | 29 ------- ...default_fingerprint_generator_with_args.py | 0 .../playwright_with_fingerprint_generator.py | 0 .../playwright_crawler_adaptive/handler.py | 16 ++++ .../init_beautifulsoup.py | 15 ++++ .../init_parsel.py | 15 ++++ .../init_prediction.py} | 13 +-- .../pre_nav_hooks.py | 33 ++++++++ .../max_tasks_per_minute_example.py | 2 + .../min_and_max_concurrency_example.py | 2 + ...ession_management_basic.py => sm_basic.py} | 0 ...t_beautifulsoup.py => sm_beautifulsoup.py} | 0 ...{session_management_http.py => sm_http.py} | 0 ...sion_management_parsel.py => sm_parsel.py} | 0 ...agement_playwright.py => sm_playwright.py} | 0 ...agement_standalone.py => sm_standalone.py} | 0 docs/guides/http_clients.mdx | 14 ++-- docs/guides/http_crawlers.mdx | 13 +-- docs/guides/playwright_crawler.mdx | 20 ++--- docs/guides/playwright_crawler_adaptive.mdx | 46 +++++------ docs/guides/proxy_management.mdx | 44 +++++----- docs/guides/request_loaders.mdx | 21 +++-- docs/guides/scaling_crawlers.mdx | 14 ++-- docs/guides/session_management.mdx | 50 ++++++------ docs/guides/storages.mdx | 80 +++++++++---------- 29 files changed, 253 insertions(+), 219 deletions(-) delete mode 100644 docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py delete mode 100644 docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py delete mode 100644 docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py delete mode 100644 docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py rename docs/guides/code_examples/{browser_fingerprint => avoid_blocking}/default_fingerprint_generator_with_args.py (100%) rename docs/guides/code_examples/{browser_fingerprint => avoid_blocking}/playwright_with_fingerprint_generator.py (100%) create mode 100644 docs/guides/code_examples/playwright_crawler_adaptive/handler.py create mode 100644 docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py create mode 100644 docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py rename docs/guides/code_examples/{adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py => playwright_crawler_adaptive/init_prediction.py} (89%) create mode 100644 docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py rename docs/guides/code_examples/session_management/{session_management_basic.py => sm_basic.py} (100%) rename docs/guides/code_examples/session_management/{session_management_beautifulsoup.py => sm_beautifulsoup.py} (100%) rename docs/guides/code_examples/session_management/{session_management_http.py => sm_http.py} (100%) rename docs/guides/code_examples/session_management/{session_management_parsel.py => sm_parsel.py} (100%) rename docs/guides/code_examples/session_management/{session_management_playwright.py => sm_playwright.py} (100%) rename docs/guides/code_examples/session_management/{session_management_standalone.py => sm_standalone.py} (100%) diff --git a/docs/guides/avoid_blocking.mdx b/docs/guides/avoid_blocking.mdx index daccf1c4d8..3ada4e2446 100644 --- a/docs/guides/avoid_blocking.mdx +++ b/docs/guides/avoid_blocking.mdx @@ -3,13 +3,15 @@ id: avoid-blocking title: Avoid getting blocked description: How to avoid getting blocked when scraping --- + import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightDefaultFingerprintGenerator from '!!raw-loader!./code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py'; -import PlaywrightDefaultFingerprintGeneratorWithArgs from '!!raw-loader!./code_examples/browser_fingerprint/default_fingerprint_generator_with_args.py'; -import PlaywrightWithCamoufox from '!!raw-loader!../examples/code_examples/playwright_crawler_with_camoufox.py'; +import PlaywrightDefaultFingerprintGenerator from '!!raw-loader!roa-loader!./code_examples/avoid_blocking/playwright_with_fingerprint_generator.py'; +import PlaywrightWithCamoufox from '!!raw-loader!roa-loader!../examples/code_examples/playwright_crawler_with_camoufox.py'; +import PlaywrightDefaultFingerprintGeneratorWithArgs from '!!raw-loader!./code_examples/avoid_blocking/default_fingerprint_generator_with_args.py'; A scraper might get blocked for numerous reasons. Let's narrow it down to the two main ones. The first is a bad or blocked IP address. You can learn about this topic in the [proxy management guide](./proxy-management). The second reason is [browser fingerprints](https://pixelprivacy.com/resources/browser-fingerprinting/) (or signatures), which we will explore more in this guide. Check the [Apify Academy anti-scraping course](https://docs.apify.com/academy/anti-scraping) to gain a deeper theoretical understanding of blocking and learn a few tips and tricks. @@ -19,9 +21,9 @@ Browser fingerprint is a collection of browser attributes and significant featur Changing browser fingerprints can be a tedious job. Luckily, Crawlee provides this feature with minimal configuration necessary - the usage of fingerprints in <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> is enabled by default. You can customize the fingerprints by using the `fingerprint_generator` argument of the <ApiLink to="class/PlaywrightCrawler#__init__">`PlaywrightCrawler.__init__`</ApiLink>, either pass your own implementation of <ApiLink to="class/FingerprintGenerator">`FingerprintGenerator`</ApiLink> or use <ApiLink to="class/BrowserforgeFingerprintGenerator">`DefaultFingerprintGenerator`</ApiLink>. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {PlaywrightDefaultFingerprintGenerator} -</CodeBlock> +</RunnableCodeBlock> In certain cases we want to narrow down the fingerprints used - e.g. specify a certain operating system, locale or browser. This is also possible with Crawlee - the crawler can have the generation algorithm customized to reflect the particular browser version and many more. For description of fingerprint generation options please see <ApiLink to="class/HeaderGeneratorOptions">`HeaderGeneratorOptions`</ApiLink>, <ApiLink to="class/ScreenOptions">`ScreenOptions`</ApiLink> and <ApiLink to="class/BrowserforgeFingerprintGenerator#__init__">`DefaultFingerprintGenerator.__init__`</ApiLink> See the example bellow: @@ -35,9 +37,9 @@ If you do not want to use fingerprints, then pass `fingerprint_generator=None` a In some cases even <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> with fingerprints is not enough. You can try using <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> together with [Camoufox](https://camoufox.com/). See the example integration below: -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {PlaywrightWithCamoufox} -</CodeBlock> +</RunnableCodeBlock> **Related links** diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py b/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py deleted file mode 100644 index 5dc12e54ef..0000000000 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py +++ /dev/null @@ -1,13 +0,0 @@ -from datetime import timedelta - -from crawlee.crawlers import AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext - -crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser() - - -@crawler.router.default_handler -async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: - # Locate element h2 within 5 seconds - h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000)) - # Do stuff with element found by the selector - context.log.info(h2) diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py b/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py deleted file mode 100644 index 68aa981eab..0000000000 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py +++ /dev/null @@ -1,8 +0,0 @@ -from crawlee.crawlers import AdaptivePlaywrightCrawler - -crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( - # Arguments relevant only for PlaywrightCrawler - playwright_crawler_specific_kwargs={'headless': False, 'browser_type': 'chromium'}, - # Common arguments relevant to all crawlers - max_crawl_depth=5, -) diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py b/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py deleted file mode 100644 index 1cb64576b3..0000000000 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py +++ /dev/null @@ -1,8 +0,0 @@ -from crawlee.crawlers import AdaptivePlaywrightCrawler - -crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( - # Arguments relevant only for PlaywrightCrawler - playwright_crawler_specific_kwargs={'headless': False, 'browser_type': 'chromium'}, - # Common arguments relevant to all crawlers - max_crawl_depth=5, -) diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py b/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py deleted file mode 100644 index 656997576d..0000000000 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py +++ /dev/null @@ -1,29 +0,0 @@ -from playwright.async_api import Route - -from crawlee.crawlers import ( - AdaptivePlaywrightCrawler, - AdaptivePlaywrightPreNavCrawlingContext, -) - -crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser() - - -@crawler.pre_navigation_hook -async def hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None: - """Hook executed both in static sub crawler and playwright sub crawler. - - Trying to access `context.page` in this hook would raise `AdaptiveContextError` - for pages crawled without playwright.""" - - context.log.info(f'pre navigation hook for: {context.request.url}') - - -@crawler.pre_navigation_hook(playwright_only=True) -async def hook_playwright(context: AdaptivePlaywrightPreNavCrawlingContext) -> None: - """Hook executed only in playwright sub crawler.""" - - async def some_routing_function(route: Route) -> None: - await route.continue_() - - await context.page.route('*/**', some_routing_function) - context.log.info(f'Playwright only pre navigation hook for: {context.request.url}') diff --git a/docs/guides/code_examples/browser_fingerprint/default_fingerprint_generator_with_args.py b/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py similarity index 100% rename from docs/guides/code_examples/browser_fingerprint/default_fingerprint_generator_with_args.py rename to docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py diff --git a/docs/guides/code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py b/docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py similarity index 100% rename from docs/guides/code_examples/browser_fingerprint/playwright_with_fingerprint_generator.py rename to docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/handler.py b/docs/guides/code_examples/playwright_crawler_adaptive/handler.py new file mode 100644 index 0000000000..ed018ddc82 --- /dev/null +++ b/docs/guides/code_examples/playwright_crawler_adaptive/handler.py @@ -0,0 +1,16 @@ +from datetime import timedelta + +from crawlee.crawlers import AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext + + +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser() + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + # Locate element h2 within 5 seconds + h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000)) + # Do stuff with element found by the selector + context.log.info(h2) + + await crawler.run(['https://crawlee.dev/']) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py new file mode 100644 index 0000000000..5dd2215493 --- /dev/null +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py @@ -0,0 +1,15 @@ +from crawlee.crawlers import AdaptivePlaywrightCrawler + + +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + # Arguments relevant only for PlaywrightCrawler + playwright_crawler_specific_kwargs={ + 'headless': False, + 'browser_type': 'chromium', + }, + # Common arguments relevant to all crawlers + max_crawl_depth=5, + ) + + # ... diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py new file mode 100644 index 0000000000..102bc9f9c7 --- /dev/null +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py @@ -0,0 +1,15 @@ +from crawlee.crawlers import AdaptivePlaywrightCrawler + + +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( + # Arguments relevant only for PlaywrightCrawler + playwright_crawler_specific_kwargs={ + 'headless': False, + 'browser_type': 'chromium', + }, + # Common arguments relevant to all crawlers + max_crawl_depth=5, + ) + + # ... diff --git a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py similarity index 89% rename from docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py rename to docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py index f1d8ec1001..d7c2599de7 100644 --- a/docs/guides/code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py @@ -54,8 +54,11 @@ def result_comparator( ) # For example compare `push_data` calls. -crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( - rendering_type_predictor=CustomRenderingTypePredictor(), - result_checker=result_checker, - result_comparator=result_comparator, -) +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser( + rendering_type_predictor=CustomRenderingTypePredictor(), + result_checker=result_checker, + result_comparator=result_comparator, + ) + + # ... diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py b/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py new file mode 100644 index 0000000000..7d33750501 --- /dev/null +++ b/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py @@ -0,0 +1,33 @@ +from playwright.async_api import Route + +from crawlee.crawlers import ( + AdaptivePlaywrightCrawler, + AdaptivePlaywrightPreNavCrawlingContext, +) + + +async def main() -> None: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser() + + @crawler.pre_navigation_hook + async def hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None: + """Hook executed both in static sub crawler and playwright sub crawler. + + Trying to access `context.page` in this hook would raise `AdaptiveContextError` + for pages crawled without playwright. + """ + context.log.info(f'pre navigation hook for: {context.request.url}') + + @crawler.pre_navigation_hook(playwright_only=True) + async def hook_playwright(context: AdaptivePlaywrightPreNavCrawlingContext) -> None: + """Hook executed only in playwright sub crawler.""" + + async def some_routing_function(route: Route) -> None: + await route.continue_() + + await context.page.route('*/**', some_routing_function) + context.log.info( + f'Playwright only pre navigation hook for: {context.request.url}' + ) + + await crawler.run(['https://crawlee.dev/']) diff --git a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py b/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py index e375198698..3871a5bad7 100644 --- a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py +++ b/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py @@ -15,3 +15,5 @@ async def main() -> None: # Apply the defined concurrency settings to the crawler. concurrency_settings=concurrency_settings, ) + + # ... diff --git a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py b/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py index 3ec02aead6..496f059290 100644 --- a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py +++ b/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py @@ -17,3 +17,5 @@ async def main() -> None: # Use the configured concurrency settings for the crawler. concurrency_settings=concurrency_settings, ) + + # ... diff --git a/docs/guides/code_examples/session_management/session_management_basic.py b/docs/guides/code_examples/session_management/sm_basic.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_basic.py rename to docs/guides/code_examples/session_management/sm_basic.py diff --git a/docs/guides/code_examples/session_management/session_management_beautifulsoup.py b/docs/guides/code_examples/session_management/sm_beautifulsoup.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_beautifulsoup.py rename to docs/guides/code_examples/session_management/sm_beautifulsoup.py diff --git a/docs/guides/code_examples/session_management/session_management_http.py b/docs/guides/code_examples/session_management/sm_http.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_http.py rename to docs/guides/code_examples/session_management/sm_http.py diff --git a/docs/guides/code_examples/session_management/session_management_parsel.py b/docs/guides/code_examples/session_management/sm_parsel.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_parsel.py rename to docs/guides/code_examples/session_management/sm_parsel.py diff --git a/docs/guides/code_examples/session_management/session_management_playwright.py b/docs/guides/code_examples/session_management/sm_playwright.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_playwright.py rename to docs/guides/code_examples/session_management/sm_playwright.py diff --git a/docs/guides/code_examples/session_management/session_management_standalone.py b/docs/guides/code_examples/session_management/sm_standalone.py similarity index 100% rename from docs/guides/code_examples/session_management/session_management_standalone.py rename to docs/guides/code_examples/session_management/sm_standalone.py diff --git a/docs/guides/http_clients.mdx b/docs/guides/http_clients.mdx index d6ed89c5b9..2d79dabf8d 100644 --- a/docs/guides/http_clients.mdx +++ b/docs/guides/http_clients.mdx @@ -7,10 +7,10 @@ description: Crawlee supports multiple HTTP clients when making requests. import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BsCurlImpersonateExample from '!!raw-loader!./code_examples/http_clients/curl_impersonate_example.py'; -import BsHttpxExample from '!!raw-loader!./code_examples/http_clients/httpx_example.py'; +import BsCurlImpersonateExample from '!!raw-loader!roa-loader!./code_examples/http_clients/curl_impersonate_example.py'; +import BsHttpxExample from '!!raw-loader!roa-loader!./code_examples/http_clients/httpx_example.py'; HTTP clients are utilized by the HTTP-based crawlers (e.g. <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>) to communicate with web servers. They use external HTTP libraries for communication, rather than a browser. Examples of such libraries include [httpx](https://pypi.org/project/httpx/), [aiohttp](https://pypi.org/project/aiohttp/) or [curl-cffi](https://pypi.org/project/curl-cffi/). After retrieving page content, an HTML parsing library is typically used to facilitate data extraction. Examples of such libraries are [beautifulsoup](https://pypi.org/project/beautifulsoup4/), [parsel](https://pypi.org/project/parsel/), [selectolax](https://pypi.org/project/selectolax/) or [pyquery](https://pypi.org/project/pyquery/). These crawlers are faster than browser-based crawlers but they cannot execute client-side JavaScript. @@ -20,14 +20,14 @@ In Crawlee we currently have two HTTP clients: <ApiLink to="class/HttpxHttpClien <Tabs> <TabItem value="BeautifulSoupHttpxExample" label="BeautifulSoupCrawler with HTTPX"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {BsHttpxExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="BeautifulSoupCurlImpersonateExample" label="BeautifulSoupCrawler with Curl impersonate"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {BsCurlImpersonateExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> diff --git a/docs/guides/http_crawlers.mdx b/docs/guides/http_crawlers.mdx index c8479699ba..42541fe456 100644 --- a/docs/guides/http_crawlers.mdx +++ b/docs/guides/http_crawlers.mdx @@ -7,29 +7,32 @@ description: Crawlee supports multiple HTTP crawlers that can be used to extract import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; Generic class <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink> is parent to <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>, <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> and <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink> and it could be used as parent for your crawler with custom content parsing requirements. It already includes almost all the functionality to crawl webpages and the only missing part is the parser that should be used to parse HTTP responses, and a context dataclass that defines what context helpers will be available to user handler functions. ## `BeautifulSoupCrawler` + <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> uses <ApiLink to="class/BeautifulSoupParser">`BeautifulSoupParser`</ApiLink> to parse the HTTP response and makes it available in <ApiLink to="class/BeautifulSoupCrawlingContext">`BeautifulSoupCrawlingContext`</ApiLink> in the `.soup` or `.parsed_content` attribute. ## `ParselCrawler` + <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> uses <ApiLink to="class/ParselParser">`ParselParser`</ApiLink> to parse the HTTP response and makes it available in <ApiLink to="class/ParselCrawlingContext">`ParselCrawlingContext`</ApiLink> in the `.selector` or `.parsed_content` attribute. ## `HttpCrawler` + <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink> uses <ApiLink to="class/NoParser">`NoParser`</ApiLink> that does not parse the HTTP response at all and is to be used if no parsing is required. -## Creating your own HTTP crawler. +## Creating your own HTTP crawler + ### Why? + In case you want to use some custom parser for parsing HTTP responses, and the rest of the <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink> functionality suit your needs. ### How? + You need to define at least 2 new classes and decide what will be the type returned by the parser's `parse` method. -Parser will inherit from <ApiLink to="class/AbstractHttpParser">`AbstractHttpParser`</ApiLink> and it will need to implement all it's abstract methods. -Crawler will inherit from <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink> and it will need to implement all it's abstract methods. -Newly defined parser is then used in the `parser` argument of `AbstractHttpCrawler.__init__` method. +Parser will inherit from <ApiLink to="class/AbstractHttpParser">`AbstractHttpParser`</ApiLink> and it will need to implement all it's abstract methods. Crawler will inherit from <ApiLink to="class/AbstractHttpCrawler">`AbstractHttpCrawler`</ApiLink> and it will need to implement all it's abstract methods. Newly defined parser is then used in the `parser` argument of `AbstractHttpCrawler.__init__` method. To get better idea and as an example please see one of our own HTTP-based crawlers mentioned above. diff --git a/docs/guides/playwright_crawler.mdx b/docs/guides/playwright_crawler.mdx index 8fc2ccf40a..124f09e8ad 100644 --- a/docs/guides/playwright_crawler.mdx +++ b/docs/guides/playwright_crawler.mdx @@ -6,11 +6,13 @@ description: How to use the PlaywrightCrawler and its related components. import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import MultipleLaunchExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/multiple_launch_example.py'; +import BrowserConfigurationExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/browser_configuration_example.py'; +import PreNavigationExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/pre_navigation_hook_example.py'; -import MultipleLaunchExample from '!!raw-loader!./code_examples/playwright_crawler/multiple_launch_example.py'; -import BrowserConfigurationExample from '!!raw-loader!./code_examples/playwright_crawler/browser_configuration_example.py'; import PluginBrowserConfigExample from '!!raw-loader!./code_examples/playwright_crawler/plugin_browser_configuration_example.py'; -import PreNavigationExample from '!!raw-loader!./code_examples/playwright_crawler/pre_navigation_hook_example.py'; A <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> is a browser-based crawler. In contrast to HTTP-based crawlers like <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> or <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>, it uses a real browser to render pages and extract data. It is built on top of the [Playwright](https://playwright.dev/python/) browser automation library. While browser-based crawlers are typically slower and less efficient than HTTP-based crawlers, they can handle dynamic, client-side rendered sites that standard HTTP-based crawlers cannot manage. @@ -35,17 +37,17 @@ The <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> uses oth The <ApiLink to="class/BrowserPool">`BrowserPool`</ApiLink> allows you to manage multiple browsers. Each browser instance is managed by a separate <ApiLink to="class/PlaywrightBrowserPlugin">`PlaywrightBrowserPlugin`</ApiLink> and can be configured independently. This is useful for scenarios like testing multiple configurations or implementing browser rotation to help avoid blocks or detect different site behaviors. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {MultipleLaunchExample} -</CodeBlock> +</RunnableCodeBlock> ## Browser launch and context configuration The <ApiLink to="class/PlaywrightBrowserPlugin">`PlaywrightBrowserPlugin`</ApiLink> provides access to all relevant Playwright configuration options for both [browser launches](https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch) and [new browser contexts](https://playwright.dev/python/docs/api/class-browser#browser-new-context). You can specify these options in the constructor of <ApiLink to="class/PlaywrightBrowserPlugin">`PlaywrightBrowserPlugin`</ApiLink> or <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>: -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {BrowserConfigurationExample} -</CodeBlock> +</RunnableCodeBlock> You can also configure each plugin used by <ApiLink to="class/BrowserPool">`BrowserPool`</ApiLink>: @@ -59,9 +61,9 @@ For an example of how to implement a custom browser plugin, see the [Camoufox ex In some use cases, you may need to configure the [page](https://playwright.dev/python/docs/api/class-page) before it navigates to the target URL. For instance, you might set navigation timeouts or manipulate other page-level settings. For such cases you can use the <ApiLink to="class/PlaywrightCrawler#pre_navigation_hook">`pre_navigation_hook`</ApiLink> method of the <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>. This method is called before the page navigates to the target URL and allows you to configure the page instance. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {PreNavigationExample} -</CodeBlock> +</RunnableCodeBlock> ## Conclusion diff --git a/docs/guides/playwright_crawler_adaptive.mdx b/docs/guides/playwright_crawler_adaptive.mdx index 583456e9a0..696bc15163 100644 --- a/docs/guides/playwright_crawler_adaptive.mdx +++ b/docs/guides/playwright_crawler_adaptive.mdx @@ -8,14 +8,14 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import AdaptivePlaywrightCrawlerInitBeautifulSoup from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_beautifulsoup.py'; -import AdaptivePlaywrightCrawlerInitParsel from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_parsel.py'; -import AdaptivePlaywrightCrawlerInitPrediction from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_init_prediction.py'; -import AdaptivePlaywrightCrawlerHandler from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_handler.py'; -import AdaptivePlaywrightCrawlerPreNavHooks from '!!raw-loader!./code_examples/adaptive_playwright_crawler/adaptive_playwright_crawler_pre_nav_hooks.py'; - +import AdaptivePlaywrightCrawlerHandler from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_adaptive/handler.py'; +import AdaptivePlaywrightCrawlerPreNavHooks from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_adaptive/pre_nav_hooks.py'; +import AdaptivePlaywrightCrawlerInitBeautifulSoup from '!!raw-loader!./code_examples/playwright_crawler_adaptive/init_beautifulsoup.py'; +import AdaptivePlaywrightCrawlerInitParsel from '!!raw-loader!./code_examples/playwright_crawler_adaptive/init_parsel.py'; +import AdaptivePlaywrightCrawlerInitPrediction from '!!raw-loader!./code_examples/playwright_crawler_adaptive/init_prediction.py'; An <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink> is a combination of <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and some implementation of HTTP-based crawler such as <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> or <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. It uses a more limited crawling context interface so that it is able to switch to HTTP-only crawling when it detects that it may bring a performance benefit. @@ -42,30 +42,29 @@ Request handler for <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlayw See the following example about how to create request handler and use context helpers: -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {AdaptivePlaywrightCrawlerHandler} -</CodeBlock> - +</RunnableCodeBlock> ## Crawler configuration + To use <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink> it is recommended to use one of the prepared factory methods that will create the crawler with specific HTTP-based sub crawler variant: <ApiLink to="class/AdaptivePlaywrightCrawler#with_beautifulsoup_static_parser">`AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser`</ApiLink> or <ApiLink to="class/AdaptivePlaywrightCrawler#with_parsel_static_parser">`AdaptivePlaywrightCrawler.with_parsel_static_parser`</ApiLink>. <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink> is internally composed of two sub crawlers and you can do a detailed configuration of both of them. For detailed configuration options of the sub crawlers, please refer to their pages: <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>, <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink>, <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. In the following example you can see how to create and configure <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink> with two different HTTP-based sub crawlers: - <Tabs> - <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler" default> -<CodeBlock className="language-python"> - {AdaptivePlaywrightCrawlerInitBeautifulSoup} -</CodeBlock> - </TabItem> -<TabItem value="ParselCrawler" label="ParselCrawler" default> -<CodeBlock className="language-python"> - {AdaptivePlaywrightCrawlerInitParsel} -</CodeBlock> - </TabItem> + <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler" default> + <CodeBlock className="language-python"> + {AdaptivePlaywrightCrawlerInitBeautifulSoup} + </CodeBlock> + </TabItem> + <TabItem value="ParselCrawler" label="ParselCrawler"> + <CodeBlock className="language-python"> + {AdaptivePlaywrightCrawlerInitParsel} + </CodeBlock> + </TabItem> </Tabs> ### Prediction related arguments @@ -84,13 +83,12 @@ See the following example about how to pass prediction related arguments: {AdaptivePlaywrightCrawlerInitPrediction} </CodeBlock> - - ## Page configuration with pre-navigation hooks + In some use cases, you may need to configure the [page](https://playwright.dev/python/docs/api/class-page) before it navigates to the target URL. For instance, you might set navigation timeouts or manipulate other page-level settings. For such cases you can use the <ApiLink to="class/AdaptivePlaywrightCrawler#pre_navigation_hook">`pre_navigation_hook`</ApiLink> method of the <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink>. This method is called before the page navigates to the target URL and allows you to configure the page instance. Due to the dynamic nature of <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink> it is possible that the hook will be executed for HTTP-based sub crawler or playwright-based sub crawler. Using [page](https://playwright.dev/python/docs/api/class-page) object for hook that will be executed on HTTP-based sub crawler will raise an exception. To overcome this you can use optional argument `playwright_only` = `True` when registering the hook. See the following example about how to register the pre navigation hooks: -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {AdaptivePlaywrightCrawlerPreNavHooks} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/guides/proxy_management.mdx b/docs/guides/proxy_management.mdx index 64980f0b97..38385ac950 100644 --- a/docs/guides/proxy_management.mdx +++ b/docs/guides/proxy_management.mdx @@ -8,16 +8,18 @@ import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import QuickStartExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/quick_start_example.py'; +import IntegrationBsExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/integration_bs_example.py'; +import IntegrationPwExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/integration_pw_example.py'; +import TiersBsExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/tiers_bs_example.py'; +import TiersPwExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/tiers_pw_example.py'; +import InspectionBsExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/inspecting_bs_example.py'; +import InspectionPwExample from '!!raw-loader!roa-loader!./code_examples/proxy_management/inspecting_pw_example.py'; -import QuickStartExample from '!!raw-loader!./code_examples/proxy_management/quick_start_example.py'; -import IntegrationBsExample from '!!raw-loader!./code_examples/proxy_management/integration_bs_example.py'; -import IntegrationPwExample from '!!raw-loader!./code_examples/proxy_management/integration_pw_example.py'; import SessionBsExample from '!!raw-loader!./code_examples/proxy_management/session_bs_example.py'; import SessionPwExample from '!!raw-loader!./code_examples/proxy_management/session_pw_example.py'; -import InspectionBsExample from '!!raw-loader!./code_examples/proxy_management/inspecting_bs_example.py'; -import InspectionPwExample from '!!raw-loader!./code_examples/proxy_management/inspecting_pw_example.py'; -import TiersBsExample from '!!raw-loader!./code_examples/proxy_management/tiers_bs_example.py'; -import TiersPwExample from '!!raw-loader!./code_examples/proxy_management/tiers_pw_example.py'; [IP address blocking](https://en.wikipedia.org/wiki/IP_address_blocking) is one of the oldest and most effective ways of preventing access to a website. It is therefore paramount for a good web scraping library to provide easy to use but powerful tools which can work around IP blocking. The most powerful weapon in our anti IP blocking arsenal is a [proxy server](https://en.wikipedia.org/wiki/Proxy_server). @@ -29,9 +31,9 @@ With Crawlee we can use our own proxy servers or proxy servers acquired from thi If you already have proxy URLs of your own, you can start using them immediately in only a few lines of code. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {QuickStartExample} -</CodeBlock> +</RunnableCodeBlock> Examples of how to use our proxy URLs with crawlers are shown below in [Crawler integration](#crawler-integration) section. @@ -45,14 +47,14 @@ All our proxy needs are managed by the <ApiLink to="class/ProxyConfiguration">`P <Tabs> <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {IntegrationBsExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {IntegrationPwExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -89,14 +91,14 @@ In an active tier, Crawlee will alternate between proxies in a round-robin fashi <Tabs> <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {TiersBsExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {TiersPwExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -106,13 +108,13 @@ The <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> an <Tabs> <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {InspectionBsExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {InspectionPwExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> diff --git a/docs/guides/request_loaders.mdx b/docs/guides/request_loaders.mdx index fc17fbadbc..73fe374a62 100644 --- a/docs/guides/request_loaders.mdx +++ b/docs/guides/request_loaders.mdx @@ -7,12 +7,11 @@ description: How to manage the requests your crawler will go through. import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RlBasicExample from '!!raw-loader!./code_examples/request_loaders/rl_basic_example.py'; - -import TandemExample from '!!raw-loader!./code_examples/request_loaders/tandem_example.py'; -import ExplicitTandemExample from '!!raw-loader!./code_examples/request_loaders/tandem_example_explicit.py'; +import RlBasicExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/rl_basic_example.py'; +import TandemExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/tandem_example.py'; +import ExplicitTandemExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/tandem_example_explicit.py'; The [`request_loaders`](https://github.com/apify/crawlee-python/tree/master/src/crawlee/request_loaders) sub-package extends the functionality of the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>, providing additional tools for managing URLs. If you are new to Crawlee, and you do not know the `RequestQueue`, consider starting with the [Storages](https://crawlee.dev/python/docs/guides/storages) guide first. Request loaders define how requests are fetched and stored, enabling various use cases, such as reading URLs from files, external APIs or combining multiple sources together. @@ -109,9 +108,9 @@ The <ApiLink to="class/RequestList">`RequestList`</ApiLink> can accept an asynch Here is a basic example of working with the <ApiLink to="class/RequestList">`RequestList`</ApiLink>: -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {RlBasicExample} -</CodeBlock> +</RunnableCodeBlock> ## Request manager @@ -127,14 +126,14 @@ This sections describes the combination of the <ApiLink to="class/RequestList">` <Tabs groupId="request_manager_tandem"> <TabItem value="request_manager_tandem_explicit" label="Explicitly usage"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {ExplicitTandemExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="request_manager_tandem_helper" label="Using to_tandem helper" default> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {TandemExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> diff --git a/docs/guides/scaling_crawlers.mdx b/docs/guides/scaling_crawlers.mdx index cbe12ad56e..5dce8ac640 100644 --- a/docs/guides/scaling_crawlers.mdx +++ b/docs/guides/scaling_crawlers.mdx @@ -5,10 +5,10 @@ description: Learn how to scale your crawlers by controlling concurrency and lim --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import MaxTasksPerMinuteExample from '!!raw-loader!./code_examples/scaling_crawlers/max_tasks_per_minute_example.py'; -import MinAndMaxConcurrencyExample from '!!raw-loader!./code_examples/scaling_crawlers/min_and_max_concurrency_example.py'; +import MaxTasksPerMinuteExample from '!!raw-loader!roa-loader!./code_examples/scaling_crawlers/max_tasks_per_minute_example.py'; +import MinAndMaxConcurrencyExample from '!!raw-loader!roa-loader!./code_examples/scaling_crawlers/min_and_max_concurrency_example.py'; As we build our crawler, we may want to control how many tasks it performs at any given time. In other words, how many requests it makes to the web we are trying to scrape. Crawlee offers several options to fine-tune the number of parallel tasks, limit the number of requests per minute, and optimize scaling based on available system resources. @@ -22,9 +22,9 @@ All of these options are available across all crawlers provided by Crawlee. In t The `max_tasks_per_minute` setting in <ApiLink to="class/ConcurrencySettings">`ConcurrencySettings`</ApiLink> controls how many total tasks the crawler can process per minute. It ensures that tasks are spread evenly throughout the minute, preventing a sudden burst at the `max_concurrency` limit followed by idle time. By default, this is set to `Infinity`, meaning the crawler can run at full speed, limited only by `max_concurrency`. Use this option if you want to throttle your crawler to avoid overwhelming the target website with continuous requests. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {MaxTasksPerMinuteExample} -</CodeBlock> +</RunnableCodeBlock> ## Minimum and maximum concurrency @@ -40,9 +40,9 @@ If you set `min_concurrency` too high compared to the available system resources The `desired_concurrency` option in the <ApiLink to="class/ConcurrencySettings">`ConcurrencySettings`</ApiLink> specifies the initial number of parallel tasks to start with, assuming sufficient resources are available. It defaults to the same value as `min_concurrency`. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {MinAndMaxConcurrencyExample} -</CodeBlock> +</RunnableCodeBlock> ## Autoscaled pool diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdx index 20b4dfd1b2..a3a1385db1 100644 --- a/docs/guides/session_management.mdx +++ b/docs/guides/session_management.mdx @@ -7,16 +7,16 @@ description: How to manage your cookies, proxy IP rotations and more. import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BasicSource from '!!raw-loader!./code_examples/session_management/session_management_basic.py'; -import HttpSource from '!!raw-loader!./code_examples/session_management/session_management_http.py'; -import BeautifulSoupSource from '!!raw-loader!./code_examples/session_management/session_management_beautifulsoup.py'; -import ParselSource from '!!raw-loader!./code_examples/session_management/session_management_parsel.py'; -import PlaywrightSource from '!!raw-loader!./code_examples/session_management/session_management_playwright.py'; -import StandaloneSource from '!!raw-loader!./code_examples/session_management/session_management_standalone.py'; -import OneSession from '!!raw-loader!./code_examples/session_management/one_session_http.py'; -import MultiSessions from '!!raw-loader!./code_examples/session_management/multi_sessions_http.py'; +import BasicSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_basic.py'; +import HttpSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_http.py'; +import BeautifulSoupSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_beautifulsoup.py'; +import ParselSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_parsel.py'; +import PlaywrightSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_playwright.py'; +import StandaloneSource from '!!raw-loader!roa-loader!./code_examples/session_management/sm_standalone.py'; +import OneSession from '!!raw-loader!roa-loader!./code_examples/session_management/one_session_http.py'; +import MultiSessions from '!!raw-loader!roa-loader!./code_examples/session_management/multi_sessions_http.py'; The <ApiLink to="class/SessionPool">`SessionPool`</ApiLink> class provides a robust way to manage the rotation of proxy IP addresses, cookies, and other custom settings in Crawlee. Its primary advantage is the ability to filter out blocked or non-functional proxies, ensuring that your scraper avoids retrying requests through known problematic proxies. @@ -36,34 +36,34 @@ Now, let's explore examples of how to use the <ApiLink to="class/SessionPool">`S <Tabs groupId="session_pool"> <TabItem value="basic" label="BasicSource"> - <CodeBlock language="py"> + <RunnableCodeBlock className="language-python" language="python"> {BasicSource} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="http" label="HttpCrawler"> - <CodeBlock language="py"> + <RunnableCodeBlock className="language-python" language="python"> {HttpSource} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="beautifulsoup" label="BeautifulSoupCrawler"> - <CodeBlock language="py"> + <RunnableCodeBlock className="language-python" language="python"> {BeautifulSoupSource} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="parsel" label="ParselCrawler"> - <CodeBlock language="py"> + <RunnableCodeBlock className="language-python" language="python"> {ParselSource} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="playwright" label="PlaywrightCrawler"> - <CodeBlock language="py"> + <RunnableCodeBlock className="language-python" language="python"> {PlaywrightSource} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="standalone" label="Standalone"> - <CodeBlock language="py"> + <RunnableCodeBlock className="language-python" language="python"> {StandaloneSource} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -77,9 +77,9 @@ In some cases, you need full control over session usage. For example, when worki When working with a site that requires authentication, we typically don't want multiple sessions with different browser fingerprints or client parameters accessing the site. In this case, we need to configure the <ApiLink to="class/SessionPool">`SessionPool`</ApiLink> appropriately: -<CodeBlock language="py"> +<RunnableCodeBlock className="language-python" language="python"> {OneSession} -</CodeBlock> +</RunnableCodeBlock> ## Binding requests to specific sessions @@ -89,6 +89,6 @@ In some cases, we need to achieve the same behavior but using multiple sessions To do this, use the `session_id` parameter for the <ApiLink to="class/Request">`Request`</ApiLink> object to bind a request to a specific session: -<CodeBlock language="py"> +<RunnableCodeBlock className="language-python" language="python"> {MultiSessions} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/guides/storages.mdx b/docs/guides/storages.mdx index 4615a2ee83..3be168b683 100644 --- a/docs/guides/storages.mdx +++ b/docs/guides/storages.mdx @@ -7,24 +7,24 @@ description: How to work with storages in Crawlee, how to manage requests and ho import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RqBasicExample from '!!raw-loader!./code_examples/storages/rq_basic_example.py'; -import RqWithCrawlerExample from '!!raw-loader!./code_examples/storages/rq_with_crawler_example.py'; -import RqWithCrawlerExplicitExample from '!!raw-loader!./code_examples/storages/rq_with_crawler_explicit_example.py'; -import RqHelperAddRequestsExample from '!!raw-loader!./code_examples/storages/helper_add_requests_example.py'; -import RqHelperEnqueueLinksExample from '!!raw-loader!./code_examples/storages/helper_enqueue_links_example.py'; +import RqBasicExample from '!!raw-loader!roa-loader!./code_examples/storages/rq_basic_example.py'; +import RqWithCrawlerExample from '!!raw-loader!roa-loader!./code_examples/storages/rq_with_crawler_example.py'; +import RqWithCrawlerExplicitExample from '!!raw-loader!roa-loader!./code_examples/storages/rq_with_crawler_explicit_example.py'; +import RqHelperAddRequestsExample from '!!raw-loader!roa-loader!./code_examples/storages/helper_add_requests_example.py'; +import RqHelperEnqueueLinksExample from '!!raw-loader!roa-loader!./code_examples/storages/helper_enqueue_links_example.py'; -import DatasetBasicExample from '!!raw-loader!./code_examples/storages/dataset_basic_example.py'; -import DatasetWithCrawlerExample from '!!raw-loader!./code_examples/storages/dataset_with_crawler_example.py'; -import DatasetWithCrawerExplicitExample from '!!raw-loader!./code_examples/storages/dataset_with_crawler_explicit_example.py'; +import DatasetBasicExample from '!!raw-loader!roa-loader!./code_examples/storages/dataset_basic_example.py'; +import DatasetWithCrawlerExample from '!!raw-loader!roa-loader!./code_examples/storages/dataset_with_crawler_example.py'; +import DatasetWithCrawerExplicitExample from '!!raw-loader!roa-loader!./code_examples/storages/dataset_with_crawler_explicit_example.py'; -import KvsBasicExample from '!!raw-loader!./code_examples/storages/kvs_basic_example.py'; -import KvsWithCrawlerExample from '!!raw-loader!./code_examples/storages/kvs_with_crawler_example.py'; -import KvsWithCrawlerExplicitExample from '!!raw-loader!./code_examples/storages/kvs_with_crawler_explicit_example.py'; +import KvsBasicExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_basic_example.py'; +import KvsWithCrawlerExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_with_crawler_example.py'; +import KvsWithCrawlerExplicitExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_with_crawler_explicit_example.py'; -import CleaningDoNotPurgeExample from '!!raw-loader!./code_examples/storages/cleaning_do_not_purge_example.py'; -import CleaningPurgeExplicitlyExample from '!!raw-loader!./code_examples/storages/cleaning_purge_explicitly_example.py'; +import CleaningDoNotPurgeExample from '!!raw-loader!roa-loader!./code_examples/storages/cleaning_do_not_purge_example.py'; +import CleaningPurgeExplicitlyExample from '!!raw-loader!roa-loader!./code_examples/storages/cleaning_purge_explicitly_example.py'; Crawlee offers multiple storage types for managing and persisting your crawling data. Request-oriented storages, such as the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>, help you store and deduplicate URLs, while result-oriented storages, like <ApiLink to="class/Dataset">`Dataset`</ApiLink> and <ApiLink to="class/KeyValueStore">`KeyValueStore`</ApiLink>, focus on storing and retrieving scraping results. This guide helps you choose the storage type that suits your needs. @@ -75,19 +75,19 @@ The following code demonstrates the usage of the <ApiLink to="class/RequestQueue <Tabs groupId="request_queue"> <TabItem value="request_queue_basic_example" label="Basic usage" default> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {RqBasicExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="request_queue_with_crawler" label="Usage with Crawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {RqWithCrawlerExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="request_queue_with_crawler_explicit" label="Explicit usage with Crawler" default> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {RqWithCrawlerExplicitExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -100,14 +100,14 @@ Crawlee provides helper functions to simplify interactions with the <ApiLink to= <Tabs groupId="request_helpers"> <TabItem value="request_helper_add_requests" label="Add requests" default> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {RqHelperAddRequestsExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="request_helper_enqueue_links" label="Enqueue links"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {RqHelperEnqueueLinksExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -136,19 +136,19 @@ The following code demonstrates basic operations of the dataset: <Tabs groupId="dataset_storage"> <TabItem value="dataset_basic_example" label="Basic usage" default> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {DatasetBasicExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="dataset_with_crawler" label="Usage with Crawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {DatasetWithCrawlerExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="dataset_with_crawler_explicit" label="Explicit usage with Crawler" default> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {DatasetWithCrawerExplicitExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -176,19 +176,19 @@ The following code demonstrates the usage of the <ApiLink to="class/KeyValueStor <Tabs groupId="kv_storage"> <TabItem value="kvs_basic_example" label="Basic usage" default> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {KvsBasicExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="kvs_with_crawler" label="Usage with Crawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {KvsWithCrawlerExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="kvs_with_crawler_explicit" label="Explicit usage with Crawler" default> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {KvsWithCrawlerExplicitExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -204,17 +204,17 @@ Crawlee provides the following helper function to simplify interactions with the Default storages are purged before the crawler starts, unless explicitly configured otherwise. For that case, see <ApiLink to="class/Configuration#purge_on_start">`Configuration.purge_on_start`</ApiLink>. This cleanup happens as soon as a storage is accessed, either when you open a storage (e.g. using <ApiLink to="class/RequestQueue#open">`RequestQueue.open`</ApiLink>, <ApiLink to="class/Dataset#open">`Dataset.open`</ApiLink>, <ApiLink to="class/KeyValueStore#open">`KeyValueStore.open`</ApiLink>) or when interacting with a storage through one of the helper functions (e.g. <ApiLink to="class/PushDataFunction">`push_data`</ApiLink>), which implicitly opens the result storage. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {CleaningDoNotPurgeExample} -</CodeBlock> +</RunnableCodeBlock> If you do not explicitly interact with storages in your code, the purging will occur automatically when the <ApiLink to="class/BasicCrawler#run">`BasicCrawler.run`</ApiLink> method is invoked. If you need to purge storages earlier, you can call <ApiLink to="class/MemoryStorageClient#purge_on_start">`MemoryStorageClient.purge_on_start`</ApiLink> directly if you are using the default storage client. This method triggers the purging process for the underlying storage implementation you are currently using. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {CleaningPurgeExplicitlyExample} -</CodeBlock> +</RunnableCodeBlock> ## Conclusion From 6214b414fd26c5230214392451af05ef2a8dca1e Mon Sep 17 00:00:00 2001 From: Vlada Dusek <v.dusek96@gmail.com> Date: Wed, 2 Apr 2025 18:01:52 +0200 Subject: [PATCH 10/16] line break for import beautiful soup examples --- .../code_examples/apify/crawler_as_actor_example.py | 5 ++++- docs/examples/code_examples/add_data_to_dataset_bs.py | 5 ++++- docs/examples/code_examples/beautifulsoup_crawler_stop.py | 5 ++++- docs/examples/code_examples/crawl_all_links_on_website_bs.py | 5 ++++- docs/examples/code_examples/crawl_multiple_urls_bs.py | 5 ++++- .../code_examples/crawl_specific_links_on_website_bs.py | 5 ++++- .../crawl_website_with_relative_links_all_links.py | 5 ++++- .../crawl_website_with_relative_links_same_domain.py | 5 ++++- .../crawl_website_with_relative_links_same_hostname.py | 5 ++++- .../crawl_website_with_relative_links_same_origin.py | 5 ++++- .../code_examples/export_entire_dataset_to_file_csv.py | 5 ++++- .../code_examples/export_entire_dataset_to_file_json.py | 5 ++++- .../code_examples/http_clients/curl_impersonate_example.py | 5 ++++- docs/guides/code_examples/http_clients/httpx_example.py | 5 ++++- .../code_examples/proxy_management/inspecting_bs_example.py | 5 ++++- .../code_examples/proxy_management/integration_bs_example.py | 5 ++++- .../code_examples/proxy_management/tiers_bs_example.py | 5 ++++- .../code_examples/session_management/sm_beautifulsoup.py | 5 ++++- .../code_examples/storages/dataset_with_crawler_example.py | 5 ++++- .../storages/dataset_with_crawler_explicit_example.py | 5 ++++- .../code_examples/storages/helper_add_requests_example.py | 5 ++++- .../code_examples/storages/helper_enqueue_links_example.py | 5 ++++- docs/introduction/code_examples/02_bs.py | 5 ++++- docs/introduction/code_examples/02_bs_better.py | 5 ++++- docs/introduction/code_examples/03_enqueue_strategy.py | 5 ++++- docs/introduction/code_examples/03_finding_new_links.py | 5 ++++- docs/introduction/code_examples/03_globs.py | 5 ++++- docs/introduction/code_examples/03_original_code.py | 5 ++++- docs/introduction/code_examples/03_transform_request.py | 5 ++++- docs/introduction/code_examples/04_sanity_check.py | 3 ++- .../code_examples/beautifulsoup_crawler_example.py | 5 ++++- 31 files changed, 122 insertions(+), 31 deletions(-) diff --git a/docs/deployment/code_examples/apify/crawler_as_actor_example.py b/docs/deployment/code_examples/apify/crawler_as_actor_example.py index 8637340557..b986bf9129 100644 --- a/docs/deployment/code_examples/apify/crawler_as_actor_example.py +++ b/docs/deployment/code_examples/apify/crawler_as_actor_example.py @@ -1,6 +1,9 @@ from apify import Actor -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/add_data_to_dataset_bs.py b/docs/examples/code_examples/add_data_to_dataset_bs.py index 5044a35bb0..4e8b41d046 100644 --- a/docs/examples/code_examples/add_data_to_dataset_bs.py +++ b/docs/examples/code_examples/add_data_to_dataset_bs.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/beautifulsoup_crawler_stop.py b/docs/examples/code_examples/beautifulsoup_crawler_stop.py index 21904533a7..8a54109dbf 100644 --- a/docs/examples/code_examples/beautifulsoup_crawler_stop.py +++ b/docs/examples/code_examples/beautifulsoup_crawler_stop.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/crawl_all_links_on_website_bs.py b/docs/examples/code_examples/crawl_all_links_on_website_bs.py index 1e1a219718..a289ab5409 100644 --- a/docs/examples/code_examples/crawl_all_links_on_website_bs.py +++ b/docs/examples/code_examples/crawl_all_links_on_website_bs.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/crawl_multiple_urls_bs.py b/docs/examples/code_examples/crawl_multiple_urls_bs.py index e80db58f81..2d6de8e57b 100644 --- a/docs/examples/code_examples/crawl_multiple_urls_bs.py +++ b/docs/examples/code_examples/crawl_multiple_urls_bs.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py b/docs/examples/code_examples/crawl_specific_links_on_website_bs.py index 65feb175aa..dc19b394c2 100644 --- a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py +++ b/docs/examples/code_examples/crawl_specific_links_on_website_bs.py @@ -1,5 +1,8 @@ from crawlee import Glob -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py b/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py index 439e70e1a2..f6eefd6a8d 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py index 6a2ee32b71..335f2db6f4 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py index f85061da4a..a5093e0e83 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py index 6b4fd29699..ca2f93e472 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/export_entire_dataset_to_file_csv.py b/docs/examples/code_examples/export_entire_dataset_to_file_csv.py index 45f3b9c666..bd0dfc2bdf 100644 --- a/docs/examples/code_examples/export_entire_dataset_to_file_csv.py +++ b/docs/examples/code_examples/export_entire_dataset_to_file_csv.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/examples/code_examples/export_entire_dataset_to_file_json.py b/docs/examples/code_examples/export_entire_dataset_to_file_json.py index 4f116a5df9..d6e6eb2f7b 100644 --- a/docs/examples/code_examples/export_entire_dataset_to_file_json.py +++ b/docs/examples/code_examples/export_entire_dataset_to_file_json.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/guides/code_examples/http_clients/curl_impersonate_example.py b/docs/guides/code_examples/http_clients/curl_impersonate_example.py index 8f3cb49b19..5785f3576e 100644 --- a/docs/guides/code_examples/http_clients/curl_impersonate_example.py +++ b/docs/guides/code_examples/http_clients/curl_impersonate_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) from crawlee.http_clients import CurlImpersonateHttpClient diff --git a/docs/guides/code_examples/http_clients/httpx_example.py b/docs/guides/code_examples/http_clients/httpx_example.py index a113445fba..826b7522c5 100644 --- a/docs/guides/code_examples/http_clients/httpx_example.py +++ b/docs/guides/code_examples/http_clients/httpx_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) from crawlee.http_clients import HttpxHttpClient diff --git a/docs/guides/code_examples/proxy_management/inspecting_bs_example.py b/docs/guides/code_examples/proxy_management/inspecting_bs_example.py index a25af60f5b..768b59ff3b 100644 --- a/docs/guides/code_examples/proxy_management/inspecting_bs_example.py +++ b/docs/guides/code_examples/proxy_management/inspecting_bs_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) from crawlee.proxy_configuration import ProxyConfiguration diff --git a/docs/guides/code_examples/proxy_management/integration_bs_example.py b/docs/guides/code_examples/proxy_management/integration_bs_example.py index c47f0a76a9..0f4d12f354 100644 --- a/docs/guides/code_examples/proxy_management/integration_bs_example.py +++ b/docs/guides/code_examples/proxy_management/integration_bs_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) from crawlee.proxy_configuration import ProxyConfiguration diff --git a/docs/guides/code_examples/proxy_management/tiers_bs_example.py b/docs/guides/code_examples/proxy_management/tiers_bs_example.py index 66183cd8d6..017549b163 100644 --- a/docs/guides/code_examples/proxy_management/tiers_bs_example.py +++ b/docs/guides/code_examples/proxy_management/tiers_bs_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) from crawlee.proxy_configuration import ProxyConfiguration diff --git a/docs/guides/code_examples/session_management/sm_beautifulsoup.py b/docs/guides/code_examples/session_management/sm_beautifulsoup.py index 6077502a4b..da8781c56b 100644 --- a/docs/guides/code_examples/session_management/sm_beautifulsoup.py +++ b/docs/guides/code_examples/session_management/sm_beautifulsoup.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) from crawlee.proxy_configuration import ProxyConfiguration from crawlee.sessions import SessionPool diff --git a/docs/guides/code_examples/storages/dataset_with_crawler_example.py b/docs/guides/code_examples/storages/dataset_with_crawler_example.py index 41b8258260..7813bb6e77 100644 --- a/docs/guides/code_examples/storages/dataset_with_crawler_example.py +++ b/docs/guides/code_examples/storages/dataset_with_crawler_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py b/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py index a88282e75e..9771f9bc8a 100644 --- a/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py +++ b/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) from crawlee.storages import Dataset diff --git a/docs/guides/code_examples/storages/helper_add_requests_example.py b/docs/guides/code_examples/storages/helper_add_requests_example.py index db344310c5..d524278f7f 100644 --- a/docs/guides/code_examples/storages/helper_add_requests_example.py +++ b/docs/guides/code_examples/storages/helper_add_requests_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/guides/code_examples/storages/helper_enqueue_links_example.py b/docs/guides/code_examples/storages/helper_enqueue_links_example.py index 72862cbf15..7745755adf 100644 --- a/docs/guides/code_examples/storages/helper_enqueue_links_example.py +++ b/docs/guides/code_examples/storages/helper_enqueue_links_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/introduction/code_examples/02_bs.py b/docs/introduction/code_examples/02_bs.py index 7da7389f3c..5d580141ba 100644 --- a/docs/introduction/code_examples/02_bs.py +++ b/docs/introduction/code_examples/02_bs.py @@ -1,5 +1,8 @@ # Add import of crawler and crawling context. -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) from crawlee.storages import RequestQueue diff --git a/docs/introduction/code_examples/02_bs_better.py b/docs/introduction/code_examples/02_bs_better.py index dddfb79181..c0d897ce5c 100644 --- a/docs/introduction/code_examples/02_bs_better.py +++ b/docs/introduction/code_examples/02_bs_better.py @@ -1,5 +1,8 @@ # You don't need to import RequestQueue anymore. -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/introduction/code_examples/03_enqueue_strategy.py b/docs/introduction/code_examples/03_enqueue_strategy.py index 322769adb3..6a7ecd9eaf 100644 --- a/docs/introduction/code_examples/03_enqueue_strategy.py +++ b/docs/introduction/code_examples/03_enqueue_strategy.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/introduction/code_examples/03_finding_new_links.py b/docs/introduction/code_examples/03_finding_new_links.py index 67a935285c..8925fab220 100644 --- a/docs/introduction/code_examples/03_finding_new_links.py +++ b/docs/introduction/code_examples/03_finding_new_links.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/introduction/code_examples/03_globs.py b/docs/introduction/code_examples/03_globs.py index c605479472..caec75346b 100644 --- a/docs/introduction/code_examples/03_globs.py +++ b/docs/introduction/code_examples/03_globs.py @@ -1,5 +1,8 @@ from crawlee import Glob -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/introduction/code_examples/03_original_code.py b/docs/introduction/code_examples/03_original_code.py index bf19fd2ad8..5b9fd7a071 100644 --- a/docs/introduction/code_examples/03_original_code.py +++ b/docs/introduction/code_examples/03_original_code.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: diff --git a/docs/introduction/code_examples/03_transform_request.py b/docs/introduction/code_examples/03_transform_request.py index fd2be7a20a..6f528a01c2 100644 --- a/docs/introduction/code_examples/03_transform_request.py +++ b/docs/introduction/code_examples/03_transform_request.py @@ -1,7 +1,10 @@ from __future__ import annotations from crawlee import HttpHeaders, RequestOptions, RequestTransformAction -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) def transform_request( diff --git a/docs/introduction/code_examples/04_sanity_check.py b/docs/introduction/code_examples/04_sanity_check.py index b83a021b6e..1da52fd425 100644 --- a/docs/introduction/code_examples/04_sanity_check.py +++ b/docs/introduction/code_examples/04_sanity_check.py @@ -1,4 +1,5 @@ -# Instead of BeautifulSoupCrawler let's use Playwright to be able to render JavaScript. +# Instead of BeautifulSoupCrawler let's use Playwright to be able +# to render JavaScript. from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext diff --git a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py index c69aeea8b7..b2625c59e7 100644 --- a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py +++ b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py @@ -1,4 +1,7 @@ -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) async def main() -> None: From 38854a547b3755a337eaca96274c9921b616c04f Mon Sep 17 00:00:00 2001 From: Vlada Dusek <v.dusek96@gmail.com> Date: Wed, 2 Apr 2025 18:10:39 +0200 Subject: [PATCH 11/16] examples --- docs/examples/add_data_to_dataset.mdx | 20 +++++++------- docs/examples/beautifulsoup_crawler.mdx | 8 +++--- .../capture_screenshot_using_playwright.mdx | 8 +++--- docs/examples/code_examples/parsel_crawler.py | 6 ++++- .../playwright_crawler_with_camoufox.py | 3 ++- docs/examples/crawl_all_links_on_website.mdx | 14 +++++----- docs/examples/crawl_multiple_urls.mdx | 14 +++++----- .../crawl_specific_links_on_website.mdx | 14 +++++----- .../crawl_website_with_relative_links.mdx | 26 +++++++++---------- docs/examples/crawler_keep_alive.mdx | 8 +++--- docs/examples/crawler_stop.mdx | 8 +++--- .../export_entire_dataset_to_file.mdx | 14 +++++----- docs/examples/fill_and_submit_web_form.mdx | 14 +++++----- docs/examples/json_logging.mdx | 8 +++--- docs/examples/parsel_crawler.mdx | 8 +++--- docs/examples/playwright_crawler.mdx | 8 +++--- docs/examples/playwright_crawler_adaptive.mdx | 8 +++--- ...playwright_crawler_with_block_requests.mdx | 8 +++--- .../playwright_crawler_with_camoufox.mdx | 8 +++--- ...ght_crawler_with_fingerprint_generator.mdx | 8 +++--- 20 files changed, 109 insertions(+), 104 deletions(-) diff --git a/docs/examples/add_data_to_dataset.mdx b/docs/examples/add_data_to_dataset.mdx index c5e3f1472d..aa4164cacf 100644 --- a/docs/examples/add_data_to_dataset.mdx +++ b/docs/examples/add_data_to_dataset.mdx @@ -6,24 +6,24 @@ title: Add data to dataset import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/add_data_to_dataset_bs.py'; -import PlaywrightExample from '!!raw-loader!./code_examples/add_data_to_dataset_pw.py'; -import DatasetExample from '!!raw-loader!./code_examples/add_data_to_dataset_dataset.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_bs.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_pw.py'; +import DatasetExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_dataset.py'; This example demonstrates how to store extracted data into datasets using the <ApiLink to="class/PushDataFunction#open">`context.push_data`</ApiLink> helper function. If the specified dataset does not already exist, it will be created automatically. Additionally, you can save data to custom datasets by providing `dataset_id` or `dataset_name` parameters to the <ApiLink to="class/PushDataFunction#open">`push_data`</ApiLink> function. <Tabs groupId="main"> <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {BeautifulSoupExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {PlaywrightExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> @@ -35,6 +35,6 @@ Each item in the dataset will be stored in its own file within the following dir For more control, you can also open a dataset manually using the asynchronous constructor <ApiLink to="class/Dataset#open">`Dataset.open`</ApiLink> -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {DatasetExample} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/beautifulsoup_crawler.mdx b/docs/examples/beautifulsoup_crawler.mdx index 7ab92083ac..160e4c4d65 100644 --- a/docs/examples/beautifulsoup_crawler.mdx +++ b/docs/examples/beautifulsoup_crawler.mdx @@ -4,12 +4,12 @@ title: BeautifulSoup crawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/beautifulsoup_crawler.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler.py'; This example demonstrates how to use <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> to crawl a list of URLs, load each URL using a plain HTTP request, parse the HTML using the [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) library and extract some data from it - the page title and all `<h1>`, `<h2>` and `<h3>` tags. This setup is perfect for scraping specific elements from web pages. Thanks to the well-known BeautifulSoup, you can easily navigate the HTML structure and retrieve the data you need with minimal code. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {BeautifulSoupExample} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/capture_screenshot_using_playwright.mdx b/docs/examples/capture_screenshot_using_playwright.mdx index ffbaffcd93..614693b1e8 100644 --- a/docs/examples/capture_screenshot_using_playwright.mdx +++ b/docs/examples/capture_screenshot_using_playwright.mdx @@ -4,9 +4,9 @@ title: Capture screenshots using Playwright --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import CaptureScreenshotExample from '!!raw-loader!./code_examples/capture_screenshot_using_playwright.py'; +import CaptureScreenshotExample from '!!raw-loader!roa-loader!./code_examples/capture_screenshot_using_playwright.py'; This example demonstrates how to capture screenshots of web pages using <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and store them in the key-value store. @@ -14,6 +14,6 @@ The <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> is confi The captured screenshots are stored in the key-value store, which is suitable for managing and storing files in various formats. In this case, screenshots are stored as PNG images with a unique key generated from the URL of the page. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {CaptureScreenshotExample} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/code_examples/parsel_crawler.py b/docs/examples/code_examples/parsel_crawler.py index 52b6e98c86..9ba403d832 100644 --- a/docs/examples/code_examples/parsel_crawler.py +++ b/docs/examples/code_examples/parsel_crawler.py @@ -1,4 +1,8 @@ -from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext +from crawlee.crawlers import ( + BasicCrawlingContext, + ParselCrawler, + ParselCrawlingContext, +) # Regex for identifying email addresses on a webpage. EMAIL_REGEX = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' diff --git a/docs/examples/code_examples/playwright_crawler_with_camoufox.py b/docs/examples/code_examples/playwright_crawler_with_camoufox.py index 1a03578540..a09dd5f8df 100644 --- a/docs/examples/code_examples/playwright_crawler_with_camoufox.py +++ b/docs/examples/code_examples/playwright_crawler_with_camoufox.py @@ -1,4 +1,5 @@ -# Camoufox is external package and needs to be installed. It is not included in crawlee. +# Camoufox is external package and needs to be installed. +# It is not included in Crawlee. from camoufox import AsyncNewBrowser from typing_extensions import override diff --git a/docs/examples/crawl_all_links_on_website.mdx b/docs/examples/crawl_all_links_on_website.mdx index f50ea953a9..f17c63920f 100644 --- a/docs/examples/crawl_all_links_on_website.mdx +++ b/docs/examples/crawl_all_links_on_website.mdx @@ -6,10 +6,10 @@ title: Crawl all links on website import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_all_links_on_website_bs.py'; -import PlaywrightExample from '!!raw-loader!./code_examples/crawl_all_links_on_website_pw.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_bs.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_pw.py'; This example uses the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> helper to add new links to the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink> as the crawler navigates from page to page. By automatically discovering and enqueuing all links on a given page, the crawler can systematically scrape an entire website. This approach is ideal for web scraping tasks where you need to collect data from multiple interconnected pages. @@ -21,13 +21,13 @@ If no options are given, by default the method will only add links that are unde <Tabs groupId="main"> <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {BeautifulSoupExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {PlaywrightExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> diff --git a/docs/examples/crawl_multiple_urls.mdx b/docs/examples/crawl_multiple_urls.mdx index b5c1c7f8f7..2d3d370283 100644 --- a/docs/examples/crawl_multiple_urls.mdx +++ b/docs/examples/crawl_multiple_urls.mdx @@ -6,22 +6,22 @@ title: Crawl multiple URLs import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_multiple_urls_bs.py'; -import PlaywrightExample from '!!raw-loader!./code_examples/crawl_multiple_urls_pw.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_bs.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_pw.py'; This example demonstrates how to crawl a specified list of URLs using different crawlers. You'll learn how to set up the crawler, define a request handler, and run the crawler with multiple URLs. This setup is useful for scraping data from multiple pages or websites concurrently. <Tabs groupId="main"> <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {BeautifulSoupExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {PlaywrightExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> diff --git a/docs/examples/crawl_specific_links_on_website.mdx b/docs/examples/crawl_specific_links_on_website.mdx index b2e206b811..c1dcd8446c 100644 --- a/docs/examples/crawl_specific_links_on_website.mdx +++ b/docs/examples/crawl_specific_links_on_website.mdx @@ -6,22 +6,22 @@ title: Crawl specific links on website import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_specific_links_on_website_bs.py'; -import PlaywrightExample from '!!raw-loader!./code_examples/crawl_specific_links_on_website_pw.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_bs.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_pw.py'; This example demonstrates how to crawl a website while targeting specific patterns of links. By utilizing the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> helper, you can pass `include` or `exclude` parameters to improve your crawling strategy. This approach ensures that only the links matching the specified patterns are added to the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>. Both `include` and `exclude` support lists of globs or regular expressions. This functionality is great for focusing on relevant sections of a website and avoiding scraping unnecessary or irrelevant content. <Tabs groupId="main"> <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {BeautifulSoupExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {PlaywrightExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> diff --git a/docs/examples/crawl_website_with_relative_links.mdx b/docs/examples/crawl_website_with_relative_links.mdx index 96d1301924..4cf7bee845 100644 --- a/docs/examples/crawl_website_with_relative_links.mdx +++ b/docs/examples/crawl_website_with_relative_links.mdx @@ -6,12 +6,12 @@ title: Crawl website with relative links import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import AllLinksExample from '!!raw-loader!./code_examples/crawl_website_with_relative_links_all_links.py'; -import SameDomainExample from '!!raw-loader!./code_examples/crawl_website_with_relative_links_same_domain.py'; -import SameHostnameExample from '!!raw-loader!./code_examples/crawl_website_with_relative_links_same_hostname.py'; -import SameOriginExample from '!!raw-loader!./code_examples/crawl_website_with_relative_links_same_origin.py'; +import AllLinksExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_all_links.py'; +import SameDomainExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_domain.py'; +import SameHostnameExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_hostname.py'; +import SameOriginExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_origin.py'; When crawling a website, you may encounter various types of links that you wish to include in your crawl. To facilitate this, we provide the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> method on the crawler context, which will automatically find and add these links to the crawler's <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>. This method simplifies the process of handling different types of links, including relative links, by automatically resolving them based on the page's context. @@ -30,23 +30,23 @@ For these examples, we are using the <ApiLink to="class/BeautifulSoupCrawler">`B <Tabs groupId="main"> <TabItem value="all_links" label="All links"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {AllLinksExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="same-domain" label="Same domain"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {SameDomainExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="same-hostname" label="Same hostname"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {SameHostnameExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="same-origin" label="Same origin"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {SameOriginExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> diff --git a/docs/examples/crawler_keep_alive.mdx b/docs/examples/crawler_keep_alive.mdx index 60127d9849..2e6c6640c7 100644 --- a/docs/examples/crawler_keep_alive.mdx +++ b/docs/examples/crawler_keep_alive.mdx @@ -4,12 +4,12 @@ title: Keep a Crawler alive waiting for more requests --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/beautifulsoup_crawler_keep_alive.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_keep_alive.py'; This example demonstrates how to keep crawler alive even when there are no requests at the moment by using `keep_alive=True` argument of <ApiLink to="class/BasicCrawler#__init__">`BasicCrawler.__init__`</ApiLink>. This is available to all crawlers that inherit from <ApiLink to="class/BasicCrawler">`BasicCrawler`</ApiLink> and in the example below it is shown on <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. To stop the crawler that was started with `keep_alive=True` you can call `crawler.stop()`. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {BeautifulSoupExample} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/crawler_stop.mdx b/docs/examples/crawler_stop.mdx index 9a0f635ab9..4ea7f28565 100644 --- a/docs/examples/crawler_stop.mdx +++ b/docs/examples/crawler_stop.mdx @@ -4,12 +4,12 @@ title: Stopping a Crawler with stop method --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulSoupExample from '!!raw-loader!./code_examples/beautifulsoup_crawler_stop.py'; +import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_stop.py'; This example demonstrates how to use `stop` method of <ApiLink to="class/BasicCrawler">`BasicCrawler`</ApiLink> to stop crawler once the crawler finds what it is looking for. This method is available to all crawlers that inherit from <ApiLink to="class/BasicCrawler">`BasicCrawler`</ApiLink> and in the example below it is shown on <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. Simply call `crawler.stop()` to stop the crawler. It will not continue to crawl through new requests. Requests that are already being concurrently processed are going to get finished. It is possible to call `stop` method with optional argument `reason` that is a string that will be used in logs and it can improve logs readability especially if you have multiple different conditions for triggering `stop`. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {BeautifulSoupExample} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/export_entire_dataset_to_file.mdx b/docs/examples/export_entire_dataset_to_file.mdx index 5c673e90ff..72418ebe66 100644 --- a/docs/examples/export_entire_dataset_to_file.mdx +++ b/docs/examples/export_entire_dataset_to_file.mdx @@ -6,10 +6,10 @@ title: Export entire dataset to file import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import JsonExample from '!!raw-loader!./code_examples/export_entire_dataset_to_file_json.py'; -import CsvExample from '!!raw-loader!./code_examples/export_entire_dataset_to_file_csv.py'; +import JsonExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_json.py'; +import CsvExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_csv.py'; This example demonstrates how to use the <ApiLink to="class/BasicCrawler#export_data">`BasicCrawler.export_data`</ApiLink> method of the crawler to export the entire default dataset to a single file. This method supports exporting data in either CSV or JSON format. @@ -21,13 +21,13 @@ For these examples, we are using the <ApiLink to="class/BeautifulSoupCrawler">`B <Tabs groupId="main"> <TabItem value="json" label="JSON"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {JsonExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> <TabItem value="csv" label="CSV"> - <CodeBlock className="language-python"> + <RunnableCodeBlock className="language-python" language="python"> {CsvExample} - </CodeBlock> + </RunnableCodeBlock> </TabItem> </Tabs> diff --git a/docs/examples/fill_and_submit_web_form.mdx b/docs/examples/fill_and_submit_web_form.mdx index f8b1fdbc1f..841a2616ee 100644 --- a/docs/examples/fill_and_submit_web_form.mdx +++ b/docs/examples/fill_and_submit_web_form.mdx @@ -6,10 +6,10 @@ title: Fill and submit web form import ApiLink from '@site/src/components/ApiLink'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RequestExample from '!!raw-loader!./code_examples/fill_and_submit_web_form_request.py'; -import CrawlerExample from '!!raw-loader!./code_examples/fill_and_submit_web_form_crawler.py'; +import RequestExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_request.py'; +import CrawlerExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_crawler.py'; This example demonstrates how to fill and submit a web form using the <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink> crawler. The same approach applies to any crawler that inherits from it, such as the <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> or <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink>. @@ -42,9 +42,9 @@ The "Payload" tab will display the form fields and their submitted values. This Now, let's create a POST request with the form fields and their values using the <ApiLink to="class/Request">`Request`</ApiLink> class, specifically its <ApiLink to="class/Request#from_url">`Request.from_url`</ApiLink> constructor: -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {RequestExample} -</CodeBlock> +</RunnableCodeBlock> Alternatively, you can send form data as URL parameters using the `url` argument. It depends on the form and how it is implemented. However, sending the data as a POST request body using the `payload` is generally a better approach. @@ -52,9 +52,9 @@ Alternatively, you can send form data as URL parameters using the `url` argument Finally, let's implement the crawler and run it with the prepared request. Although we are using the <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink>, the process is the same for any crawler that inherits from it. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {CrawlerExample} -</CodeBlock> +</RunnableCodeBlock> ## Running the crawler diff --git a/docs/examples/json_logging.mdx b/docs/examples/json_logging.mdx index 8db5a3a1c1..06dd2ac492 100644 --- a/docs/examples/json_logging.mdx +++ b/docs/examples/json_logging.mdx @@ -4,17 +4,17 @@ title: Сonfigure JSON logging --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import JsonLoggingExample from '!!raw-loader!./code_examples/configure_json_logging.py'; +import JsonLoggingExample from '!!raw-loader!roa-loader!./code_examples/configure_json_logging.py'; This example demonstrates how to configure JSON line (JSONL) logging with Crawlee. By using the `use_table_logs=False` parameter, you can disable table-formatted statistics logs, which makes it easier to parse logs with external tools or to serialize them as JSON. The example shows how to integrate with the popular [`loguru`](https://github.com/delgan/loguru) library to capture Crawlee logs and format them as JSONL (one JSON object per line). This approach works well when you need to collect logs for analysis, monitoring, or when integrating with logging platforms like ELK Stack, Grafana Loki, or similar systems. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {JsonLoggingExample} -</CodeBlock> +</RunnableCodeBlock> Here's an example of what a crawler statistics log entry in JSONL format. diff --git a/docs/examples/parsel_crawler.mdx b/docs/examples/parsel_crawler.mdx index f59b58f62c..b0eca7eb28 100644 --- a/docs/examples/parsel_crawler.mdx +++ b/docs/examples/parsel_crawler.mdx @@ -4,12 +4,12 @@ title: Parsel crawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import ParselCrawlerExample from '!!raw-loader!./code_examples/parsel_crawler.py'; +import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler.py'; This example shows how to use <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> to crawl a website or a list of URLs. Each URL is loaded using a plain HTTP request and the response is parsed using [Parsel](https://pypi.org/project/parsel/) library which supports CSS and XPath selectors for HTML responses and JMESPath for JSON responses. We can extract data from all kinds of complex HTML structures using XPath. In this example, we will use Parsel to crawl github.com and extract page title, URL and emails found in the webpage. The default handler will scrape data from the current webpage and enqueue all the links found in the webpage for continuous scraping. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {ParselCrawlerExample} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/playwright_crawler.mdx b/docs/examples/playwright_crawler.mdx index 912e411bde..70b0bc8afb 100644 --- a/docs/examples/playwright_crawler.mdx +++ b/docs/examples/playwright_crawler.mdx @@ -4,9 +4,9 @@ title: Playwright crawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightCrawlerExample from '!!raw-loader!./code_examples/playwright_crawler.py'; +import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler.py'; This example demonstrates how to use <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> to recursively scrape the Hacker news website using headless Chromium and Playwright. @@ -14,6 +14,6 @@ The <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> manages A **pre-navigation hook** can be used to perform actions before navigating to the URL. This hook provides further flexibility in controlling environment and preparing for navigation. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {PlaywrightCrawlerExample} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/playwright_crawler_adaptive.mdx b/docs/examples/playwright_crawler_adaptive.mdx index a967354516..c1f8875df8 100644 --- a/docs/examples/playwright_crawler_adaptive.mdx +++ b/docs/examples/playwright_crawler_adaptive.mdx @@ -4,9 +4,9 @@ title: AdaptivePlaywrightCrawler --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import AdaptivePlaywrightCrawlerExample from '!!raw-loader!./code_examples/adaptive_playwright_crawler.py'; +import AdaptivePlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/adaptive_playwright_crawler.py'; This example demonstrates how to use <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink>. An <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink> is a combination of <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and some implementation of HTTP-based crawler such as <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> or <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. It uses a more limited crawling context interface so that it is able to switch to HTTP-only crawling when it detects that it may bring a performance benefit. @@ -15,6 +15,6 @@ A [pre-navigation hook](/python/docs/guides/adaptive-playwright-crawler#page-con For more detailed description please see [AdaptivePlaywrightCrawler guide](/python/docs/guides/adaptive-playwright-crawler 'AdaptivePlaywrightCrawler guide') -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {AdaptivePlaywrightCrawlerExample} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/playwright_crawler_with_block_requests.mdx b/docs/examples/playwright_crawler_with_block_requests.mdx index e4f2ab33b7..d7d5e15928 100644 --- a/docs/examples/playwright_crawler_with_block_requests.mdx +++ b/docs/examples/playwright_crawler_with_block_requests.mdx @@ -4,9 +4,9 @@ title: Playwright crawler with block requests --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightBlockRequests from '!!raw-loader!./code_examples/playwright_block_requests.py'; +import PlaywrightBlockRequests from '!!raw-loader!roa-loader!./code_examples/playwright_block_requests.py'; This example demonstrates how to optimize your <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> performance by blocking unnecessary network requests. @@ -22,6 +22,6 @@ By default, <ApiLink to="class/BlockRequestsFunction">`block_requests`</ApiLink> You can also replace the default patterns list with your own by providing `url_patterns`, or extend it by passing additional patterns in `extra_url_patterns`. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {PlaywrightBlockRequests} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/playwright_crawler_with_camoufox.mdx b/docs/examples/playwright_crawler_with_camoufox.mdx index 9c4a30d2d2..b627c9ba34 100644 --- a/docs/examples/playwright_crawler_with_camoufox.mdx +++ b/docs/examples/playwright_crawler_with_camoufox.mdx @@ -4,9 +4,9 @@ title: Playwright crawler with Camoufox --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightCrawlerExampleWithCamoufox from '!!raw-loader!./code_examples/playwright_crawler_with_camoufox.py'; +import PlaywrightCrawlerExampleWithCamoufox from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_camoufox.py'; This example demonstrates how to integrate Camoufox into <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> using <ApiLink to="class/BrowserPool">`BrowserPool`</ApiLink> with custom <ApiLink to="class/PlaywrightBrowserPlugin">`PlaywrightBrowserPlugin`</ApiLink>. @@ -21,6 +21,6 @@ For more details please refer to: https://github.com/daijro/camoufox/tree/main/p The example code after PlayWrightCrawler instantiation is similar to example describing the use of Playwright Crawler. The main difference is that in this example Camoufox will be used as the browser through BrowserPool. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {PlaywrightCrawlerExampleWithCamoufox} -</CodeBlock> +</RunnableCodeBlock> diff --git a/docs/examples/playwright_crawler_with_fingerprint_generator.mdx b/docs/examples/playwright_crawler_with_fingerprint_generator.mdx index 0551149ec2..e1c41e4c06 100644 --- a/docs/examples/playwright_crawler_with_fingerprint_generator.mdx +++ b/docs/examples/playwright_crawler_with_fingerprint_generator.mdx @@ -4,14 +4,14 @@ title: Playwright crawler with fingerprint generator --- import ApiLink from '@site/src/components/ApiLink'; -import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import PlaywrightCrawlerExample from '!!raw-loader!./code_examples/playwright_crawler_with_fingerprint_generator.py'; +import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_fingerprint_generator.py'; This example demonstrates how to use <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> together with <ApiLink to="class/FingerprintGenerator">`FingerprintGenerator`</ApiLink> that will populate several browser attributes to mimic real browser fingerprint. To read more about fingerprints please see: https://docs.apify.com/academy/anti-scraping/techniques/fingerprinting. You can implement your own fingerprint generator or use <ApiLink to="class/BrowserforgeFingerprintGenerator">`DefaultFingerprintGenerator`</ApiLink>. To use the generator initialize it with the desired fingerprint options. The generator will try to create fingerprint based on those options. Unspecified options will be automatically selected by the generator from the set of reasonable values. If some option is important for you, do not rely on the default and explicitly define it. -<CodeBlock className="language-python"> +<RunnableCodeBlock className="language-python" language="python"> {PlaywrightCrawlerExample} -</CodeBlock> +</RunnableCodeBlock> From 070ec7004e57cd0730cfe9237f395ec1558e1f79 Mon Sep 17 00:00:00 2001 From: Vlada Dusek <v.dusek96@gmail.com> Date: Thu, 3 Apr 2025 09:44:48 +0200 Subject: [PATCH 12/16] fix spacing --- docs/examples/code_examples/playwright_crawler_with_camoufox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/code_examples/playwright_crawler_with_camoufox.py b/docs/examples/code_examples/playwright_crawler_with_camoufox.py index a09dd5f8df..c843e8372a 100644 --- a/docs/examples/code_examples/playwright_crawler_with_camoufox.py +++ b/docs/examples/code_examples/playwright_crawler_with_camoufox.py @@ -1,4 +1,4 @@ -# Camoufox is external package and needs to be installed. +# Camoufox is external package and needs to be installed. # It is not included in Crawlee. from camoufox import AsyncNewBrowser from typing_extensions import override From 0ca6df530f757ff4d0f0439df9c881ac83f5e750 Mon Sep 17 00:00:00 2001 From: Vlada Dusek <v.dusek96@gmail.com> Date: Thu, 3 Apr 2025 11:49:01 +0200 Subject: [PATCH 13/16] fix spacing --- docs/introduction/06_scraping.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/introduction/06_scraping.mdx b/docs/introduction/06_scraping.mdx index 443819a3de..51c86e5835 100644 --- a/docs/introduction/06_scraping.mdx +++ b/docs/introduction/06_scraping.mdx @@ -33,8 +33,7 @@ By splitting the `request.url`, we can extract the manufacturer name directly fr ```python # context.request.url: -# https://warehouse-theme-metal.myshopify.com/products/ -# sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440 +# https://warehouse-theme-metal.myshopify.com/products/sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440 # Split the URL and get the last part. url_part = context.request.url.split('/').pop() From fb7c6588f6e952487c1bb717af8afb33135bb638 Mon Sep 17 00:00:00 2001 From: Vlada Dusek <v.dusek96@gmail.com> Date: Thu, 3 Apr 2025 11:53:39 +0200 Subject: [PATCH 14/16] Revert "rm asyncio run" This reverts commit a3710c07cd038ea0adf1a61d6173bb00ba1f81db. --- .../code_examples/adaptive_playwright_crawler.py | 5 +++++ .../examples/code_examples/add_data_to_dataset_bs.py | 11 +++++++---- .../examples/code_examples/add_data_to_dataset_pw.py | 6 ++++++ docs/examples/code_examples/beautifulsoup_crawler.py | 5 +++++ .../beautifulsoup_crawler_keep_alive.py | 4 ++++ .../code_examples/beautifulsoup_crawler_stop.py | 11 +++++++---- .../capture_screenshot_using_playwright.py | 6 ++++++ .../examples/code_examples/configure_json_logging.py | 5 +++++ .../code_examples/crawl_all_links_on_website_bs.py | 11 +++++++---- .../code_examples/crawl_all_links_on_website_pw.py | 6 ++++++ .../examples/code_examples/crawl_multiple_urls_bs.py | 11 +++++++---- .../examples/code_examples/crawl_multiple_urls_pw.py | 6 ++++++ .../crawl_specific_links_on_website_bs.py | 6 ++++++ .../crawl_specific_links_on_website_pw.py | 6 ++++++ .../crawl_website_with_relative_links_all_links.py | 11 +++++++---- .../crawl_website_with_relative_links_same_domain.py | 11 +++++++---- ...rawl_website_with_relative_links_same_hostname.py | 11 +++++++---- .../crawl_website_with_relative_links_same_origin.py | 11 +++++++---- .../export_entire_dataset_to_file_csv.py | 11 +++++++---- .../export_entire_dataset_to_file_json.py | 11 +++++++---- .../fill_and_submit_web_form_crawler.py | 5 +++++ docs/examples/code_examples/parsel_crawler.py | 12 +++++++----- .../code_examples/playwright_block_requests.py | 6 ++++++ docs/examples/code_examples/playwright_crawler.py | 6 ++++++ .../playwright_crawler_with_camoufox.py | 9 +++++++-- .../playwright_crawler_with_fingerprint_generator.py | 6 ++++++ .../playwright_with_fingerprint_generator.py | 6 ++++++ .../http_clients/curl_impersonate_example.py | 11 +++++++---- .../code_examples/http_clients/httpx_example.py | 11 +++++++---- .../browser_configuration_example.py | 6 ++++++ .../playwright_crawler/multiple_launch_example.py | 6 ++++++ .../pre_navigation_hook_example.py | 6 ++++++ .../proxy_management/inspecting_bs_example.py | 11 +++++++---- .../proxy_management/inspecting_pw_example.py | 6 ++++++ .../proxy_management/integration_bs_example.py | 11 +++++++---- .../proxy_management/integration_pw_example.py | 6 ++++++ .../proxy_management/quick_start_example.py | 6 ++++++ .../proxy_management/tiers_bs_example.py | 11 +++++++---- .../proxy_management/tiers_pw_example.py | 6 ++++++ .../request_loaders/rl_basic_example.py | 6 ++++++ .../code_examples/request_loaders/tandem_example.py | 5 +++++ .../request_loaders/tandem_example_explicit.py | 5 +++++ .../scaling_crawlers/max_tasks_per_minute_example.py | 6 ++++++ .../min_and_max_concurrency_example.py | 6 ++++++ .../session_management/multi_sessions_http.py | 5 +++++ .../session_management/one_session_http.py | 5 +++++ .../code_examples/session_management/sm_basic.py | 5 +++++ .../session_management/sm_beautifulsoup.py | 11 +++++++---- .../code_examples/session_management/sm_http.py | 5 +++++ .../code_examples/session_management/sm_parsel.py | 6 ++++++ .../session_management/sm_playwright.py | 6 ++++++ .../session_management/sm_standalone.py | 6 ++++++ .../storages/cleaning_do_not_purge_example.py | 6 ++++++ .../storages/cleaning_purge_explicitly_example.py | 6 ++++++ .../code_examples/storages/dataset_basic_example.py | 6 ++++++ .../storages/dataset_with_crawler_example.py | 11 +++++++---- .../dataset_with_crawler_explicit_example.py | 11 +++++++---- .../storages/helper_add_requests_example.py | 11 +++++++---- .../storages/helper_enqueue_links_example.py | 11 +++++++---- .../code_examples/storages/kvs_basic_example.py | 6 ++++++ .../storages/kvs_with_crawler_example.py | 6 ++++++ .../storages/kvs_with_crawler_explicit_example.py | 6 ++++++ .../code_examples/storages/rq_basic_example.py | 6 ++++++ .../storages/rq_with_crawler_example.py | 6 ++++++ .../storages/rq_with_crawler_explicit_example.py | 6 ++++++ docs/introduction/code_examples/02_bs.py | 6 ++++++ docs/introduction/code_examples/02_bs_better.py | 6 ++++++ docs/introduction/code_examples/02_request_queue.py | 6 ++++++ .../code_examples/03_finding_new_links.py | 11 +++++++---- docs/introduction/code_examples/03_original_code.py | 11 +++++++---- docs/introduction/code_examples/04_sanity_check.py | 9 +++++++-- .../introduction/code_examples/05_crawling_detail.py | 6 ++++++ .../code_examples/05_crawling_listing.py | 6 ++++++ docs/introduction/code_examples/06_scraping.py | 6 ++++++ docs/introduction/code_examples/07_final_code.py | 6 ++++++ docs/introduction/code_examples/08_main.py | 6 ++++++ docs/introduction/code_examples/09_apify_sdk.py | 6 ++++++ .../code_examples/beautifulsoup_crawler_example.py | 11 +++++++---- .../code_examples/parsel_crawler_example.py | 6 ++++++ .../code_examples/playwright_crawler_example.py | 6 ++++++ 80 files changed, 494 insertions(+), 101 deletions(-) diff --git a/docs/examples/code_examples/adaptive_playwright_crawler.py b/docs/examples/code_examples/adaptive_playwright_crawler.py index 1827a5ef13..f2851d502b 100644 --- a/docs/examples/code_examples/adaptive_playwright_crawler.py +++ b/docs/examples/code_examples/adaptive_playwright_crawler.py @@ -1,3 +1,4 @@ +import asyncio from datetime import timedelta from playwright.async_api import Route @@ -58,3 +59,7 @@ async def some_routing_function(route: Route) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://warehouse-theme-metal.myshopify.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/add_data_to_dataset_bs.py b/docs/examples/code_examples/add_data_to_dataset_bs.py index 4e8b41d046..4318cbe0d4 100644 --- a/docs/examples/code_examples/add_data_to_dataset_bs.py +++ b/docs/examples/code_examples/add_data_to_dataset_bs.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -30,3 +29,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: 'https://example.com', ] ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/add_data_to_dataset_pw.py b/docs/examples/code_examples/add_data_to_dataset_pw.py index 9b07ac90db..8eb714aef3 100644 --- a/docs/examples/code_examples/add_data_to_dataset_pw.py +++ b/docs/examples/code_examples/add_data_to_dataset_pw.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -27,3 +29,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: 'https://example.com', ] ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/beautifulsoup_crawler.py b/docs/examples/code_examples/beautifulsoup_crawler.py index 73634ed4f0..5e9701d7cb 100644 --- a/docs/examples/code_examples/beautifulsoup_crawler.py +++ b/docs/examples/code_examples/beautifulsoup_crawler.py @@ -1,3 +1,4 @@ +import asyncio from datetime import timedelta from crawlee.crawlers import ( @@ -50,3 +51,7 @@ async def some_hook(context: BasicCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py b/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py index 2f10cd148d..38e5623939 100644 --- a/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py +++ b/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py @@ -50,3 +50,7 @@ async def request_handler(context: BasicCrawlingContext) -> None: await crawler.run() await asyncio.gather(add_request_later_task1, add_request_later_task2) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/beautifulsoup_crawler_stop.py b/docs/examples/code_examples/beautifulsoup_crawler_stop.py index 8a54109dbf..2069bd6ecb 100644 --- a/docs/examples/code_examples/beautifulsoup_crawler_stop.py +++ b/docs/examples/code_examples/beautifulsoup_crawler_stop.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -36,3 +35,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/capture_screenshot_using_playwright.py b/docs/examples/code_examples/capture_screenshot_using_playwright.py index 4a02f51dc6..e4b4c1ec22 100644 --- a/docs/examples/code_examples/capture_screenshot_using_playwright.py +++ b/docs/examples/code_examples/capture_screenshot_using_playwright.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.storages import KeyValueStore @@ -39,3 +41,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: 'https://example.com', ] ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/configure_json_logging.py b/docs/examples/code_examples/configure_json_logging.py index 1831cc5fd6..25cb37c745 100644 --- a/docs/examples/code_examples/configure_json_logging.py +++ b/docs/examples/code_examples/configure_json_logging.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import inspect import logging import sys @@ -84,3 +85,7 @@ async def request_handler(context: HttpCrawlingContext) -> None: # Run the crawler await crawler.run(['https://www.crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_all_links_on_website_bs.py b/docs/examples/code_examples/crawl_all_links_on_website_bs.py index a289ab5409..ad5ef62f54 100644 --- a/docs/examples/code_examples/crawl_all_links_on_website_bs.py +++ b/docs/examples/code_examples/crawl_all_links_on_website_bs.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -20,3 +19,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_all_links_on_website_pw.py b/docs/examples/code_examples/crawl_all_links_on_website_pw.py index 83e7659e5b..4a6fb6e616 100644 --- a/docs/examples/code_examples/crawl_all_links_on_website_pw.py +++ b/docs/examples/code_examples/crawl_all_links_on_website_pw.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -17,3 +19,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_multiple_urls_bs.py b/docs/examples/code_examples/crawl_multiple_urls_bs.py index 2d6de8e57b..e8cf82f2bc 100644 --- a/docs/examples/code_examples/crawl_multiple_urls_bs.py +++ b/docs/examples/code_examples/crawl_multiple_urls_bs.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -20,3 +19,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: 'https://example.com', ] ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_multiple_urls_pw.py b/docs/examples/code_examples/crawl_multiple_urls_pw.py index ef4eccaf01..b18d04c8ad 100644 --- a/docs/examples/code_examples/crawl_multiple_urls_pw.py +++ b/docs/examples/code_examples/crawl_multiple_urls_pw.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -17,3 +19,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: 'https://example.com', ] ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py b/docs/examples/code_examples/crawl_specific_links_on_website_bs.py index dc19b394c2..f0e77c4b5e 100644 --- a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py +++ b/docs/examples/code_examples/crawl_specific_links_on_website_bs.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee import Glob from crawlee.crawlers import ( BeautifulSoupCrawler, @@ -24,3 +26,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_specific_links_on_website_pw.py b/docs/examples/code_examples/crawl_specific_links_on_website_pw.py index 144c55e1b0..98a2f0435b 100644 --- a/docs/examples/code_examples/crawl_specific_links_on_website_pw.py +++ b/docs/examples/code_examples/crawl_specific_links_on_website_pw.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee import Glob from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -21,3 +23,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py b/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py index f6eefd6a8d..b253a9566f 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -21,3 +20,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py index 335f2db6f4..0fa264ef20 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -21,3 +20,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py index a5093e0e83..0259cafe67 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -21,3 +20,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py b/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py index ca2f93e472..46e9f32759 100644 --- a/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py +++ b/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -21,3 +20,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/export_entire_dataset_to_file_csv.py b/docs/examples/code_examples/export_entire_dataset_to_file_csv.py index bd0dfc2bdf..115474fc61 100644 --- a/docs/examples/code_examples/export_entire_dataset_to_file_csv.py +++ b/docs/examples/code_examples/export_entire_dataset_to_file_csv.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -32,3 +31,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Export the entire dataset to a CSV file. await crawler.export_data_csv(path='results.csv') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/export_entire_dataset_to_file_json.py b/docs/examples/code_examples/export_entire_dataset_to_file_json.py index d6e6eb2f7b..5c871fb228 100644 --- a/docs/examples/code_examples/export_entire_dataset_to_file_json.py +++ b/docs/examples/code_examples/export_entire_dataset_to_file_json.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -32,3 +31,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Export the entire dataset to a JSON file. await crawler.export_data_json(path='results.json') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/fill_and_submit_web_form_crawler.py b/docs/examples/code_examples/fill_and_submit_web_form_crawler.py index e27c74e9b3..c00a784411 100644 --- a/docs/examples/code_examples/fill_and_submit_web_form_crawler.py +++ b/docs/examples/code_examples/fill_and_submit_web_form_crawler.py @@ -1,3 +1,4 @@ +import asyncio from urllib.parse import urlencode from crawlee import Request @@ -34,3 +35,7 @@ async def request_handler(context: HttpCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run([request]) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/parsel_crawler.py b/docs/examples/code_examples/parsel_crawler.py index 9ba403d832..61ddb7484e 100644 --- a/docs/examples/code_examples/parsel_crawler.py +++ b/docs/examples/code_examples/parsel_crawler.py @@ -1,8 +1,6 @@ -from crawlee.crawlers import ( - BasicCrawlingContext, - ParselCrawler, - ParselCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext # Regex for identifying email addresses on a webpage. EMAIL_REGEX = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' @@ -43,3 +41,7 @@ async def some_hook(context: BasicCrawlingContext) -> None: # Export the entire dataset to a JSON file. await crawler.export_data_json(path='results.json') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_block_requests.py b/docs/examples/code_examples/playwright_block_requests.py index 9a6af4c65f..991a67aede 100644 --- a/docs/examples/code_examples/playwright_block_requests.py +++ b/docs/examples/code_examples/playwright_block_requests.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import ( PlaywrightCrawler, PlaywrightCrawlingContext, @@ -28,3 +30,7 @@ async def navigation_hook(context: PlaywrightPreNavCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_crawler.py b/docs/examples/code_examples/playwright_crawler.py index e9cebfc028..f35332b063 100644 --- a/docs/examples/code_examples/playwright_crawler.py +++ b/docs/examples/code_examples/playwright_crawler.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import ( PlaywrightCrawler, PlaywrightCrawlingContext, @@ -59,3 +61,7 @@ async def log_navigation_url(context: PlaywrightPreNavCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://news.ycombinator.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_crawler_with_camoufox.py b/docs/examples/code_examples/playwright_crawler_with_camoufox.py index c843e8372a..691197da55 100644 --- a/docs/examples/code_examples/playwright_crawler_with_camoufox.py +++ b/docs/examples/code_examples/playwright_crawler_with_camoufox.py @@ -1,5 +1,6 @@ -# Camoufox is external package and needs to be installed. -# It is not included in Crawlee. +import asyncio + +# Camoufox is external package and needs to be installed. It is not included in crawlee. from camoufox import AsyncNewBrowser from typing_extensions import override @@ -62,3 +63,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://news.ycombinator.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py b/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py index 6e1c363e3f..a5d80797fb 100644 --- a/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py +++ b/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.fingerprint_suite import ( DefaultFingerprintGenerator, @@ -36,3 +38,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://news.ycombinator.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py b/docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py index f0ec2db446..5e1c8d2668 100644 --- a/docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py +++ b/docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -15,3 +17,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://news.ycombinator.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/http_clients/curl_impersonate_example.py b/docs/guides/code_examples/http_clients/curl_impersonate_example.py index 5785f3576e..28813a2f46 100644 --- a/docs/guides/code_examples/http_clients/curl_impersonate_example.py +++ b/docs/guides/code_examples/http_clients/curl_impersonate_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.http_clients import CurlImpersonateHttpClient @@ -37,3 +36,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/http_clients/httpx_example.py b/docs/guides/code_examples/http_clients/httpx_example.py index 826b7522c5..5c3c4883cb 100644 --- a/docs/guides/code_examples/http_clients/httpx_example.py +++ b/docs/guides/code_examples/http_clients/httpx_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.http_clients import HttpxHttpClient @@ -37,3 +36,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py b/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py index 977c56a8b1..10ff84eba0 100644 --- a/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py +++ b/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -35,3 +37,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py b/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py index 8f228cf326..59219b89c7 100644 --- a/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py +++ b/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -30,3 +32,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev', 'https://apify.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler/pre_navigation_hook_example.py b/docs/guides/code_examples/playwright_crawler/pre_navigation_hook_example.py index bf6a919d22..5a36456679 100644 --- a/docs/guides/code_examples/playwright_crawler/pre_navigation_hook_example.py +++ b/docs/guides/code_examples/playwright_crawler/pre_navigation_hook_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import ( PlaywrightCrawler, PlaywrightCrawlingContext, @@ -26,3 +28,7 @@ async def log_navigation_url(context: PlaywrightPreNavCrawlingContext) -> None: # Run the crawler with the initial list of URLs. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/inspecting_bs_example.py b/docs/guides/code_examples/proxy_management/inspecting_bs_example.py index 768b59ff3b..b6035097d6 100644 --- a/docs/guides/code_examples/proxy_management/inspecting_bs_example.py +++ b/docs/guides/code_examples/proxy_management/inspecting_bs_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -23,3 +22,7 @@ async def default_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/inspecting_pw_example.py b/docs/guides/code_examples/proxy_management/inspecting_pw_example.py index f82c04cd9b..e193972399 100644 --- a/docs/guides/code_examples/proxy_management/inspecting_pw_example.py +++ b/docs/guides/code_examples/proxy_management/inspecting_pw_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -20,3 +22,7 @@ async def default_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/integration_bs_example.py b/docs/guides/code_examples/proxy_management/integration_bs_example.py index 0f4d12f354..63a2e703e3 100644 --- a/docs/guides/code_examples/proxy_management/integration_bs_example.py +++ b/docs/guides/code_examples/proxy_management/integration_bs_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -27,3 +26,7 @@ async def default_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/integration_pw_example.py b/docs/guides/code_examples/proxy_management/integration_pw_example.py index 394551d706..e8e0ea8821 100644 --- a/docs/guides/code_examples/proxy_management/integration_pw_example.py +++ b/docs/guides/code_examples/proxy_management/integration_pw_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -24,3 +26,7 @@ async def default_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/quick_start_example.py b/docs/guides/code_examples/proxy_management/quick_start_example.py index 3941f1440e..3b43a48312 100644 --- a/docs/guides/code_examples/proxy_management/quick_start_example.py +++ b/docs/guides/code_examples/proxy_management/quick_start_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.proxy_configuration import ProxyConfiguration @@ -13,3 +15,7 @@ async def main() -> None: proxy_url_1 = await proxy_configuration.new_url() # http://proxy-1.com/ proxy_url_2 = await proxy_configuration.new_url() # http://proxy-2.com/ proxy_url_3 = await proxy_configuration.new_url() # http://proxy-1.com/ + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/tiers_bs_example.py b/docs/guides/code_examples/proxy_management/tiers_bs_example.py index 017549b163..37f69e6419 100644 --- a/docs/guides/code_examples/proxy_management/tiers_bs_example.py +++ b/docs/guides/code_examples/proxy_management/tiers_bs_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -34,3 +33,7 @@ async def default_handler(context: BeautifulSoupCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/tiers_pw_example.py b/docs/guides/code_examples/proxy_management/tiers_pw_example.py index 53500e3786..2dcb5ad3bd 100644 --- a/docs/guides/code_examples/proxy_management/tiers_pw_example.py +++ b/docs/guides/code_examples/proxy_management/tiers_pw_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration @@ -31,3 +33,7 @@ async def default_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial list of requests. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/rl_basic_example.py b/docs/guides/code_examples/request_loaders/rl_basic_example.py index ec1e347a7a..abe4d55584 100644 --- a/docs/guides/code_examples/request_loaders/rl_basic_example.py +++ b/docs/guides/code_examples/request_loaders/rl_basic_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.request_loaders import RequestList @@ -19,3 +21,7 @@ async def main() -> None: # And mark it as handled. await request_list.mark_request_as_handled(request) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/tandem_example.py b/docs/guides/code_examples/request_loaders/tandem_example.py index 338590d6af..0739d483ea 100644 --- a/docs/guides/code_examples/request_loaders/tandem_example.py +++ b/docs/guides/code_examples/request_loaders/tandem_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import ParselCrawler, ParselCrawlingContext from crawlee.request_loaders import RequestList @@ -19,3 +21,6 @@ async def handler(context: ParselCrawlingContext) -> None: await context.enqueue_links() await crawler.run() + + +asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/tandem_example_explicit.py b/docs/guides/code_examples/request_loaders/tandem_example_explicit.py index 11a6008e1b..ff34e1af5d 100644 --- a/docs/guides/code_examples/request_loaders/tandem_example_explicit.py +++ b/docs/guides/code_examples/request_loaders/tandem_example_explicit.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import ParselCrawler, ParselCrawlingContext from crawlee.request_loaders import RequestList, RequestManagerTandem from crawlee.storages import RequestQueue @@ -22,3 +24,6 @@ async def handler(context: ParselCrawlingContext) -> None: await context.enqueue_links() await crawler.run() + + +asyncio.run(main()) diff --git a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py b/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py index 3871a5bad7..cbc1130bc7 100644 --- a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py +++ b/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee import ConcurrencySettings from crawlee.crawlers import BeautifulSoupCrawler @@ -17,3 +19,7 @@ async def main() -> None: ) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py b/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py index 496f059290..4d491446d0 100644 --- a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py +++ b/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee import ConcurrencySettings from crawlee.crawlers import BeautifulSoupCrawler @@ -19,3 +21,7 @@ async def main() -> None: ) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/multi_sessions_http.py b/docs/guides/code_examples/session_management/multi_sessions_http.py index 2ad8ff8802..aba4d5e9d0 100644 --- a/docs/guides/code_examples/session_management/multi_sessions_http.py +++ b/docs/guides/code_examples/session_management/multi_sessions_http.py @@ -1,3 +1,4 @@ +import asyncio from datetime import timedelta from itertools import count from typing import Callable @@ -78,3 +79,7 @@ async def error_processing(context: BasicCrawlingContext, error: Exception) -> N ] await crawler.run(init_requests) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/one_session_http.py b/docs/guides/code_examples/session_management/one_session_http.py index 58dc6815bb..28cec44b63 100644 --- a/docs/guides/code_examples/session_management/one_session_http.py +++ b/docs/guides/code_examples/session_management/one_session_http.py @@ -1,3 +1,4 @@ +import asyncio from datetime import timedelta from crawlee import ConcurrencySettings, Request @@ -49,3 +50,7 @@ async def error_processing(context: BasicCrawlingContext, error: Exception) -> N crawler.stop() await crawler.run([Request.from_url('https://example.org/', label='session_init')]) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/sm_basic.py b/docs/guides/code_examples/session_management/sm_basic.py index 9241f184be..958ad5a665 100644 --- a/docs/guides/code_examples/session_management/sm_basic.py +++ b/docs/guides/code_examples/session_management/sm_basic.py @@ -1,3 +1,4 @@ +import asyncio import re from crawlee.crawlers import BasicCrawler, BasicCrawlingContext @@ -41,3 +42,7 @@ async def default_handler(context: BasicCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/sm_beautifulsoup.py b/docs/guides/code_examples/session_management/sm_beautifulsoup.py index da8781c56b..a54fd8425f 100644 --- a/docs/guides/code_examples/session_management/sm_beautifulsoup.py +++ b/docs/guides/code_examples/session_management/sm_beautifulsoup.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration from crawlee.sessions import SessionPool @@ -37,3 +36,7 @@ async def default_handler(context: BeautifulSoupCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/sm_http.py b/docs/guides/code_examples/session_management/sm_http.py index 275b0a9fe9..cd12d04bdf 100644 --- a/docs/guides/code_examples/session_management/sm_http.py +++ b/docs/guides/code_examples/session_management/sm_http.py @@ -1,3 +1,4 @@ +import asyncio import re from crawlee.crawlers import HttpCrawler, HttpCrawlingContext @@ -37,3 +38,7 @@ async def default_handler(context: HttpCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/sm_parsel.py b/docs/guides/code_examples/session_management/sm_parsel.py index 583f1e3a06..66752a63c3 100644 --- a/docs/guides/code_examples/session_management/sm_parsel.py +++ b/docs/guides/code_examples/session_management/sm_parsel.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import ParselCrawler, ParselCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration from crawlee.sessions import SessionPool @@ -34,3 +36,7 @@ async def default_handler(context: ParselCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/sm_playwright.py b/docs/guides/code_examples/session_management/sm_playwright.py index bcc1ea7aa0..46a4c4f096 100644 --- a/docs/guides/code_examples/session_management/sm_playwright.py +++ b/docs/guides/code_examples/session_management/sm_playwright.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.proxy_configuration import ProxyConfiguration from crawlee.sessions import SessionPool @@ -34,3 +36,7 @@ async def default_handler(context: PlaywrightCrawlingContext) -> None: context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/sm_standalone.py b/docs/guides/code_examples/session_management/sm_standalone.py index ae0d542273..32989dc7e0 100644 --- a/docs/guides/code_examples/session_management/sm_standalone.py +++ b/docs/guides/code_examples/session_management/sm_standalone.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.sessions import SessionPool @@ -17,3 +19,7 @@ async def main() -> None: # Lower the error_score and mark the session good. session.mark_good() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py b/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py index dceab42d16..6514863555 100644 --- a/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py +++ b/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.configuration import Configuration from crawlee.crawlers import HttpCrawler, HttpCrawlingContext @@ -15,3 +17,7 @@ async def request_handler(context: HttpCrawlingContext) -> None: context.log.info(f'Processing {context.request.url} ...') await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py b/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py index 3e3c400a4e..15435da7bf 100644 --- a/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py +++ b/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import HttpCrawler from crawlee.storage_clients import MemoryStorageClient @@ -13,3 +15,7 @@ async def main() -> None: crawler = HttpCrawler(storage_client=storage_client) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/dataset_basic_example.py b/docs/guides/code_examples/storages/dataset_basic_example.py index 26bff30518..9b67f36eb0 100644 --- a/docs/guides/code_examples/storages/dataset_basic_example.py +++ b/docs/guides/code_examples/storages/dataset_basic_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.storages import Dataset @@ -18,3 +20,7 @@ async def main() -> None: # Remove the dataset. await dataset.drop() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/dataset_with_crawler_example.py b/docs/guides/code_examples/storages/dataset_with_crawler_example.py index 7813bb6e77..7e40824166 100644 --- a/docs/guides/code_examples/storages/dataset_with_crawler_example.py +++ b/docs/guides/code_examples/storages/dataset_with_crawler_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -27,3 +26,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Export the dataset to a file. await crawler.export_data(path='dataset.csv') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py b/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py index 9771f9bc8a..7c6a613b8f 100644 --- a/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py +++ b/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.storages import Dataset @@ -32,3 +31,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Export the dataset to the key-value store. await dataset.export_to(key='dataset', content_type='csv') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/helper_add_requests_example.py b/docs/guides/code_examples/storages/helper_add_requests_example.py index d524278f7f..15104cf6fc 100644 --- a/docs/guides/code_examples/storages/helper_add_requests_example.py +++ b/docs/guides/code_examples/storages/helper_add_requests_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -14,3 +13,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await context.add_requests(['https://apify.com/']) await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/helper_enqueue_links_example.py b/docs/guides/code_examples/storages/helper_enqueue_links_example.py index 7745755adf..6c7392bc3b 100644 --- a/docs/guides/code_examples/storages/helper_enqueue_links_example.py +++ b/docs/guides/code_examples/storages/helper_enqueue_links_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -14,3 +13,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await context.enqueue_links() await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/kvs_basic_example.py b/docs/guides/code_examples/storages/kvs_basic_example.py index aa18cf88a5..7821fa75de 100644 --- a/docs/guides/code_examples/storages/kvs_basic_example.py +++ b/docs/guides/code_examples/storages/kvs_basic_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.storages import KeyValueStore @@ -18,3 +20,7 @@ async def main() -> None: # Remove the key-value store. await kvs.drop() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/kvs_with_crawler_example.py b/docs/guides/code_examples/storages/kvs_with_crawler_example.py index 781a7c18ec..732ee41f76 100644 --- a/docs/guides/code_examples/storages/kvs_with_crawler_example.py +++ b/docs/guides/code_examples/storages/kvs_with_crawler_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -27,3 +29,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial URLs. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py b/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py index 19ad386415..66a921bd04 100644 --- a/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py +++ b/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.storages import KeyValueStore @@ -28,3 +30,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Run the crawler with the initial URLs. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/rq_basic_example.py b/docs/guides/code_examples/storages/rq_basic_example.py index 9611d1ce03..9e983bb9fe 100644 --- a/docs/guides/code_examples/storages/rq_basic_example.py +++ b/docs/guides/code_examples/storages/rq_basic_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.storages import RequestQueue @@ -23,3 +25,7 @@ async def main() -> None: # Remove the request queue. await request_queue.drop() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/rq_with_crawler_example.py b/docs/guides/code_examples/storages/rq_with_crawler_example.py index 1f3fd8ea0a..ce6a34cb59 100644 --- a/docs/guides/code_examples/storages/rq_with_crawler_example.py +++ b/docs/guides/code_examples/storages/rq_with_crawler_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import HttpCrawler, HttpCrawlingContext @@ -19,3 +21,7 @@ async def request_handler(context: HttpCrawlingContext) -> None: # Run the crawler. You can optionally pass the list of initial requests. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py b/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py index c63fcccfc5..21bedad0b9 100644 --- a/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py +++ b/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import HttpCrawler, HttpCrawlingContext from crawlee.storages import RequestQueue @@ -23,3 +25,7 @@ async def request_handler(context: HttpCrawlingContext) -> None: # And execute the crawler. await crawler.run() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/02_bs.py b/docs/introduction/code_examples/02_bs.py index 5d580141ba..87af780b8c 100644 --- a/docs/introduction/code_examples/02_bs.py +++ b/docs/introduction/code_examples/02_bs.py @@ -1,3 +1,5 @@ +import asyncio + # Add import of crawler and crawling context. from crawlee.crawlers import ( BeautifulSoupCrawler, @@ -25,3 +27,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: context.log.info(f'The title of {url} is: {title}.') await crawler.run() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/02_bs_better.py b/docs/introduction/code_examples/02_bs_better.py index c0d897ce5c..a027cbb343 100644 --- a/docs/introduction/code_examples/02_bs_better.py +++ b/docs/introduction/code_examples/02_bs_better.py @@ -1,3 +1,5 @@ +import asyncio + # You don't need to import RequestQueue anymore. from crawlee.crawlers import ( BeautifulSoupCrawler, @@ -16,3 +18,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Start the crawler with the provided URLs. await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/02_request_queue.py b/docs/introduction/code_examples/02_request_queue.py index 4e11ad4aa0..e6cc5eb8c3 100644 --- a/docs/introduction/code_examples/02_request_queue.py +++ b/docs/introduction/code_examples/02_request_queue.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.storages import RequestQueue @@ -7,3 +9,7 @@ async def main() -> None: # And then you add one or more requests to it. await rq.add_request('https://crawlee.dev') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/03_finding_new_links.py b/docs/introduction/code_examples/03_finding_new_links.py index 8925fab220..e25af30c13 100644 --- a/docs/introduction/code_examples/03_finding_new_links.py +++ b/docs/introduction/code_examples/03_finding_new_links.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -19,3 +18,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await context.enqueue_links() await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/03_original_code.py b/docs/introduction/code_examples/03_original_code.py index 5b9fd7a071..976e84d562 100644 --- a/docs/introduction/code_examples/03_original_code.py +++ b/docs/introduction/code_examples/03_original_code.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -14,3 +13,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: context.log.info(f'The title of {url} is: {title}.') await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/04_sanity_check.py b/docs/introduction/code_examples/04_sanity_check.py index 1da52fd425..5bfbccd27e 100644 --- a/docs/introduction/code_examples/04_sanity_check.py +++ b/docs/introduction/code_examples/04_sanity_check.py @@ -1,5 +1,6 @@ -# Instead of BeautifulSoupCrawler let's use Playwright to be able -# to render JavaScript. +import asyncio + +# Instead of BeautifulSoupCrawler let's use Playwright to be able to render JavaScript. from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -25,3 +26,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: context.log.info(f'CATEGORY_{i + 1}: {text}') await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/05_crawling_detail.py b/docs/introduction/code_examples/05_crawling_detail.py index 6a4ab84963..a6845f23b0 100644 --- a/docs/introduction/code_examples/05_crawling_detail.py +++ b/docs/introduction/code_examples/05_crawling_detail.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -49,3 +51,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/05_crawling_listing.py b/docs/introduction/code_examples/05_crawling_listing.py index fb481058da..c9c47f57d8 100644 --- a/docs/introduction/code_examples/05_crawling_listing.py +++ b/docs/introduction/code_examples/05_crawling_listing.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -20,3 +22,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/06_scraping.py b/docs/introduction/code_examples/06_scraping.py index 9561eb3522..f1faf1c521 100644 --- a/docs/introduction/code_examples/06_scraping.py +++ b/docs/introduction/code_examples/06_scraping.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -89,3 +91,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/07_final_code.py b/docs/introduction/code_examples/07_final_code.py index b7379eca8e..a1a89167b5 100644 --- a/docs/introduction/code_examples/07_final_code.py +++ b/docs/introduction/code_examples/07_final_code.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -89,3 +91,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/08_main.py b/docs/introduction/code_examples/08_main.py index f877cc2e1d..09f33e3376 100644 --- a/docs/introduction/code_examples/08_main.py +++ b/docs/introduction/code_examples/08_main.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler from .routes import router @@ -12,3 +14,7 @@ async def main() -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/09_apify_sdk.py b/docs/introduction/code_examples/09_apify_sdk.py index f7a5acd839..fd8ceaffe7 100644 --- a/docs/introduction/code_examples/09_apify_sdk.py +++ b/docs/introduction/code_examples/09_apify_sdk.py @@ -1,3 +1,5 @@ +import asyncio + # highlight-next-line from apify import Actor @@ -17,3 +19,7 @@ async def main() -> None: ) await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py index b2625c59e7..2db8874c4b 100644 --- a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py +++ b/docs/quick-start/code_examples/beautifulsoup_crawler_example.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -26,3 +25,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: # Add first URL to the queue and start the crawl. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/quick-start/code_examples/parsel_crawler_example.py b/docs/quick-start/code_examples/parsel_crawler_example.py index 297abd2b84..f8ed2a3e9c 100644 --- a/docs/quick-start/code_examples/parsel_crawler_example.py +++ b/docs/quick-start/code_examples/parsel_crawler_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import ParselCrawler, ParselCrawlingContext @@ -23,3 +25,7 @@ async def request_handler(context: ParselCrawlingContext) -> None: # Add first URL to the queue and start the crawl. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/quick-start/code_examples/playwright_crawler_example.py b/docs/quick-start/code_examples/playwright_crawler_example.py index fa6d8a6de4..1bc30ae320 100644 --- a/docs/quick-start/code_examples/playwright_crawler_example.py +++ b/docs/quick-start/code_examples/playwright_crawler_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext @@ -23,3 +25,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Add first URL to the queue and start the crawl. await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) From f2da9395cf8e09479301563c2d73e988e1221097 Mon Sep 17 00:00:00 2001 From: Vlada Dusek <v.dusek96@gmail.com> Date: Thu, 3 Apr 2025 11:54:42 +0200 Subject: [PATCH 15/16] add if main then asyncio run --- docs/guides/code_examples/request_loaders/tandem_example.py | 3 ++- .../code_examples/request_loaders/tandem_example_explicit.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/guides/code_examples/request_loaders/tandem_example.py b/docs/guides/code_examples/request_loaders/tandem_example.py index 0739d483ea..b0e83138ca 100644 --- a/docs/guides/code_examples/request_loaders/tandem_example.py +++ b/docs/guides/code_examples/request_loaders/tandem_example.py @@ -23,4 +23,5 @@ async def handler(context: ParselCrawlingContext) -> None: await crawler.run() -asyncio.run(main()) +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/request_loaders/tandem_example_explicit.py b/docs/guides/code_examples/request_loaders/tandem_example_explicit.py index ff34e1af5d..17ba20c392 100644 --- a/docs/guides/code_examples/request_loaders/tandem_example_explicit.py +++ b/docs/guides/code_examples/request_loaders/tandem_example_explicit.py @@ -26,4 +26,5 @@ async def handler(context: ParselCrawlingContext) -> None: await crawler.run() -asyncio.run(main()) +if __name__ == '__main__': + asyncio.run(main()) From c535bf270cca747ee2f7f75d992586e136a1714d Mon Sep 17 00:00:00 2001 From: Vlada Dusek <v.dusek96@gmail.com> Date: Thu, 3 Apr 2025 12:08:37 +0200 Subject: [PATCH 16/16] add missing asyncio runs --- .../apify/crawler_as_actor_example.py | 11 ++-- .../code_examples/apify/get_public_url.py | 6 +++ .../apify/log_with_config_example.py | 6 +++ .../apify/proxy_advanced_example.py | 6 +++ .../code_examples/apify/proxy_example.py | 6 +++ .../add_data_to_dataset_dataset.py | 6 +++ .../crawl_specific_links_on_website_bs.py | 5 +- .../fill_and_submit_web_form_request.py | 41 ++++++++------- ...default_fingerprint_generator_with_args.py | 18 +++++-- .../plugin_browser_configuration_example.py | 50 +++++++++++-------- .../playwright_crawler_adaptive/handler.py | 5 ++ .../init_beautifulsoup.py | 6 +++ .../init_parsel.py | 6 +++ .../init_prediction.py | 6 +++ .../pre_nav_hooks.py | 6 +++ .../proxy_management/session_bs_example.py | 8 +++ .../proxy_management/session_pw_example.py | 8 +++ docs/introduction/code_examples/02_bs.py | 5 +- .../code_examples/02_bs_better.py | 5 +- .../code_examples/03_enqueue_strategy.py | 11 ++-- docs/introduction/code_examples/03_globs.py | 11 ++-- .../code_examples/03_transform_request.py | 11 ++-- .../code_examples/07_first_code.py | 6 +++ .../playwright_crawler_headful_example.py | 6 +++ 24 files changed, 186 insertions(+), 69 deletions(-) diff --git a/docs/deployment/code_examples/apify/crawler_as_actor_example.py b/docs/deployment/code_examples/apify/crawler_as_actor_example.py index b986bf9129..53527d555b 100644 --- a/docs/deployment/code_examples/apify/crawler_as_actor_example.py +++ b/docs/deployment/code_examples/apify/crawler_as_actor_example.py @@ -1,9 +1,8 @@ +import asyncio + from apify import Actor -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -22,3 +21,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await context.enqueue_links() await crawler.run(['https://crawlee.dev']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/deployment/code_examples/apify/get_public_url.py b/docs/deployment/code_examples/apify/get_public_url.py index 061cbf4e97..d12cfba300 100644 --- a/docs/deployment/code_examples/apify/get_public_url.py +++ b/docs/deployment/code_examples/apify/get_public_url.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -8,3 +10,7 @@ async def main() -> None: url = store.get_public_url('your-file') Actor.log.info(f'KVS public URL: {url}') # https://api.apify.com/v2/key-value-stores/<your-store-id>/records/your-file + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/deployment/code_examples/apify/log_with_config_example.py b/docs/deployment/code_examples/apify/log_with_config_example.py index c1241fdc2e..dfefa7b5ae 100644 --- a/docs/deployment/code_examples/apify/log_with_config_example.py +++ b/docs/deployment/code_examples/apify/log_with_config_example.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor, Configuration @@ -11,3 +13,7 @@ async def main() -> None: async with Actor(config): Actor.log.info('Hello from Apify platform!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/deployment/code_examples/apify/proxy_advanced_example.py b/docs/deployment/code_examples/apify/proxy_advanced_example.py index 91064e2fb1..1b5306bd39 100644 --- a/docs/deployment/code_examples/apify/proxy_advanced_example.py +++ b/docs/deployment/code_examples/apify/proxy_advanced_example.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -12,3 +14,7 @@ async def main() -> None: ) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/deployment/code_examples/apify/proxy_example.py b/docs/deployment/code_examples/apify/proxy_example.py index 6535c2e672..d546c5cc45 100644 --- a/docs/deployment/code_examples/apify/proxy_example.py +++ b/docs/deployment/code_examples/apify/proxy_example.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -16,3 +18,7 @@ async def main() -> None: proxy_url = await proxy_configuration.new_url() Actor.log.info(f'Proxy URL: {proxy_url}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/add_data_to_dataset_dataset.py b/docs/examples/code_examples/add_data_to_dataset_dataset.py index 66234d2953..b1d9aba923 100644 --- a/docs/examples/code_examples/add_data_to_dataset_dataset.py +++ b/docs/examples/code_examples/add_data_to_dataset_dataset.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.storages import Dataset @@ -7,3 +9,7 @@ async def main() -> None: # Interact with dataset directly. await dataset.push_data({'key': 'value'}) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py b/docs/examples/code_examples/crawl_specific_links_on_website_bs.py index f0e77c4b5e..8dfc1bdf85 100644 --- a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py +++ b/docs/examples/code_examples/crawl_specific_links_on_website_bs.py @@ -1,10 +1,7 @@ import asyncio from crawlee import Glob -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: diff --git a/docs/examples/code_examples/fill_and_submit_web_form_request.py b/docs/examples/code_examples/fill_and_submit_web_form_request.py index 0c992f8294..14dc6c479d 100644 --- a/docs/examples/code_examples/fill_and_submit_web_form_request.py +++ b/docs/examples/code_examples/fill_and_submit_web_form_request.py @@ -1,21 +1,28 @@ +import asyncio from urllib.parse import urlencode from crawlee import Request -# Prepare a POST request to the form endpoint. -request = Request.from_url( - url='https://httpbin.org/post', - method='POST', - headers={'content-type': 'application/x-www-form-urlencoded'}, - payload=urlencode( - { - 'custname': 'John Doe', - 'custtel': '1234567890', - 'custemail': 'johndoe@example.com', - 'size': 'large', - 'topping': ['bacon', 'cheese', 'mushroom'], - 'delivery': '13:00', - 'comments': 'Please ring the doorbell upon arrival.', - } - ).encode(), -) + +async def main() -> None: + # Prepare a POST request to the form endpoint. + request = Request.from_url( + url='https://httpbin.org/post', + method='POST', + headers={'content-type': 'application/x-www-form-urlencoded'}, + payload=urlencode( + { + 'custname': 'John Doe', + 'custtel': '1234567890', + 'custemail': 'johndoe@example.com', + 'size': 'large', + 'topping': ['bacon', 'cheese', 'mushroom'], + 'delivery': '13:00', + 'comments': 'Please ring the doorbell upon arrival.', + } + ).encode(), + ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py b/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py index 14241e76ab..a6d2072ad3 100644 --- a/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py +++ b/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py @@ -1,10 +1,20 @@ +import asyncio + from crawlee.fingerprint_suite import ( DefaultFingerprintGenerator, HeaderGeneratorOptions, ScreenOptions, ) -fingerprint_generator = DefaultFingerprintGenerator( - header_options=HeaderGeneratorOptions(browsers=['chromium']), - screen_options=ScreenOptions(min_width=400), -) + +async def main() -> None: + fingerprint_generator = DefaultFingerprintGenerator( + header_options=HeaderGeneratorOptions(browsers=['chromium']), + screen_options=ScreenOptions(min_width=400), + ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py b/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py index eac1f8850a..6db2fb589d 100644 --- a/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py +++ b/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py @@ -1,25 +1,35 @@ +import asyncio + from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin from crawlee.crawlers import PlaywrightCrawler -crawler = PlaywrightCrawler( - browser_pool=BrowserPool( - plugins=[ - PlaywrightBrowserPlugin( - browser_type='chromium', - browser_launch_options={ - 'headless': False, - 'channel': 'msedge', - 'slow_mo': 200, - }, - browser_new_context_options={ - 'color_scheme': 'dark', - 'extra_http_headers': { - 'Custom-Header': 'my-header', - 'Accept-Language': 'en', + +async def main() -> None: + crawler = PlaywrightCrawler( + browser_pool=BrowserPool( + plugins=[ + PlaywrightBrowserPlugin( + browser_type='chromium', + browser_launch_options={ + 'headless': False, + 'channel': 'msedge', + 'slow_mo': 200, + }, + browser_new_context_options={ + 'color_scheme': 'dark', + 'extra_http_headers': { + 'Custom-Header': 'my-header', + 'Accept-Language': 'en', + }, + 'user_agent': 'My-User-Agent', }, - 'user_agent': 'My-User-Agent', - }, - ) - ] + ) + ] + ) ) -) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/handler.py b/docs/guides/code_examples/playwright_crawler_adaptive/handler.py index ed018ddc82..ad88e054cd 100644 --- a/docs/guides/code_examples/playwright_crawler_adaptive/handler.py +++ b/docs/guides/code_examples/playwright_crawler_adaptive/handler.py @@ -1,3 +1,4 @@ +import asyncio from datetime import timedelta from crawlee.crawlers import AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext @@ -14,3 +15,7 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: context.log.info(h2) await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py index 5dd2215493..c0008d3a29 100644 --- a/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import AdaptivePlaywrightCrawler @@ -13,3 +15,7 @@ async def main() -> None: ) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py index 102bc9f9c7..c220d53be4 100644 --- a/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import AdaptivePlaywrightCrawler @@ -13,3 +15,7 @@ async def main() -> None: ) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py b/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py index d7c2599de7..b07b1592ae 100644 --- a/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py +++ b/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee import Request from crawlee._types import RequestHandlerRunResult from crawlee.crawlers import ( @@ -62,3 +64,7 @@ async def main() -> None: ) # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py b/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py index 7d33750501..bd95bd9f8b 100644 --- a/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py +++ b/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py @@ -1,3 +1,5 @@ +import asyncio + from playwright.async_api import Route from crawlee.crawlers import ( @@ -31,3 +33,7 @@ async def some_routing_function(route: Route) -> None: ) await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/session_bs_example.py b/docs/guides/code_examples/proxy_management/session_bs_example.py index 2b0acd7f20..1243b0e488 100644 --- a/docs/guides/code_examples/proxy_management/session_bs_example.py +++ b/docs/guides/code_examples/proxy_management/session_bs_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import BeautifulSoupCrawler from crawlee.proxy_configuration import ProxyConfiguration @@ -14,3 +16,9 @@ async def main() -> None: proxy_configuration=proxy_configuration, use_session_pool=True, ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/proxy_management/session_pw_example.py b/docs/guides/code_examples/proxy_management/session_pw_example.py index de44b7911c..68309bda59 100644 --- a/docs/guides/code_examples/proxy_management/session_pw_example.py +++ b/docs/guides/code_examples/proxy_management/session_pw_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler from crawlee.proxy_configuration import ProxyConfiguration @@ -14,3 +16,9 @@ async def main() -> None: proxy_configuration=proxy_configuration, use_session_pool=True, ) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/02_bs.py b/docs/introduction/code_examples/02_bs.py index 87af780b8c..6e5ee30069 100644 --- a/docs/introduction/code_examples/02_bs.py +++ b/docs/introduction/code_examples/02_bs.py @@ -1,10 +1,7 @@ import asyncio # Add import of crawler and crawling context. -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext from crawlee.storages import RequestQueue diff --git a/docs/introduction/code_examples/02_bs_better.py b/docs/introduction/code_examples/02_bs_better.py index a027cbb343..1a985722b6 100644 --- a/docs/introduction/code_examples/02_bs_better.py +++ b/docs/introduction/code_examples/02_bs_better.py @@ -1,10 +1,7 @@ import asyncio # You don't need to import RequestQueue anymore. -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: diff --git a/docs/introduction/code_examples/03_enqueue_strategy.py b/docs/introduction/code_examples/03_enqueue_strategy.py index 6a7ecd9eaf..6aff8a1fba 100644 --- a/docs/introduction/code_examples/03_enqueue_strategy.py +++ b/docs/introduction/code_examples/03_enqueue_strategy.py @@ -1,7 +1,6 @@ -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -20,3 +19,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: ) await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/03_globs.py b/docs/introduction/code_examples/03_globs.py index caec75346b..c2f2627d95 100644 --- a/docs/introduction/code_examples/03_globs.py +++ b/docs/introduction/code_examples/03_globs.py @@ -1,8 +1,7 @@ +import asyncio + from crawlee import Glob -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext async def main() -> None: @@ -24,3 +23,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: ) await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/03_transform_request.py b/docs/introduction/code_examples/03_transform_request.py index 6f528a01c2..5f11a1cafa 100644 --- a/docs/introduction/code_examples/03_transform_request.py +++ b/docs/introduction/code_examples/03_transform_request.py @@ -1,10 +1,9 @@ from __future__ import annotations +import asyncio + from crawlee import HttpHeaders, RequestOptions, RequestTransformAction -from crawlee.crawlers import ( - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext def transform_request( @@ -44,3 +43,7 @@ async def blog_handler(context: BeautifulSoupCrawlingContext) -> None: context.log.info(f'Blog Processing {context.request.url}.') await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/introduction/code_examples/07_first_code.py b/docs/introduction/code_examples/07_first_code.py index d058692196..89de967684 100644 --- a/docs/introduction/code_examples/07_first_code.py +++ b/docs/introduction/code_examples/07_first_code.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext from crawlee.storages import Dataset @@ -14,3 +16,7 @@ async def main() -> None: async def request_handler(context: PlaywrightCrawlingContext) -> None: ... # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/quick-start/code_examples/playwright_crawler_headful_example.py b/docs/quick-start/code_examples/playwright_crawler_headful_example.py index f416f26277..403c665e51 100644 --- a/docs/quick-start/code_examples/playwright_crawler_headful_example.py +++ b/docs/quick-start/code_examples/playwright_crawler_headful_example.py @@ -1,3 +1,5 @@ +import asyncio + from crawlee.crawlers import PlaywrightCrawler @@ -11,3 +13,7 @@ async def main() -> None: ) # ... + + +if __name__ == '__main__': + asyncio.run(main())