docs: Add "Run on Apify" button

CONTRIBUTING.md

@@ -82,43 +82,11 @@ TODO:
 
 ## Documentation
 
-We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/).
+We follow the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for code documentation. All user-facing classes and functions must be documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/).
 
-Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages.
+Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with custom post-processing. Additional content is provided through markdown files in the `docs/` directory. The final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub pages.
 
-To run the documentation locally, you need to have Node.js version 20 or higher installed. Once you have the correct version of Node.js, follow these steps:
-
-Navigate to the `website/` directory:
-
-```sh
-cd website/
-```
-
-Enable Corepack, which installs Yarn automatically:
-
-```sh
-corepack enable
-```
-
-Build the API reference:
-
-```sh
-./build_api_reference.sh
-```
-
-Install the necessary dependencies:
-
-```sh
-yarn
-```
-
-Start the project in development mode with Hot Module Replacement (HMR):
-
-```sh
-yarn start
-```
-
-Or using `make`:
+To run the documentation locally, ensure you have `Node.js` 20+ installed, then run:
 
 ```sh
 make run-docs

docs/deployment/code_examples/apify/crawler_as_actor_example.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from apify import Actor
 
 from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
@@ -6,7 +8,7 @@
 async def main() -> None:
     # Wrap the crawler code in an Actor context manager.
     async with Actor:
-        crawler = BeautifulSoupCrawler(max_requests_per_crawl=50)
+        crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
 
         @crawler.router.default_handler
         async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
@@ -19,3 +21,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
             await context.enqueue_links()
 
         await crawler.run(['https://crawlee.dev'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/deployment/code_examples/apify/get_public_url.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from apify import Actor
 
 
@@ -8,3 +10,7 @@ async def main() -> None:
         url = store.get_public_url('your-file')
         Actor.log.info(f'KVS public URL: {url}')
         # https://api.apify.com/v2/key-value-stores/<your-store-id>/records/your-file
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/deployment/code_examples/apify/log_with_config_example.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from apify import Actor, Configuration
 
 
@@ -11,3 +13,7 @@ async def main() -> None:
 
     async with Actor(config):
         Actor.log.info('Hello from Apify platform!')
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/deployment/code_examples/apify/proxy_advanced_example.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from apify import Actor
 
 
@@ -12,3 +14,7 @@ async def main() -> None:
         )
 
         # ...
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/deployment/code_examples/apify/proxy_example.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from apify import Actor
 
 
@@ -16,3 +18,7 @@ async def main() -> None:
 
         proxy_url = await proxy_configuration.new_url()
         Actor.log.info(f'Proxy URL: {proxy_url}')
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/examples/add_data_to_dataset.mdx

@@ -6,24 +6,24 @@ title: Add data to dataset
 import ApiLink from '@site/src/components/ApiLink';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
-import CodeBlock from '@theme/CodeBlock';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
 
-import BeautifulSoupExample from '!!raw-loader!./code_examples/add_data_to_dataset_bs.py';
-import PlaywrightExample from '!!raw-loader!./code_examples/add_data_to_dataset_pw.py';
-import DatasetExample from '!!raw-loader!./code_examples/add_data_to_dataset_dataset.py';
+import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_bs.py';
+import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_pw.py';
+import DatasetExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_dataset.py';
 
 This example demonstrates how to store extracted data into datasets using the <ApiLink to="class/PushDataFunction#open">`context.push_data`</ApiLink> helper function. If the specified dataset does not already exist, it will be created automatically. Additionally, you can save data to custom datasets by providing `dataset_id` or `dataset_name` parameters to the <ApiLink to="class/PushDataFunction#open">`push_data`</ApiLink> function.
 
 <Tabs groupId="main">
     <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
-        <CodeBlock className="language-python">
+        <RunnableCodeBlock className="language-python" language="python">
             {BeautifulSoupExample}
-        </CodeBlock>
+        </RunnableCodeBlock>
     </TabItem>
     <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
-        <CodeBlock className="language-python">
+        <RunnableCodeBlock className="language-python" language="python">
             {PlaywrightExample}
-        </CodeBlock>
+        </RunnableCodeBlock>
     </TabItem>
 </Tabs>
 
@@ -35,6 +35,6 @@ Each item in the dataset will be stored in its own file within the following dir
 
 For more control, you can also open a dataset manually using the asynchronous constructor <ApiLink to="class/Dataset#open">`Dataset.open`</ApiLink>
 
-<CodeBlock className="language-python">
+<RunnableCodeBlock className="language-python" language="python">
     {DatasetExample}
-</CodeBlock>
+</RunnableCodeBlock>

docs/examples/beautifulsoup_crawler.mdx

@@ -4,12 +4,12 @@ title: BeautifulSoup crawler
 ---
 
 import ApiLink from '@site/src/components/ApiLink';
-import CodeBlock from '@theme/CodeBlock';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
 
-import BeautifulSoupExample from '!!raw-loader!./code_examples/beautifulsoup_crawler.py';
+import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler.py';
 
 This example demonstrates how to use <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> to crawl a list of URLs, load each URL using a plain HTTP request, parse the HTML using the [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) library and extract some data from it - the page title and all `<h1>`, `<h2>` and `<h3>` tags. This setup is perfect for scraping specific elements from web pages. Thanks to the well-known BeautifulSoup, you can easily navigate the HTML structure and retrieve the data you need with minimal code. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request.
 
-<CodeBlock className="language-python">
+<RunnableCodeBlock className="language-python" language="python">
     {BeautifulSoupExample}
-</CodeBlock>
+</RunnableCodeBlock>

docs/examples/capture_screenshot_using_playwright.mdx

@@ -4,16 +4,16 @@ title: Capture screenshots using Playwright
 ---
 
 import ApiLink from '@site/src/components/ApiLink';
-import CodeBlock from '@theme/CodeBlock';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
 
-import CaptureScreenshotExample from '!!raw-loader!./code_examples/capture_screenshot_using_playwright.py';
+import CaptureScreenshotExample from '!!raw-loader!roa-loader!./code_examples/capture_screenshot_using_playwright.py';
 
 This example demonstrates how to capture screenshots of web pages using <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and store them in the key-value store.
 
 The <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> is configured to automate the browsing and interaction with web pages. It uses headless Chromium as the browser type to perform these tasks. Each web page specified in the initial list of URLs is visited sequentially, and a screenshot of the page is captured using Playwright's `page.screenshot()` method.
 
 The captured screenshots are stored in the key-value store, which is suitable for managing and storing files in various formats. In this case, screenshots are stored as PNG images with a unique key generated from the URL of the page.
 
-<CodeBlock className="language-python">
+<RunnableCodeBlock className="language-python" language="python">
     {CaptureScreenshotExample}
-</CodeBlock>
+</RunnableCodeBlock>

docs/examples/code_examples/adaptive_playwright_crawler.py

@@ -14,7 +14,7 @@ async def main() -> None:
     # Crawler created by following factory method will use `beautifulsoup`
     # for parsing static content.
     crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
-        max_requests_per_crawl=5, playwright_crawler_specific_kwargs={'headless': False}
+        max_requests_per_crawl=10, playwright_crawler_specific_kwargs={'headless': False}
     )
 
     @crawler.router.default_handler

docs/examples/code_examples/add_data_to_dataset_dataset.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from crawlee.storages import Dataset
 
 
@@ -7,3 +9,7 @@ async def main() -> None:
 
     # Interact with dataset directly.
     await dataset.push_data({'key': 'value'})
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/examples/code_examples/fill_and_submit_web_form_request.py

@@ -1,21 +1,28 @@
+import asyncio
 from urllib.parse import urlencode
 
 from crawlee import Request
 
-# Prepare a POST request to the form endpoint.
-request = Request.from_url(
-    url='https://httpbin.org/post',
-    method='POST',
-    headers={'content-type': 'application/x-www-form-urlencoded'},
-    payload=urlencode(
-        {
-            'custname': 'John Doe',
-            'custtel': '1234567890',
-            'custemail': 'johndoe@example.com',
-            'size': 'large',
-            'topping': ['bacon', 'cheese', 'mushroom'],
-            'delivery': '13:00',
-            'comments': 'Please ring the doorbell upon arrival.',
-        }
-    ).encode(),
-)
+
+async def main() -> None:
+    # Prepare a POST request to the form endpoint.
+    request = Request.from_url(
+        url='https://httpbin.org/post',
+        method='POST',
+        headers={'content-type': 'application/x-www-form-urlencoded'},
+        payload=urlencode(
+            {
+                'custname': 'John Doe',
+                'custtel': '1234567890',
+                'custemail': 'johndoe@example.com',
+                'size': 'large',
+                'topping': ['bacon', 'cheese', 'mushroom'],
+                'delivery': '13:00',
+                'comments': 'Please ring the doorbell upon arrival.',
+            }
+        ).encode(),
+    )
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/examples/crawl_all_links_on_website.mdx

@@ -6,10 +6,10 @@ title: Crawl all links on website
 import ApiLink from '@site/src/components/ApiLink';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
-import CodeBlock from '@theme/CodeBlock';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
 
-import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_all_links_on_website_bs.py';
-import PlaywrightExample from '!!raw-loader!./code_examples/crawl_all_links_on_website_pw.py';
+import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_bs.py';
+import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_pw.py';
 
 This example uses the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> helper to add new links to the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink> as the crawler navigates from page to page. By automatically discovering and enqueuing all links on a given page, the crawler can systematically scrape an entire website. This approach is ideal for web scraping tasks where you need to collect data from multiple interconnected pages.
 
@@ -21,13 +21,13 @@ If no options are given, by default the method will only add links that are unde
 
 <Tabs groupId="main">
     <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
-        <CodeBlock className="language-python">
+        <RunnableCodeBlock className="language-python" language="python">
             {BeautifulSoupExample}
-        </CodeBlock>
+        </RunnableCodeBlock>
     </TabItem>
     <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
-        <CodeBlock className="language-python">
+        <RunnableCodeBlock className="language-python" language="python">
             {PlaywrightExample}
-        </CodeBlock>
+        </RunnableCodeBlock>
     </TabItem>
 </Tabs>

docs/examples/crawl_multiple_urls.mdx

@@ -6,22 +6,22 @@ title: Crawl multiple URLs
 import ApiLink from '@site/src/components/ApiLink';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
-import CodeBlock from '@theme/CodeBlock';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
 
-import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_multiple_urls_bs.py';
-import PlaywrightExample from '!!raw-loader!./code_examples/crawl_multiple_urls_pw.py';
+import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_bs.py';
+import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_pw.py';
 
 This example demonstrates how to crawl a specified list of URLs using different crawlers. You'll learn how to set up the crawler, define a request handler, and run the crawler with multiple URLs. This setup is useful for scraping data from multiple pages or websites concurrently.
 
 <Tabs groupId="main">
     <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
-        <CodeBlock className="language-python">
+        <RunnableCodeBlock className="language-python" language="python">
             {BeautifulSoupExample}
-        </CodeBlock>
+        </RunnableCodeBlock>
     </TabItem>
     <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
-        <CodeBlock className="language-python">
+        <RunnableCodeBlock className="language-python" language="python">
             {PlaywrightExample}
-        </CodeBlock>
+        </RunnableCodeBlock>
     </TabItem>
 </Tabs>

docs/examples/crawl_specific_links_on_website.mdx

@@ -6,22 +6,22 @@ title: Crawl specific links on website
 import ApiLink from '@site/src/components/ApiLink';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
-import CodeBlock from '@theme/CodeBlock';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
 
-import BeautifulSoupExample from '!!raw-loader!./code_examples/crawl_specific_links_on_website_bs.py';
-import PlaywrightExample from '!!raw-loader!./code_examples/crawl_specific_links_on_website_pw.py';
+import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_bs.py';
+import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_pw.py';
 
 This example demonstrates how to crawl a website while targeting specific patterns of links. By utilizing the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> helper, you can pass `include` or `exclude` parameters to improve your crawling strategy. This approach ensures that only the links matching the specified patterns are added to the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>. Both `include` and `exclude` support lists of globs or regular expressions. This functionality is great for focusing on relevant sections of a website and avoiding scraping unnecessary or irrelevant content.
 
 <Tabs groupId="main">
     <TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
-        <CodeBlock className="language-python">
+        <RunnableCodeBlock className="language-python" language="python">
             {BeautifulSoupExample}
-        </CodeBlock>
+        </RunnableCodeBlock>
     </TabItem>
     <TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
-        <CodeBlock className="language-python">
+        <RunnableCodeBlock className="language-python" language="python">
             {PlaywrightExample}
-        </CodeBlock>
+        </RunnableCodeBlock>
     </TabItem>
 </Tabs>