diff --git a/docs/examples/code_examples/resuming_paused_crawl.py b/docs/examples/code_examples/resuming_paused_crawl.py new file mode 100644 index 0000000000..e87e428469 --- /dev/null +++ b/docs/examples/code_examples/resuming_paused_crawl.py @@ -0,0 +1,40 @@ +import asyncio + +from crawlee import ConcurrencySettings, service_locator +from crawlee.crawlers import ( + BeautifulSoupCrawler, + BeautifulSoupCrawlingContext, +) + +# Disable clearing the `RequestQueue`, `KeyValueStore` and `Dataset` on each run. +# This makes the scraper continue from where it left off in the previous run. +# The recommended way to achieve this behavior is setting the environment variable +# `CRAWLEE_PURGE_ON_START=0` +configuration = service_locator.get_configuration() +configuration.purge_on_start = False + + +async def main() -> None: + crawler = BeautifulSoupCrawler( + # Let's slow down the crawler for a demonstration + concurrency_settings=ConcurrencySettings(max_tasks_per_minute=20) + ) + + @crawler.router.default_handler + async def request_handler(context: BeautifulSoupCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + + # List of links for crawl + requests = [ + 'https://crawlee.dev', + 'https://crawlee.dev/python/docs', + 'https://crawlee.dev/python/docs/examples', + 'https://crawlee.dev/python/docs/guides', + 'https://crawlee.dev/python/docs/quick-start', + ] + + await crawler.run(requests) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/resuming_paused_crawl.mdx b/docs/examples/resuming_paused_crawl.mdx new file mode 100644 index 0000000000..8d2213d11d --- /dev/null +++ b/docs/examples/resuming_paused_crawl.mdx @@ -0,0 +1,35 @@ +--- +id: resuming-paused-crawl +title: Resuming a paused crawl +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ResumeCrawl from '!!raw-loader!roa-loader!./code_examples/resuming_paused_crawl.py'; + +This example demonstrates how to resume crawling from its last state when running locally, if for some reason it was unexpectedly terminated. + +If each run should continue crawling from the previous state, you can configure this using `purge_on_start` in `Configuration`. + +Use the code below and perform 2 sequential runs. During the 1st run, stop the crawler by pressing `CTRL+C`, and the 2nd run will resume crawling from where it stopped. + + + {ResumeCrawl} + + +Perform the 1st run, interrupting the crawler with `CTRL+C` after 2 links have been processed. + +![Run with interruption](/img/resuming-paused-crawl/00.webp 'Run with interruption.') + +Now resume crawling after the pause to process the remaining 3 links. + +![Resuming crawling](/img/resuming-paused-crawl/01.webp 'Resuming crawling.') + +Alternatively, use the environment variable `CRAWLEE_PURGE_ON_START=0` instead of using `configuration.purge_on_start = False`. + +For example, when running code: + +```bash +CRAWLEE_PURGE_ON_START=0 python -m best_crawler +``` diff --git a/uv.lock b/uv.lock index 39af09df40..5041bbf443 100644 --- a/uv.lock +++ b/uv.lock @@ -600,7 +600,7 @@ toml = [ [[package]] name = "crawlee" -version = "0.6.6" +version = "0.6.7" source = { editable = "." } dependencies = [ { name = "apify-fingerprint-datapoints" }, @@ -1149,7 +1149,7 @@ name = "importlib-metadata" version = "8.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "python_full_version < '3.11'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/08/c1395a292bb23fd03bdf572a1357c5a733d3eecbab877641ceacab23db6e/importlib_metadata-8.6.1.tar.gz", hash = "sha256:310b41d755445d74569f993ccfc22838295d9fe005425094fad953d7f15c8580", size = 55767 } wheels = [ diff --git a/website/static/img/resuming-paused-crawl/00.webp b/website/static/img/resuming-paused-crawl/00.webp new file mode 100644 index 0000000000..871a01d15f Binary files /dev/null and b/website/static/img/resuming-paused-crawl/00.webp differ diff --git a/website/static/img/resuming-paused-crawl/01.webp b/website/static/img/resuming-paused-crawl/01.webp new file mode 100644 index 0000000000..e4ace7b66e Binary files /dev/null and b/website/static/img/resuming-paused-crawl/01.webp differ