Updating request inside "default_handler", "error_handler" and "failed_request_handler"

[ericvg97]

Is there a way to update the request.user_data inside these handlers? I would like to store info about the last error, or the timestamp of the different retries.

This is my handler

async def default_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
        context.log.info(f"Processing {context.request.url} ...")
        context.request.user_data["something"] = "something"
        await context.enqueue_links(strategy="same-domain")

crawler.router.default_handler(default_handler)

but looking at the db this field is not being saved. I just see ""user_data":{"__crawlee":{"state":3}"

I guess the context is either not saved after that or it is sent as a copy, but not sure from looking at the code. Any other way of doing this that I am not seeing?

[Mantisus]

I checked it, and the code provided should be processed correctly.

import asyncio

from crawlee import ConcurrencySettings
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import HttpxHttpClient
from crawlee.storage_clients import SqlStorageClient


async def main() -> None:
    async with (
        SqlStorageClient(connection_string='postgresql+asyncpg://myuser:mypassword@localhost:5432/postgres') as storage_client
            ):
        http_client = HttpxHttpClient()

        crawler = ParselCrawler(
            storage_client=storage_client,
            http_client=http_client,
            concurrency_settings=ConcurrencySettings(desired_concurrency=20),
        )

        @crawler.router.default_handler
        async def request_handler(context: ParselCrawlingContext) -> None:
            context.log.info(f'Processing URL: {context.request.url}...')
            context.request.user_data['something'] = 'something'

        await crawler.run(['https://crawlee.dev'])


if __name__ == '__main__':
    asyncio.run(main())

Could you double-check this and let me know if the data is not being updated?

[ericvg97]

The example you sahred works, but just changing ParselCrawler to AdaptativePlaywrightCrawler breaks it. This is the new code (that doesn't work)

async def main() -> None:
    async with SqlStorageClient(
        connection_string="postgresql+asyncpg://postgres:password@localhost:5432/lighthouse"
    ) as storage_client:
        http_client = HttpxHttpClient()

        crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
            max_requests_per_crawl=10000,
            playwright_crawler_specific_kwargs={
                "browser_type": "chromium",
                "headless": True,
            },
            max_session_rotations=10,
            retry_on_blocked=True,
            configure_logging=True,
            use_session_pool=True,
            max_request_retries=5,
            keep_alive=False,
        )

        @crawler.router.default_handler
        async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
            context.log.info(f"Processing URL: {context.request.url}...")
            context.request.user_data["something"] = "something"

        await crawler.run(["https://crawlee.dev"])


if __name__ == "__main__":
    asyncio.run(main())

[Mantisus]

Thanks for the details!