From 13e62e446b7be9f770ab1d06ebb7817cd3f5d5aa Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 15 Oct 2025 09:59:46 +0000 Subject: [PATCH 1/4] save context state in result after isolated processing in `SubCrawler` --- .../_adaptive_playwright_crawler.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index 055e2731d9..fbe10438d9 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -319,7 +319,7 @@ async def get_input_state( ), logger=self._logger, ) - return SubCrawlerRun(result=result) + return SubCrawlerRun(result=result, run_context=context_linked_to_result) except Exception as e: return SubCrawlerRun(exception=e) @@ -375,7 +375,8 @@ async def _run_request_handler(self, context: BasicCrawlingContext) -> None: self.track_http_only_request_handler_runs() static_run = await self._crawl_one(rendering_type='static', context=context) - if static_run.result and self.result_checker(static_run.result): + if static_run.result and static_run.run_context and self.result_checker(static_run.result): + self._update_context_from_copy(context, static_run.run_context) self._context_result_map[context] = static_run.result return if static_run.exception: @@ -406,13 +407,10 @@ async def _run_request_handler(self, context: BasicCrawlingContext) -> None: if pw_run.exception is not None: raise pw_run.exception - if pw_run.result: - self._context_result_map[context] = pw_run.result - + if pw_run.result and pw_run.run_context: if should_detect_rendering_type: detection_result: RenderingType static_run = await self._crawl_one('static', context=context, state=old_state_copy) - if static_run.result and self.result_comparator(static_run.result, pw_run.result): detection_result = 'static' else: @@ -421,6 +419,9 @@ async def _run_request_handler(self, context: BasicCrawlingContext) -> None: context.log.debug(f'Detected rendering type {detection_result} for {context.request.url}') self.rendering_type_predictor.store_result(context.request, detection_result) + self._update_context_from_copy(context, pw_run.run_context) + self._context_result_map[context] = pw_run.result + def pre_navigation_hook( self, hook: Callable[[AdaptivePlaywrightPreNavCrawlingContext], Awaitable[None]] | None = None, @@ -455,8 +456,19 @@ def track_browser_request_handler_runs(self) -> None: def track_rendering_type_mispredictions(self) -> None: self.statistics.state.rendering_type_mispredictions += 1 + def _update_context_from_copy(self, context: BasicCrawlingContext, context_copy: BasicCrawlingContext) -> None: + """Update mutable fields of `context` from `context_copy`. + + Uses object.__setattr__ to bypass frozen dataclass restrictions, + allowing state synchronization after isolated crawler execution. + """ + object.__setattr__(context, 'request', context_copy.request) + object.__setattr__(context, 'session', context_copy.session) + object.__setattr__(context, 'proxy_info', context_copy.proxy_info) + @dataclass(frozen=True) class SubCrawlerRun: result: RequestHandlerRunResult | None = None exception: Exception | None = None + run_context: BasicCrawlingContext | None = None From ead964daba7e31f4900a35366dce6858fcfa1c3b Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Fri, 17 Oct 2025 00:53:48 +0000 Subject: [PATCH 2/4] add test --- .../_adaptive_playwright_crawler.py | 21 ++++++++-- .../test_adaptive_playwright_crawler.py | 40 ++++++++++++++++++- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index fbe10438d9..06a8f14fd5 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -462,9 +462,24 @@ def _update_context_from_copy(self, context: BasicCrawlingContext, context_copy: Uses object.__setattr__ to bypass frozen dataclass restrictions, allowing state synchronization after isolated crawler execution. """ - object.__setattr__(context, 'request', context_copy.request) - object.__setattr__(context, 'session', context_copy.session) - object.__setattr__(context, 'proxy_info', context_copy.proxy_info) + updating_attributes = ('request', 'session', 'proxy_info') + + for attr_name in updating_attributes: + original_obj = getattr(context, attr_name, None) + updated_obj = getattr(context_copy, attr_name, None) + + # Skip if either object is None or they are the same object + if ( + original_obj is None + or updated_obj is None + or original_obj is updated_obj + or not hasattr(updated_obj, '__dict__') + ): + continue + + # Copy all attributes using __dict__ + for obj_attr_name, obj_attr_value in updated_obj.__dict__.items(): + object.__setattr__(original_obj, obj_attr_name, obj_attr_value) @dataclass(frozen=True) diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 5c21328860..085735266c 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -29,8 +29,9 @@ from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawling_context import ( AdaptiveContextError, ) +from crawlee.sessions import SessionPool from crawlee.statistics import Statistics -from crawlee.storages import KeyValueStore +from crawlee.storages import KeyValueStore, RequestQueue if TYPE_CHECKING: from collections.abc import AsyncGenerator, Iterator @@ -727,3 +728,40 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: await crawler.run(test_urls[:1]) mocked_h3_handler.assert_called_once_with(None) + + +async def test_change_context_state_after_handling(server_url: URL) -> None: + """Test that context state is saved after handling the request.""" + static_only_predictor_no_detection = _SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0])) + + test_mapper = {} + request_queue = await RequestQueue.open(name='state-test') + async with SessionPool() as session_pool: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=static_only_predictor_no_detection, + session_pool=session_pool, + request_manager=request_queue, + ) + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + if context.session is not None: + test_mapper['session_id'] = context.session.id + context.session.user_data['session_state'] = True + context.request.user_data['request_state'] = True + + request = Request.from_url(str(server_url)) + + await crawler.run([request]) + + assert test_mapper['session_id'] is not None + + session = await session_pool.get_session_by_id(test_mapper['session_id']) + check_request = await request_queue.get_request(request.unique_key) + + assert session is not None + assert check_request is not None + assert session.user_data.get('session_state') is True + assert check_request.user_data.get('request_state') is True + + await request_queue.drop() From 2ecfd6c8301a356cb7862cba3bc0bedbff60c354 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 23 Oct 2025 00:33:46 +0000 Subject: [PATCH 3/4] up tests --- .../_adaptive_playwright_crawler.py | 2 +- .../test_adaptive_playwright_crawler.py | 61 ++++++++++++++++--- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index 06a8f14fd5..ff841c2aa0 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -462,7 +462,7 @@ def _update_context_from_copy(self, context: BasicCrawlingContext, context_copy: Uses object.__setattr__ to bypass frozen dataclass restrictions, allowing state synchronization after isolated crawler execution. """ - updating_attributes = ('request', 'session', 'proxy_info') + updating_attributes = ('request', 'session') for attr_name in updating_attributes: original_obj = getattr(context, attr_name, None) diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 085735266c..6fb34ab1c0 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -730,38 +730,79 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: mocked_h3_handler.assert_called_once_with(None) -async def test_change_context_state_after_handling(server_url: URL) -> None: +@pytest.mark.parametrize( + 'test_input', + [ + pytest.param( + TestInput( + expected_pw_count=0, + expected_static_count=2, + rendering_types=cycle(['static']), + detection_probability_recommendation=cycle([0]), + ), + id='Static only', + ), + pytest.param( + TestInput( + expected_pw_count=2, + expected_static_count=0, + rendering_types=cycle(['client only']), + detection_probability_recommendation=cycle([0]), + ), + id='Client only', + ), + pytest.param( + TestInput( + expected_pw_count=2, + expected_static_count=2, + rendering_types=cycle(['static', 'client only']), + detection_probability_recommendation=cycle([1]), + ), + id='Enforced rendering type detection', + ), + ], +) +async def test_change_context_state_after_handling(test_input: TestInput, server_url: URL) -> None: """Test that context state is saved after handling the request.""" - static_only_predictor_no_detection = _SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0])) + predictor = _SimpleRenderingTypePredictor( + rendering_types=test_input.rendering_types, + detection_probability_recommendation=test_input.detection_probability_recommendation, + ) - test_mapper = {} request_queue = await RequestQueue.open(name='state-test') + used_session_id = None + async with SessionPool() as session_pool: crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( - rendering_type_predictor=static_only_predictor_no_detection, + rendering_type_predictor=predictor, session_pool=session_pool, request_manager=request_queue, ) @crawler.router.default_handler async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + nonlocal used_session_id + if context.session is not None: - test_mapper['session_id'] = context.session.id + used_session_id = context.session.id context.session.user_data['session_state'] = True - context.request.user_data['request_state'] = True - request = Request.from_url(str(server_url)) + if isinstance(context.request.user_data['request_state'], list): + context.request.user_data['request_state'].append('handler') + + request = Request.from_url(str(server_url), user_data={'request_state': ['initial']}) await crawler.run([request]) - assert test_mapper['session_id'] is not None + assert used_session_id is not None - session = await session_pool.get_session_by_id(test_mapper['session_id']) + session = await session_pool.get_session_by_id(used_session_id) check_request = await request_queue.get_request(request.unique_key) assert session is not None assert check_request is not None assert session.user_data.get('session_state') is True - assert check_request.user_data.get('request_state') is True + # Check that request user data was updated in the handler and only onse. + assert check_request.user_data.get('request_state') == ['initial', 'handler'] await request_queue.drop() From 4b4d6e820ec2e4e0d2c32c0536e392847f4c5617 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 23 Oct 2025 17:28:21 +0000 Subject: [PATCH 4/4] sync only specific attributes --- .../_adaptive_playwright_crawler.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index d8b08604e7..a3139d3da1 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -458,24 +458,22 @@ def _update_context_from_copy(self, context: BasicCrawlingContext, context_copy: Uses object.__setattr__ to bypass frozen dataclass restrictions, allowing state synchronization after isolated crawler execution. """ - updating_attributes = ('request', 'session') - - for attr_name in updating_attributes: - original_obj = getattr(context, attr_name, None) - updated_obj = getattr(context_copy, attr_name, None) - - # Skip if either object is None or they are the same object - if ( - original_obj is None - or updated_obj is None - or original_obj is updated_obj - or not hasattr(updated_obj, '__dict__') - ): + updating_attributes = { + 'request': ('headers', 'user_data'), + 'session': ('_user_data', '_usage_count', '_error_score', '_cookies'), + } + + for attr, sub_attrs in updating_attributes.items(): + original_sub_obj = getattr(context, attr) + copy_sub_obj = getattr(context_copy, attr) + + # Check that both sub objects are not None + if original_sub_obj is None or copy_sub_obj is None: continue - # Copy all attributes using __dict__ - for obj_attr_name, obj_attr_value in updated_obj.__dict__.items(): - object.__setattr__(original_obj, obj_attr_name, obj_attr_value) + for sub_attr in sub_attrs: + new_value = getattr(copy_sub_obj, sub_attr) + object.__setattr__(original_sub_obj, sub_attr, new_value) @dataclass(frozen=True)