From 96b75684c3115897ab3a7c0caf4f0b31aa9ca88d Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:27:46 +0530 Subject: [PATCH 1/3] fix: handle psutil.AccessDenied in get_memory_info On Linux, get_memory_info reads a process's PSS via memory_full_info, which psutil documents may require elevated privileges. In restricted environments (hardened containers with hidepid/restricted /proc, or an unreadable child subprocess) this raises psutil.AccessDenied, which is not a subclass of psutil.NoSuchProcess and so was not caught by the existing suppress around the child loop; the current-process read was unguarded entirely. The exception propagated out of the recurring system-info task, which has no error handling, silently stopping the autoscaler's CPU/memory snapshots. Suppress AccessDenied alongside NoSuchProcess when summing child memory, and fall back to RSS for the current process when PSS is denied. Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_utils/system.py | 15 +++++++---- tests/unit/_utils/test_system.py | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py index 56eeaadf24..757b19e062 100644 --- a/src/crawlee/_utils/system.py +++ b/src/crawlee/_utils/system.py @@ -122,14 +122,19 @@ def get_memory_info() -> MemoryInfo: logger.debug('Calling get_memory_info()...') current_process = psutil.Process(os.getpid()) - # Retrieve estimated memory usage of the current process. - current_size_bytes = _get_used_memory(current_process) + # Retrieve estimated memory usage of the current process. On Linux `_get_used_memory` reads PSS via + # `memory_full_info`, which can raise `AccessDenied` in restricted environments (e.g. hardened containers); + # fall back to RSS, which a process can always read for itself. + try: + current_size_bytes = _get_used_memory(current_process) + except psutil.AccessDenied: + current_size_bytes = int(current_process.memory_info().rss) # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS. for child in current_process.children(recursive=True): - # Ignore any NoSuchProcess exception that might occur if a child process ends before we retrieve - # its memory usage. - with suppress(psutil.NoSuchProcess): + # Ignore a child that ends before we retrieve its memory usage (`NoSuchProcess`) or that we are not + # allowed to inspect (`AccessDenied`, e.g. an unreadable subprocess in a restricted environment). + with suppress(psutil.NoSuchProcess, psutil.AccessDenied): current_size_bytes += _get_used_memory(child) vm = psutil.virtual_memory() diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py index 1813b151a6..9e094f355f 100644 --- a/tests/unit/_utils/test_system.py +++ b/tests/unit/_utils/test_system.py @@ -5,8 +5,10 @@ from multiprocessing.shared_memory import SharedMemory from typing import TYPE_CHECKING +import psutil import pytest +from crawlee._utils import system from crawlee._utils.byte_size import ByteSize from crawlee._utils.system import get_cpu_info, get_memory_info @@ -21,6 +23,49 @@ def test_get_memory_info_returns_valid_values() -> None: assert memory_info.current_size < memory_info.total_size +def test_get_memory_info_skips_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None: + """A child process we are not allowed to inspect must be skipped, not abort the whole snapshot. + + In restricted environments (e.g. hardened containers) reading a child's memory can raise + `psutil.AccessDenied`, which is not a subclass of `psutil.NoSuchProcess` and so was not suppressed. + """ + child = psutil.Process() # any process object works; only `_get_used_memory` behavior matters here + + monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [child]) + + def fake_get_used_memory(process: psutil.Process) -> int: + if process is child: + raise psutil.AccessDenied(pid=child.pid) + return 100 + + monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory) + + memory_info = get_memory_info() + + # The unreadable child is skipped, so only the current process (100) is counted. + assert memory_info.current_size == ByteSize(100) + + +def test_get_memory_info_falls_back_to_rss_when_current_process_access_denied( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """If PSS for the current process is denied, fall back to RSS instead of crashing. + + On Linux `_get_used_memory` reads PSS via `memory_full_info`, which may require elevated privileges. + """ + monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: []) + + def fake_get_used_memory(process: psutil.Process) -> int: + raise psutil.AccessDenied(pid=process.pid) + + monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory) + + memory_info = get_memory_info() + + # RSS of the current process is a positive value below total system memory. + assert ByteSize(0) < memory_info.current_size < memory_info.total_size + + def test_get_cpu_info_returns_valid_values() -> None: cpu_info = get_cpu_info() assert 0 <= cpu_info.used_ratio <= 1 From cd699de9a89d49da689e5a393e7c04416ad90725 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:30:30 +0530 Subject: [PATCH 2/3] fix(system): preserve child RSS when PSS is denied Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_utils/system.py | 14 +++++++++----- tests/unit/_utils/test_system.py | 20 +++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py index 757b19e062..139e1cea02 100644 --- a/src/crawlee/_utils/system.py +++ b/src/crawlee/_utils/system.py @@ -128,14 +128,18 @@ def get_memory_info() -> MemoryInfo: try: current_size_bytes = _get_used_memory(current_process) except psutil.AccessDenied: + logger.debug('PSS access denied for the current process, falling back to RSS.') current_size_bytes = int(current_process.memory_info().rss) # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS. - for child in current_process.children(recursive=True): - # Ignore a child that ends before we retrieve its memory usage (`NoSuchProcess`) or that we are not - # allowed to inspect (`AccessDenied`, e.g. an unreadable subprocess in a restricted environment). - with suppress(psutil.NoSuchProcess, psutil.AccessDenied): - current_size_bytes += _get_used_memory(child) + with suppress(psutil.NoSuchProcess, psutil.AccessDenied): + for child in current_process.children(recursive=True): + try: + current_size_bytes += _get_used_memory(child) + except psutil.AccessDenied: # noqa: PERF203 + logger.debug('PSS access denied for child process %s, falling back to RSS.', child.pid) + with suppress(psutil.NoSuchProcess, psutil.AccessDenied): + current_size_bytes += int(child.memory_info().rss) vm = psutil.virtual_memory() diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py index 9e094f355f..812d808db7 100644 --- a/tests/unit/_utils/test_system.py +++ b/tests/unit/_utils/test_system.py @@ -3,6 +3,7 @@ import sys from multiprocessing import get_context, synchronize from multiprocessing.shared_memory import SharedMemory +from types import SimpleNamespace from typing import TYPE_CHECKING import psutil @@ -23,15 +24,11 @@ def test_get_memory_info_returns_valid_values() -> None: assert memory_info.current_size < memory_info.total_size -def test_get_memory_info_skips_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None: - """A child process we are not allowed to inspect must be skipped, not abort the whole snapshot. - - In restricted environments (e.g. hardened containers) reading a child's memory can raise - `psutil.AccessDenied`, which is not a subclass of `psutil.NoSuchProcess` and so was not suppressed. - """ +def test_get_memory_info_falls_back_to_rss_for_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None: child = psutil.Process() # any process object works; only `_get_used_memory` behavior matters here monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [child]) + monkeypatch.setattr(child, 'memory_info', lambda: SimpleNamespace(rss=50)) def fake_get_used_memory(process: psutil.Process) -> int: if process is child: @@ -42,7 +39,16 @@ def fake_get_used_memory(process: psutil.Process) -> int: memory_info = get_memory_info() - # The unreadable child is skipped, so only the current process (100) is counted. + assert memory_info.current_size == ByteSize(150) + + +@pytest.mark.parametrize('error', [psutil.AccessDenied(), psutil.NoSuchProcess(pid=1)]) +def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.MonkeyPatch, error: psutil.Error) -> None: + monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: (_ for _ in ()).throw(error)) + monkeypatch.setattr(system, '_get_used_memory', lambda _process: 100) + + memory_info = get_memory_info() + assert memory_info.current_size == ByteSize(100) From 8d257e5bf0ceaa675ee419ba05f178d0e075a8bb Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:07:51 +0530 Subject: [PATCH 3/3] fix(system): continue after child process exits Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_utils/system.py | 30 +++++++++++++++--------------- tests/unit/_utils/test_system.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/crawlee/_utils/system.py b/src/crawlee/_utils/system.py index 139e1cea02..6b321bb536 100644 --- a/src/crawlee/_utils/system.py +++ b/src/crawlee/_utils/system.py @@ -122,24 +122,16 @@ def get_memory_info() -> MemoryInfo: logger.debug('Calling get_memory_info()...') current_process = psutil.Process(os.getpid()) - # Retrieve estimated memory usage of the current process. On Linux `_get_used_memory` reads PSS via - # `memory_full_info`, which can raise `AccessDenied` in restricted environments (e.g. hardened containers); - # fall back to RSS, which a process can always read for itself. - try: - current_size_bytes = _get_used_memory(current_process) - except psutil.AccessDenied: - logger.debug('PSS access denied for the current process, falling back to RSS.') - current_size_bytes = int(current_process.memory_info().rss) + current_size_bytes = _get_used_memory_with_rss_fallback(current_process) # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS. + children: list[psutil.Process] = [] with suppress(psutil.NoSuchProcess, psutil.AccessDenied): - for child in current_process.children(recursive=True): - try: - current_size_bytes += _get_used_memory(child) - except psutil.AccessDenied: # noqa: PERF203 - logger.debug('PSS access denied for child process %s, falling back to RSS.', child.pid) - with suppress(psutil.NoSuchProcess, psutil.AccessDenied): - current_size_bytes += int(child.memory_info().rss) + children = current_process.children(recursive=True) + + for child in children: + with suppress(psutil.NoSuchProcess, psutil.AccessDenied): + current_size_bytes += _get_used_memory_with_rss_fallback(child) vm = psutil.virtual_memory() @@ -148,3 +140,11 @@ def get_memory_info() -> MemoryInfo: current_size=ByteSize(current_size_bytes), system_wide_used_size=ByteSize(vm.total - vm.available), ) + + +def _get_used_memory_with_rss_fallback(process: psutil.Process) -> int: + try: + return _get_used_memory(process) + except psutil.AccessDenied: + logger.debug('PSS access denied for process %s, falling back to RSS.', process.pid) + return int(process.memory_info().rss) diff --git a/tests/unit/_utils/test_system.py b/tests/unit/_utils/test_system.py index 812d808db7..5b7c32d03d 100644 --- a/tests/unit/_utils/test_system.py +++ b/tests/unit/_utils/test_system.py @@ -25,7 +25,8 @@ def test_get_memory_info_returns_valid_values() -> None: def test_get_memory_info_falls_back_to_rss_for_children_with_access_denied(monkeypatch: pytest.MonkeyPatch) -> None: - child = psutil.Process() # any process object works; only `_get_used_memory` behavior matters here + """Test that denied child PSS reads fall back to RSS.""" + child = psutil.Process() monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [child]) monkeypatch.setattr(child, 'memory_info', lambda: SimpleNamespace(rss=50)) @@ -42,8 +43,15 @@ def fake_get_used_memory(process: psutil.Process) -> int: assert memory_info.current_size == ByteSize(150) -@pytest.mark.parametrize('error', [psutil.AccessDenied(), psutil.NoSuchProcess(pid=1)]) +@pytest.mark.parametrize( + 'error', + [ + pytest.param(psutil.AccessDenied(), id='access denied'), + pytest.param(psutil.NoSuchProcess(pid=1), id='no such process'), + ], +) def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.MonkeyPatch, error: psutil.Error) -> None: + """Test that failure to list children does not abort memory collection.""" monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: (_ for _ in ()).throw(error)) monkeypatch.setattr(system, '_get_used_memory', lambda _process: 100) @@ -52,6 +60,26 @@ def test_get_memory_info_handles_failure_to_list_children(monkeypatch: pytest.Mo assert memory_info.current_size == ByteSize(100) +def test_get_memory_info_continues_after_child_exits(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that a child exiting mid-iteration does not hide later children.""" + exited_child = psutil.Process() + live_child = psutil.Process() + monkeypatch.setattr(psutil.Process, 'children', lambda *_args, **_kwargs: [exited_child, live_child]) + + def fake_get_used_memory(process: psutil.Process) -> int: + if process is exited_child: + raise psutil.NoSuchProcess(pid=process.pid) + if process is live_child: + return 40 + return 100 + + monkeypatch.setattr(system, '_get_used_memory', fake_get_used_memory) + + memory_info = get_memory_info() + + assert memory_info.current_size == ByteSize(140) + + def test_get_memory_info_falls_back_to_rss_when_current_process_access_denied( monkeypatch: pytest.MonkeyPatch, ) -> None: