From e265d92acfb36a5a0fa1b410792738ad90900b2e Mon Sep 17 00:00:00 2001 From: James Nightingale Date: Sun, 23 Aug 2026 21:32:06 +0000 Subject: [PATCH] Dump the compile traceback before the runner kills the process The watchdog shipped in #1517 never actually dumped. Its CI default was a flat 300s and the workspace smoke cap is also 300s, so the dump timer and the runner's kill timer raced and the kill won -- a SIGKILLed process writes no traceback. Measured: 20 stalled runs across four CI legs produced heartbeats from every one of them and not a single stack (autolens_workspace_test#271). Derive the CI default from BUILD_SCRIPT_TIMEOUT -- the per-script cap the workspace runners and PyAutoHands both enforce -- at 80% of it, so the traceback has time to reach stderr before the kill lands. 300s cap dumps at 240s; 1800s cap dumps at 1440s. With no usable cap advertised, fall back to a flat 240s rather than computing a fraction of zero, which would silently disable the dump. An explicit PYAUTOFIT_JAX_COMPILE_DUMP_SECS still wins. Four new tests, one of them pinning the invariant directly: the threshold is strictly below the cap for every cap. Verified end-to-end: with a 10s cap the dump fires at 8s and the traceback is captured, where before nothing was written at all. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015qk7hoavMnFyPtW4toYn8K --- autofit/non_linear/jax_compile.py | 29 ++++++++++++--- test_autofit/non_linear/test_jax_compile.py | 40 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/autofit/non_linear/jax_compile.py b/autofit/non_linear/jax_compile.py index f57fe1c2f..83e63c1e3 100644 --- a/autofit/non_linear/jax_compile.py +++ b/autofit/non_linear/jax_compile.py @@ -9,10 +9,15 @@ # How often a compile that is still running reports that it is still alive. DEFAULT_HEARTBEAT_SECONDS = 30.0 -# How long a first compile may run under CI before it dumps its own traceback. -# Off by default off-CI: an interactive user watching a slow compile does not -# want a traceback on stderr, they want the heartbeat above. -DEFAULT_CI_DUMP_SECONDS = 300.0 +# How long a first compile may run under CI before it dumps its own traceback, +# when the runner has not told us its own cap. Off by default off-CI: an +# interactive user watching a slow compile does not want a traceback on stderr, +# they want the heartbeat above. +DEFAULT_CI_DUMP_SECONDS = 240.0 + +# Fraction of the runner's per-script cap at which to dump. The dump is only +# ever useful STRICTLY BEFORE the kill -- see `dump_traceback_seconds`. +DUMP_FRACTION_OF_CAP = 0.8 def _env_seconds(name, default): @@ -60,8 +65,22 @@ def dump_traceback_seconds(): runner -- the alternative, threading an environment variable through each workspace's `config/build/env_vars_*.yaml`, is more repos touched for the same effect, and the workspace scripts are user-facing documentation. + + Under CI the default is derived from `BUILD_SCRIPT_TIMEOUT`, the per-script + cap the workspace runners and PyAutoHands both enforce. **A dump scheduled + at or after that cap never happens**: the runner SIGKILLs the process group + on expiry, and a killed process writes no traceback. The first CI use of + this watchdog hit exactly that -- a flat 300s default against a 300s smoke + cap produced heartbeats from 20 stalled runs and not one stack + (autolens_workspace_test#271). Dumping at a fraction of the cap leaves the + traceback time to reach stderr before the kill lands. """ - default = DEFAULT_CI_DUMP_SECONDS if os.environ.get("CI") else 0.0 + if not os.environ.get("CI"): + default = 0.0 + else: + cap = _env_seconds("BUILD_SCRIPT_TIMEOUT", 0.0) + default = cap * DUMP_FRACTION_OF_CAP if cap > 0 else DEFAULT_CI_DUMP_SECONDS + return _env_seconds("PYAUTOFIT_JAX_COMPILE_DUMP_SECS", default) diff --git a/test_autofit/non_linear/test_jax_compile.py b/test_autofit/non_linear/test_jax_compile.py index e3981f300..0bbd95eef 100644 --- a/test_autofit/non_linear/test_jax_compile.py +++ b/test_autofit/non_linear/test_jax_compile.py @@ -256,6 +256,7 @@ def test_the_heartbeat_interval_comes_from_the_environment(monkeypatch): def test_the_traceback_dump_defaults_on_under_ci_and_off_elsewhere(monkeypatch): monkeypatch.delenv("PYAUTOFIT_JAX_COMPILE_DUMP_SECS", raising=False) + monkeypatch.delenv("BUILD_SCRIPT_TIMEOUT", raising=False) monkeypatch.delenv("CI", raising=False) assert jax_compile.dump_traceback_seconds() == 0.0 @@ -272,6 +273,45 @@ def test_the_traceback_dump_defaults_on_under_ci_and_off_elsewhere(monkeypatch): assert jax_compile.dump_traceback_seconds() == 0.0 +def test_the_dump_lands_strictly_before_the_runner_kills_the_process(monkeypatch): + monkeypatch.delenv("PYAUTOFIT_JAX_COMPILE_DUMP_SECS", raising=False) + monkeypatch.setenv("CI", "true") + + # The defect this pins (autolens_workspace_test#271): a flat 300s default + # against a 300s smoke cap meant the runner's SIGKILL always beat the dump, + # so 20 stalled CI runs produced heartbeats and not one traceback. A killed + # process writes no stack, so the threshold MUST be under the cap. + for cap in ("300", "1800", "60"): + monkeypatch.setenv("BUILD_SCRIPT_TIMEOUT", cap) + assert jax_compile.dump_traceback_seconds() < float(cap) + + monkeypatch.setenv("BUILD_SCRIPT_TIMEOUT", "300") + assert jax_compile.dump_traceback_seconds() == 240.0 + + monkeypatch.setenv("BUILD_SCRIPT_TIMEOUT", "1800") + assert jax_compile.dump_traceback_seconds() == 1440.0 + + +def test_an_unusable_runner_cap_falls_back_to_the_flat_ci_default(monkeypatch): + monkeypatch.delenv("PYAUTOFIT_JAX_COMPILE_DUMP_SECS", raising=False) + monkeypatch.setenv("CI", "true") + + # No cap advertised, or a meaningless one: there is nothing to derive from, + # so use the flat default rather than computing a fraction of zero (which + # would silently disable the dump). + for cap in ("", "0", "not-a-number"): + monkeypatch.setenv("BUILD_SCRIPT_TIMEOUT", cap) + assert jax_compile.dump_traceback_seconds() == jax_compile.DEFAULT_CI_DUMP_SECONDS + + +def test_an_explicit_dump_threshold_still_wins_over_the_derived_one(monkeypatch): + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("BUILD_SCRIPT_TIMEOUT", "1800") + monkeypatch.setenv("PYAUTOFIT_JAX_COMPILE_DUMP_SECS", "90") + + assert jax_compile.dump_traceback_seconds() == 90.0 + + def test_a_malformed_interval_falls_back_rather_than_raising(monkeypatch): monkeypatch.setenv("PYAUTOFIT_JAX_COMPILE_HEARTBEAT_SECS", "soon") assert jax_compile.heartbeat_seconds() == jax_compile.DEFAULT_HEARTBEAT_SECONDS