From f31b4874ea9c670937586a7c9719784174ad7d42 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 16:54:50 +0000 Subject: [PATCH] fix: serialise NumPy scalars in search output rather than crashing A successful fit could die at its output step. `DirectoryPaths.save_json` and `Samples.info_to_json` both used a bare `json.dump`, and there was no JSON encoder anywhere in `autofit/`, so a single NumPy scalar in an otherwise ordinary dict raised: TypeError: Object of type float32 is not JSON serializable Reproduced against the unfixed tree: `paths.save_json(name="c", object_dict={"clipped": np.float32(4.0)})` raises exactly that. Why it stayed hidden. `np.float64` subclasses Python's `float`, so `json` serialises it without help; `np.float32` subclasses nothing `json` knows, and neither does `np.int32`/`np.int64` (not a Python `int` on every platform) or `np.bool_`. A float64 run is fine, and a float32 run is fine right up until the moment it writes its results -- at which point the whole computation is thrown away at the last step. It was found on the `imaging/mge` profiling cell during the Clipper prototype (autolens_profiling#128), on a code path that cell had apparently never taken: it did not fire while 14 of 16 lanes were dead, because the float32 values never reached the saved object. It fires precisely when lane survival improves, so it stands directly in the way of the phase-2 Clipper validation campaign. Adds `NumpyEncoder` in `autofit/tools/util.py` -- `np.ndarray` -> `tolist()`, `np.generic` -> `item()`, anything else deferred to the base class so a genuinely unserialisable object still raises. Wired into both output-path writers. Coercion belongs at the encoder rather than at each producer. The producers are search-specific and new ones keep appearing: `samples_info` in particular is a search's own diagnostic channel, so every new counter added to it is another chance to reintroduce this. The encoder closes the class; chasing producers closes one instance. `float64` is unchanged, asserted byte-for-byte against a bare `json.dumps`, and `.item()` widens float32 to a Python double without inventing precision. 13 new tests. Full suite: 1804 passed, 4 skipped, 1 failed -- the failure (`test_nautilus.py::test__single_core_builds_no_pool`) is pre-existing and reproduces identically on a clean tree. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01FzF2XmKQaqZRWZfZMxvTNR --- autofit/non_linear/paths/directory.py | 8 +- autofit/non_linear/samples/samples.py | 9 +- autofit/tools/util.py | 33 ++++++ test_autofit/tools/test_numpy_encoder.py | 128 +++++++++++++++++++++++ 4 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 test_autofit/tools/test_numpy_encoder.py diff --git a/autofit/non_linear/paths/directory.py b/autofit/non_linear/paths/directory.py index 394846970..ae153a4d5 100644 --- a/autofit/non_linear/paths/directory.py +++ b/autofit/non_linear/paths/directory.py @@ -13,7 +13,7 @@ from autonerves.dictable import to_dict, from_dict from autonerves.output import conditional_output, should_output from autofit.text import formatter -from autofit.tools.util import open_ +from autofit.tools.util import open_, NumpyEncoder from autofit.non_linear.samples.samples import Samples from .abstract import AbstractPaths, _test_mode_segment @@ -76,8 +76,12 @@ def save_json(self, name, object_dict: Union[dict, list], prefix: str = ""): prefix A prefix to add to the path which is the name of the folder the file is saved in. """ + # ``NumpyEncoder``: a stray ``np.float32`` (or any NumPy scalar that is + # not a ``float64``) would otherwise raise ``TypeError`` here, at the + # very end of a successful fit, throwing the whole run away at its + # output step. See the encoder's docstring for why float64 hid this. with open_(self._path_for_json(name, prefix), "w+") as f: - json.dump(object_dict, f, indent=4) + json.dump(object_dict, f, indent=4, cls=NumpyEncoder) def load_json(self, name, prefix: str = ""): with open_(self._path_for_json(name, prefix)) as f: diff --git a/autofit/non_linear/samples/samples.py b/autofit/non_linear/samples/samples.py index acacd449b..26ca91ab6 100644 --- a/autofit/non_linear/samples/samples.py +++ b/autofit/non_linear/samples/samples.py @@ -16,6 +16,7 @@ from autofit.non_linear.test_mode import skip_checks from autofit.mapper.prior_model.abstract import AbstractPriorModel from autofit.non_linear.samples.sample import Sample +from autofit.tools.util import NumpyEncoder from .summary import SamplesSummary from .interface import SamplesInterface, to_instance @@ -334,8 +335,14 @@ def write_table(self, filename: Union[str, Path]): ) def info_to_json(self, filename): + # ``NumpyEncoder`` for the same reason as ``DirectoryPaths.save_json``. + # ``samples_info`` is the search's own diagnostic channel -- step + # counters, timings, per-search settings -- so it is the dict most + # likely to carry a NumPy scalar straight out of a search's internals, + # and every new counter added to it is another chance to reintroduce + # the crash. with open(filename, "w") as outfile: - json.dump(self.samples_info, outfile) + json.dump(self.samples_info, outfile, cls=NumpyEncoder) @property def max_log_likelihood_sample(self) -> Sample: diff --git a/autofit/tools/util.py b/autofit/tools/util.py index 533ba2318..ebcdfb3e7 100644 --- a/autofit/tools/util.py +++ b/autofit/tools/util.py @@ -13,6 +13,39 @@ from autonerves import conf +class NumpyEncoder(json.JSONEncoder): + """ + A ``json`` encoder that can serialise NumPy scalars and arrays. + + ``json`` serialises ``np.float64`` without help, because it subclasses + Python's ``float``. **``np.float32`` does not subclass anything ``json`` + knows**, so a single ``float32`` anywhere in an otherwise ordinary dict + raises ``TypeError: Object of type float32 is not JSON serializable``. The + same is true of ``np.int32``/``np.int64`` (not a Python ``int`` on every + platform) and ``np.bool_``. + + That asymmetry is why this failed so rarely and so late: a run in float64 + is fine, and a run in float32 is fine right up until the moment it writes + its results. It surfaces at the *end of a successful fit*, discarding the + whole computation at the output step. + + Coercion belongs here rather than at each producer. The producers are + search-specific and new ones keep appearing -- every new diagnostic counter + or summary field is another chance to reintroduce this -- so the encoder is + the one place that closes the class rather than one instance of it. + """ + + def default(self, o): + if isinstance(o, np.ndarray): + return o.tolist() + if isinstance(o, np.generic): + # ``.item()`` returns the nearest Python scalar: float32/float64 -> + # float, any int width -> int, bool_ -> bool. Precision is not lost + # on the way out, because a Python float IS a double. + return o.item() + return super().default(o) + + def split_paths(func): """ Split string paths if they are passed. diff --git a/test_autofit/tools/test_numpy_encoder.py b/test_autofit/tools/test_numpy_encoder.py new file mode 100644 index 000000000..1fa0b52fe --- /dev/null +++ b/test_autofit/tools/test_numpy_encoder.py @@ -0,0 +1,128 @@ +import json + +import numpy as np +import pytest + +import autofit as af +from autofit.tools.util import NumpyEncoder + + +class TestNumpyEncoder: + def test__float32_round_trips_rather_than_raising(self): + """ + The bug this closes. ``float32`` does not subclass Python ``float``, so + a bare ``json.dump`` raises ``TypeError`` on it. + """ + with pytest.raises(TypeError): + json.dumps({"x": np.float32(1.5)}) + + assert json.loads(json.dumps({"x": np.float32(1.5)}, cls=NumpyEncoder)) == { + "x": 1.5 + } + + @pytest.mark.parametrize( + "value, expected, expected_type", + [ + (np.float32(1.5), 1.5, float), + (np.float16(0.5), 0.5, float), + (np.int32(3), 3, int), + (np.int64(4), 4, int), + (np.uint8(5), 5, int), + (np.bool_(True), True, bool), + ], + ) + def test__numpy_scalars_become_plain_python( + self, value, expected, expected_type + ): + loaded = json.loads(json.dumps({"x": value}, cls=NumpyEncoder))["x"] + + assert loaded == expected + assert type(loaded) is expected_type + + def test__arrays_become_lists(self): + loaded = json.loads( + json.dumps({"x": np.array([[1, 2], [3, 4]], dtype=np.int32)}, + cls=NumpyEncoder) + ) + + assert loaded == {"x": [[1, 2], [3, 4]]} + + def test__float64_is_unchanged(self): + """ + ``float64`` already worked, because it subclasses Python ``float``. The + encoder must not alter it -- and must not lose precision routing it. + """ + value = np.float64(0.1234567890123456789) + + assert json.dumps({"x": value}) == json.dumps({"x": value}, cls=NumpyEncoder) + assert json.loads(json.dumps({"x": value}, cls=NumpyEncoder))["x"] == float( + value + ) + + def test__float32_precision_is_not_invented(self): + """ + ``.item()`` widens float32 to a Python double. The value must be the + float32's exact value, not a re-rounded decimal. + """ + loaded = json.loads(json.dumps({"x": np.float32(0.1)}, cls=NumpyEncoder))["x"] + + assert loaded == float(np.float32(0.1)) + + def test__unserialisable_objects_still_raise(self): + """ + The encoder widens what can be written; it must not silently swallow a + genuinely unserialisable object. + """ + with pytest.raises(TypeError): + json.dumps({"x": object()}, cls=NumpyEncoder) + + +class TestOutputPathsUseTheEncoder: + def test__save_json_writes_a_float32(self, output_directory): + """ + ``DirectoryPaths.save_json`` is where this fired in the field -- at the + END of a successful fit, discarding the whole run at its output step. + """ + paths = af.DirectoryPaths(name="save_json_float32") + paths._identifier = "id" + + paths.save_json(name="counters", object_dict={"clipped": np.float32(4.0)}) + + assert paths.load_json(name="counters") == {"clipped": 4.0} + + def test__samples_info_json_writes_numpy_scalars(self, output_directory, tmp_path): + """ + ``samples_info`` is the search's own diagnostic channel, so it is the + dict most likely to carry a NumPy scalar out of a search's internals. + """ + from autofit import example + + model = af.Model(example.Gaussian) + + samples = af.Samples( + model=model, + sample_list=af.Sample.from_lists( + model=model, + parameter_lists=[[1.0, 2.0, 3.0]], + log_likelihood_list=[1.0], + log_prior_list=[0.0], + weight_list=[1.0], + ), + samples_info={ + "n_clipped_lane_steps": np.int32(414), + "best_fom": np.float32(-2.5), + }, + ) + + filename = tmp_path / "info.json" + samples.info_to_json(filename=filename) + + with open(filename) as f: + loaded = json.load(f) + + # ``samples_info`` also carries an auto-added ``class_path``, so assert + # on the values under test rather than on the whole dict. + assert loaded["n_clipped_lane_steps"] == 414 + assert type(loaded["n_clipped_lane_steps"]) is int + assert loaded["best_fom"] == -2.5 + assert type(loaded["best_fom"]) is float