fix: serialise NumPy scalars in search output rather than crashing

autofit/non_linear/paths/directory.py

@@ -13,7 +13,7 @@
 from autonerves.dictable import to_dict, from_dict
 from autonerves.output import conditional_output, should_output
 from autofit.text import formatter
-from autofit.tools.util import open_
+from autofit.tools.util import open_, NumpyEncoder
 from autofit.non_linear.samples.samples import Samples
 
 from .abstract import AbstractPaths, _test_mode_segment
@@ -76,8 +76,12 @@ def save_json(self, name, object_dict: Union[dict, list], prefix: str = ""):
         prefix
             A prefix to add to the path which is the name of the folder the file is saved in.
         """
+        # ``NumpyEncoder``: a stray ``np.float32`` (or any NumPy scalar that is
+        # not a ``float64``) would otherwise raise ``TypeError`` here, at the
+        # very end of a successful fit, throwing the whole run away at its
+        # output step. See the encoder's docstring for why float64 hid this.
         with open_(self._path_for_json(name, prefix), "w+") as f:
-            json.dump(object_dict, f, indent=4)
+            json.dump(object_dict, f, indent=4, cls=NumpyEncoder)
 
     def load_json(self, name, prefix: str = ""):
         with open_(self._path_for_json(name, prefix)) as f:

autofit/non_linear/samples/samples.py

@@ -16,6 +16,7 @@
 from autofit.non_linear.test_mode import skip_checks
 from autofit.mapper.prior_model.abstract import AbstractPriorModel
 from autofit.non_linear.samples.sample import Sample
+from autofit.tools.util import NumpyEncoder
 
 from .summary import SamplesSummary
 from .interface import SamplesInterface, to_instance
@@ -334,8 +335,14 @@ def write_table(self, filename: Union[str, Path]):
         )
 
     def info_to_json(self, filename):
+        # ``NumpyEncoder`` for the same reason as ``DirectoryPaths.save_json``.
+        # ``samples_info`` is the search's own diagnostic channel -- step
+        # counters, timings, per-search settings -- so it is the dict most
+        # likely to carry a NumPy scalar straight out of a search's internals,
+        # and every new counter added to it is another chance to reintroduce
+        # the crash.
         with open(filename, "w") as outfile:
-            json.dump(self.samples_info, outfile)
+            json.dump(self.samples_info, outfile, cls=NumpyEncoder)
 
     @property
     def max_log_likelihood_sample(self) -> Sample:

autofit/tools/util.py

@@ -13,6 +13,39 @@
 from autonerves import conf
 
 
+class NumpyEncoder(json.JSONEncoder):
+    """
+    A ``json`` encoder that can serialise NumPy scalars and arrays.
+
+    ``json`` serialises ``np.float64`` without help, because it subclasses
+    Python's ``float``. **``np.float32`` does not subclass anything ``json``
+    knows**, so a single ``float32`` anywhere in an otherwise ordinary dict
+    raises ``TypeError: Object of type float32 is not JSON serializable``. The
+    same is true of ``np.int32``/``np.int64`` (not a Python ``int`` on every
+    platform) and ``np.bool_``.
+
+    That asymmetry is why this failed so rarely and so late: a run in float64
+    is fine, and a run in float32 is fine right up until the moment it writes
+    its results. It surfaces at the *end of a successful fit*, discarding the
+    whole computation at the output step.
+
+    Coercion belongs here rather than at each producer. The producers are
+    search-specific and new ones keep appearing -- every new diagnostic counter
+    or summary field is another chance to reintroduce this -- so the encoder is
+    the one place that closes the class rather than one instance of it.
+    """
+
+    def default(self, o):
+        if isinstance(o, np.ndarray):
+            return o.tolist()
+        if isinstance(o, np.generic):
+            # ``.item()`` returns the nearest Python scalar: float32/float64 ->
+            # float, any int width -> int, bool_ -> bool. Precision is not lost
+            # on the way out, because a Python float IS a double.
+            return o.item()
+        return super().default(o)
+
+
 def split_paths(func):
     """
     Split string paths if they are passed.

test_autofit/tools/test_numpy_encoder.py

@@ -0,0 +1,128 @@
+import json
+
+import numpy as np
+import pytest
+
+import autofit as af
+from autofit.tools.util import NumpyEncoder
+
+
+class TestNumpyEncoder:
+    def test__float32_round_trips_rather_than_raising(self):
+        """
+        The bug this closes. ``float32`` does not subclass Python ``float``, so
+        a bare ``json.dump`` raises ``TypeError`` on it.
+        """
+        with pytest.raises(TypeError):
+            json.dumps({"x": np.float32(1.5)})
+
+        assert json.loads(json.dumps({"x": np.float32(1.5)}, cls=NumpyEncoder)) == {
+            "x": 1.5
+        }
+
+    @pytest.mark.parametrize(
+        "value, expected, expected_type",
+        [
+            (np.float32(1.5), 1.5, float),
+            (np.float16(0.5), 0.5, float),
+            (np.int32(3), 3, int),
+            (np.int64(4), 4, int),
+            (np.uint8(5), 5, int),
+            (np.bool_(True), True, bool),
+        ],
+    )
+    def test__numpy_scalars_become_plain_python(
+        self, value, expected, expected_type
+    ):
+        loaded = json.loads(json.dumps({"x": value}, cls=NumpyEncoder))["x"]
+
+        assert loaded == expected
+        assert type(loaded) is expected_type
+
+    def test__arrays_become_lists(self):
+        loaded = json.loads(
+            json.dumps({"x": np.array([[1, 2], [3, 4]], dtype=np.int32)},
+                       cls=NumpyEncoder)
+        )
+
+        assert loaded == {"x": [[1, 2], [3, 4]]}
+
+    def test__float64_is_unchanged(self):
+        """
+        ``float64`` already worked, because it subclasses Python ``float``. The
+        encoder must not alter it -- and must not lose precision routing it.
+        """
+        value = np.float64(0.1234567890123456789)
+
+        assert json.dumps({"x": value}) == json.dumps({"x": value}, cls=NumpyEncoder)
+        assert json.loads(json.dumps({"x": value}, cls=NumpyEncoder))["x"] == float(
+            value
+        )
+
+    def test__float32_precision_is_not_invented(self):
+        """
+        ``.item()`` widens float32 to a Python double. The value must be the
+        float32's exact value, not a re-rounded decimal.
+        """
+        loaded = json.loads(json.dumps({"x": np.float32(0.1)}, cls=NumpyEncoder))["x"]
+
+        assert loaded == float(np.float32(0.1))
+
+    def test__unserialisable_objects_still_raise(self):
+        """
+        The encoder widens what can be written; it must not silently swallow a
+        genuinely unserialisable object.
+        """
+        with pytest.raises(TypeError):
+            json.dumps({"x": object()}, cls=NumpyEncoder)
+
+
+class TestOutputPathsUseTheEncoder:
+    def test__save_json_writes_a_float32(self, output_directory):
+        """
+        ``DirectoryPaths.save_json`` is where this fired in the field -- at the
+        END of a successful fit, discarding the whole run at its output step.
+        """
+        paths = af.DirectoryPaths(name="save_json_float32")
+        paths._identifier = "id"
+
+        paths.save_json(name="counters", object_dict={"clipped": np.float32(4.0)})
+
+        assert paths.load_json(name="counters") == {"clipped": 4.0}
+
+    def test__samples_info_json_writes_numpy_scalars(self, output_directory, tmp_path):
+        """
+        ``samples_info`` is the search's own diagnostic channel, so it is the
+        dict most likely to carry a NumPy scalar out of a search's internals.
+        """
+        from autofit import example
+
+        model = af.Model(example.Gaussian)
+
+        samples = af.Samples(
+            model=model,
+            sample_list=af.Sample.from_lists(
+                model=model,
+                parameter_lists=[[1.0, 2.0, 3.0]],
+                log_likelihood_list=[1.0],
+                log_prior_list=[0.0],
+                weight_list=[1.0],
+            ),
+            samples_info={
+                "n_clipped_lane_steps": np.int32(414),
+                "best_fom": np.float32(-2.5),
+            },
+        )
+
+        filename = tmp_path / "info.json"
+        samples.info_to_json(filename=filename)
+
+        with open(filename) as f:
+            loaded = json.load(f)
+
+        # ``samples_info`` also carries an auto-added ``class_path``, so assert
+        # on the values under test rather than on the whole dict.
+        assert loaded["n_clipped_lane_steps"] == 414
+        assert type(loaded["n_clipped_lane_steps"]) is int
+        assert loaded["best_fom"] == -2.5
+        assert type(loaded["best_fom"]) is float