Add interrupts, memory caps, async streaming, ETF callback results

CHANGELOG.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## Unreleased
+
+### Added
+
+- **Interrupt running Python** - `py:interrupt/1` and `py_context:interrupt/1`
+  raise `KeyboardInterrupt` in the thread executing a context; the in-flight call
+  returns `{error, interrupted}`. `py_context:call/eval/exec` now interrupt
+  automatically when their timeout expires, so `{error, timeout}` stops the Python
+  code instead of only abandoning the reply while the thread kept burning CPU.
+  Works in both `worker` and `owngil` modes, and is callable while the context
+  process is blocked in a NIF. CPython delivers async exceptions at bytecode
+  boundaries, so code blocked in a C call (`time.sleep`, a numpy kernel, a socket
+  read) is interrupted once that call returns. See `docs/interrupts.md`.
+- **Per-context memory caps** - `py_context:new(#{mode => owngil, memory_limit => Bytes})`
+  caps memory allocated by one context; exceeding it raises `MemoryError` there and
+  leaves other contexts untouched. Requires `{enable_memory_limits, true}` in the
+  application environment, since the allocator is hooked before Python starts.
+  Accounting covers obmalloc traffic only: allocations over 512 bytes (large
+  binaries, numpy buffers) are not counted, granularity is one 1 MB arena, and
+  `worker` mode returns `{error, memory_limit_requires_owngil}` because those
+  contexts share the main interpreter. `py_nif:context_memory_usage/1` reports
+  usage. See `docs/memory.md`.
+- **Async generator streaming** - `py:stream_start/3,4` accepts async generators
+  again, driving them on a private event loop and emitting the same
+  `{py_stream, Ref, ...}` events. The docs claimed this since 3.0.0 without the
+  code behind it. `py:stream/4` with kwargs and `py:stream_eval/1,2` remain
+  sync-only, which is now stated explicitly.
+
+### Changed
+
+- **Callback results cross as external term format** - results returned from an
+  Erlang callback into Python are encoded with `term_to_binary` and decoded by the
+  same `term_to_py` converter used for call arguments, replacing the Python-repr
+  string that was parsed with `ast.literal_eval`. This fixes binaries containing
+  backslashes, quotes, newlines or tabs (which produced an unparseable literal and
+  were silently handed to Python as the raw repr text), `[]` arriving as `''`,
+  float precision loss, and the base64 round-trip for pids and references, which
+  now cross as native `Pid` and `Ref` objects.
+
+  Breaking: an Erlang string returned from a callback (`"abc"`, a list of
+  integers) now reaches Python as `[97, 98, 99]` rather than `'abc'`, the same
+  conversion call arguments have always used. Return a binary (`<<"abc">>`) for a
+  Python `str`.
+
 ## 3.1.1 (2026-05-31)
 
 ### Changed

c_src/py_callback.c

@@ -946,6 +946,30 @@ static int copy_callback_results_to_nested(suspended_context_state_t *nested,
     return 0;
 }
 
+/**
+ * Decode an external-term-format binary into a Python object.
+ *
+ * Returns a NEW reference on success, NULL on failure (decode error or a
+ * conversion error, in which case a Python exception may be set).
+ */
+static PyObject *etf_binary_to_py(const char *data, size_t len) {
+    ErlNifEnv *tmp_env = enif_alloc_env();
+    if (tmp_env == NULL) {
+        return NULL;
+    }
+
+    ERL_NIF_TERM term;
+    if (enif_binary_to_term(tmp_env, (unsigned char *)data, len, &term,
+                            ERL_NIF_BIN2TERM_SAFE) == 0) {
+        enif_free_env(tmp_env);
+        return NULL;
+    }
+
+    PyObject *result = term_to_py(tmp_env, term);
+    enif_free_env(tmp_env);
+    return result;
+}
+
 /**
  * Helper to convert __etf__:base64 strings to Python objects.
  * Used for encoding pids and references in callback responses.
@@ -1003,28 +1027,8 @@ static PyObject *decode_etf_string(const char *str, Py_ssize_t len) {
         return NULL;
     }
 
-    /* Create a temporary NIF environment to decode the term */
-    ErlNifEnv *tmp_env = enif_alloc_env();
-    if (tmp_env == NULL) {
-        Py_DECREF(decoded);
-        return NULL;
-    }
-
-    /* Decode the ETF binary to an Erlang term */
-    ERL_NIF_TERM term;
-    if (enif_binary_to_term(tmp_env, (unsigned char *)bin_data, bin_len, &term, ERL_NIF_BIN2TERM_SAFE) == 0) {
-        /* Decoding failed */
-        enif_free_env(tmp_env);
-        Py_DECREF(decoded);
-        return NULL;
-    }
-
+    PyObject *result = etf_binary_to_py(bin_data, (size_t)bin_len);
     Py_DECREF(decoded);
-
-    /* Convert the term to a Python object */
-    PyObject *result = term_to_py(tmp_env, term);
-    enif_free_env(tmp_env);
-
     return result;
 }
 
@@ -1150,7 +1154,12 @@ static PyObject *convert_etf_strings(PyObject *obj) {
 
 /**
  * Helper to parse callback response data into a Python object.
- * Response format: status_byte (0=ok, 1=error) + python_repr_string
+ *
+ * Response format: status_byte + payload
+ *   0 = ok, payload is a Python source literal (legacy; kept so responses
+ *       produced by an older Erlang side still decode)
+ *   1 = error, payload is an error message
+ *   2 = ok, payload is an external-term-format binary (current encoding)
  */
 static PyObject *parse_callback_response(unsigned char *response_data, size_t response_len) {
     if (response_len < 1) {
@@ -1161,7 +1170,7 @@ static PyObject *parse_callback_response(unsigned char *response_data, size_t re
     uint8_t status = response_data[0];
 
     if (response_len < 2) {
-        if (status == 0) {
+        if (status == 0 || status == 2) {
             Py_RETURN_NONE;
         } else {
             PyErr_SetString(PyExc_RuntimeError, "Erlang callback failed");
@@ -1173,7 +1182,18 @@ static PyObject *parse_callback_response(unsigned char *response_data, size_t re
     size_t result_len = response_len - 1;
 
     PyObject *result = NULL;
-    if (status == 0) {
+    if (status == 2) {
+        /* Current encoding: external term format, decoded by the same
+         * converter that handles call arguments. */
+        result = etf_binary_to_py(result_str, result_len);
+        if (result == NULL) {
+            if (!PyErr_Occurred()) {
+                PyErr_SetString(PyExc_RuntimeError,
+                                "Failed to decode callback result");
+            }
+            return NULL;
+        }
+    } else if (status == 0) {
         /* Try to evaluate the result string as Python literal.
          * Import ast.literal_eval fresh to support subinterpreters
          * (the cached g_ast_literal_eval may be from a different interpreter). */

c_src/py_convert.c

@@ -825,6 +825,16 @@ static ERL_NIF_TERM make_py_error(ErlNifEnv *env) {
             enif_make_tuple2(env, ATOM_STOP_ITERATION, ATOM_NONE));
     }
 
+    /* KeyboardInterrupt means py_nif:context_interrupt/1 injected an async
+     * exception, so report it as a distinct reason rather than a Python error. */
+    if (PyErr_GivenExceptionMatches(type, PyExc_KeyboardInterrupt)) {
+        PyErr_Clear();
+        Py_XDECREF(type);
+        Py_XDECREF(value);
+        Py_XDECREF(traceback);
+        return enif_make_tuple2(env, ATOM_ERROR, enif_make_atom(env, "interrupted"));
+    }
+
     /* Get exception message. PyUnicode_AsUTF8 can return NULL (e.g. a str with
      * lone surrogates); never pass NULL to enif_make_string. */
     PyObject *str = PyObject_Str(value);