feat: support yield from as expression with return value

bugs.json

@@ -2,8 +2,8 @@
   "_meta": {
     "source": "MicroPython tests/basics differential vs CPython 3.13",
     "verified": "each repro re-run through VM + python3",
-    "count": 13,
-    "fixed": "85 bugs fixed and migrated to vm.json (try/finally + with unwinding cluster; 11 builtin/parser/format divergences from rows 1-11 of the low-NEWLINE triage; generator_return, string_format_modulo_int %*d, seq_unpack list targets, class_getattr dunder-on-type, builtin_slice type identity, containment dict/set-led tuple, range sequence-unpack)"
+    "count": 9,
+    "fixed": "89 bugs fixed and migrated to vm.json (try/finally + with unwinding cluster; 11 builtin/parser/format divergences from rows 1-11 of the low-NEWLINE triage; generator_return, string_format_modulo_int %*d, seq_unpack list targets, class_getattr dunder-on-type, builtin_slice type identity, containment dict/set-led tuple, range sequence-unpack; yield-from-as-expression cluster: gen_yield_from + gen_yield_from_ducktype + gen_yield_from_exc + gen_yield_from_stopped via a parse_atom Yield arm plus a LoadYieldFrom opcode that carries the subiterator return/StopIteration value, 40 regression cases)"
   },
   "bugs": [
     {
@@ -14,38 +14,6 @@
       "expected": "8",
       "actual": "19"
     },
-    {
-      "id": "gen_yield_from",
-      "category": "parser",
-      "summary": "'yield from' used as an expression whose value is consumed (print((yield from gen()))) is rejected at parse time, though yield from as a statement works and uses only supported features.",
-      "repro": "def gen():\n    yield 1\ndef gen2():\n    print((yield from gen()))\nprint(list(gen2()))",
-      "expected": "None\n[1]",
-      "actual": ""
-    },
-    {
-      "id": "gen_yield_from_ducktype",
-      "category": "parser",
-      "summary": "'yield from' used as an expression (ret = yield from MyIter()) is rejected at parse time even though that construct uses only supported features; the parser bug blocks the whole file.",
-      "repro": "class MyIter:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration(42)\ndef gen4():\n    global ret\n    ret = yield from MyIter()\nret = None\nprint(list(gen4()))\nprint(ret)",
-      "expected": "[]\n42",
-      "actual": ""
-    },
-    {
-      "id": "gen_yield_from_exc",
-      "category": "parser",
-      "summary": "Uses only supported features (yield from, exception propagation) but 'yield from' as an expression value is rejected at parse time.",
-      "repro": "def gen():\n    yield 1\n    raise ValueError\ndef gen2():\n    try:\n        print((yield from gen()))\n    except ValueError:\n        print(\"caught\")\nprint(list(gen2()))",
-      "expected": "caught\n[1]",
-      "actual": ""
-    },
-    {
-      "id": "gen_yield_from_stopped",
-      "category": "parser",
-      "summary": "Uses only supported features but bare 'yield' (no value) and 'yield from' as an expression value are rejected at parse time.",
-      "repro": "def gen():\n    return 1\n    yield\nf = gen()\ndef run():\n    print((yield from f))\ntry:\n    next(run())\nexcept StopIteration:\n    print(\"SI\")",
-      "expected": "1\nSI",
-      "actual": ""
-    },
     {
       "id": "del_deref",
       "category": "scoping",

compiler/src/modules/parser/expr.rs

@@ -226,6 +226,8 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
                     }
                 }
             }
+            // `yield` / `yield from` as an expression value (keyword already consumed).
+            TokenType::Yield => self.emit_yield(),
             TokenType::Lambda => self.parse_lambda(),
             // Caret at consumed token; skip if `advance()` already reported the error.
             _ => {

compiler/src/modules/parser/stmt.rs

@@ -44,26 +44,7 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
             }
             Some(TokenType::Yield) => {
                 self.advance();
-                // Check the line boundary first: `peek`/`eat_if` would consume the Newline.
-                if self.peek_same_line().is_none() {
-                    // Bare `yield` (no value): a line boundary ends the statement.
-                    self.chunk.emit(OpCode::LoadNone, 0);
-                    self.chunk.emit(OpCode::Yield, 0);
-                } else if self.eat_if(TokenType::From) {
-                    // `yield from`: GetIter+ForIter+Yield loop; LoadNone at end (return value not tracked).
-                    self.expr();
-                    self.chunk.emit(OpCode::GetIter, 0);
-                    let loop_start = self.chunk.instructions.len() as u16;
-                    let fi = self.emit_jump(OpCode::ForIter);
-                    self.chunk.emit(OpCode::Yield, 0);
-                    self.chunk.emit(OpCode::PopTop, 0);
-                    self.chunk.emit(OpCode::Jump, loop_start);
-                    self.patch(fi);
-                    self.chunk.emit(OpCode::LoadNone, 0);
-                } else {
-                    self.expr();
-                    self.chunk.emit(OpCode::Yield, 0);
-                }
+                self.emit_yield();
                 true
             }
             Some(TokenType::Async) => {
@@ -652,6 +633,34 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
         }
     }
 
+    /* Emit `yield` / `yield from` / bare `yield`, leaving the produced value on the stack so it works in both statement and expression position. Assumes the `yield` keyword was already consumed. */
+    pub(super) fn emit_yield(&mut self) {
+        // No value when a line boundary or a closing token follows (`(yield)`, `f(yield)`).
+        let bare = matches!(
+            self.peek_same_line(),
+            None | Some(TokenType::Rpar | TokenType::Rsqb | TokenType::Rbrace
+                | TokenType::Comma | TokenType::Colon)
+        );
+        if bare {
+            self.chunk.emit(OpCode::LoadNone, 0);
+            self.chunk.emit(OpCode::Yield, 0);
+        } else if self.eat_if(TokenType::From) {
+            // `yield from`: GetIter+ForIter+Yield loop; LoadYieldFrom pushes the subiterator's return value.
+            self.expr();
+            self.chunk.emit(OpCode::GetIter, 0);
+            let loop_start = self.chunk.instructions.len() as u16;
+            let fi = self.emit_jump(OpCode::ForIter);
+            self.chunk.emit(OpCode::Yield, 0);
+            self.chunk.emit(OpCode::PopTop, 0);
+            self.chunk.emit(OpCode::Jump, loop_start);
+            self.patch(fi);
+            self.chunk.emit(OpCode::LoadYieldFrom, 0);
+        } else {
+            self.expr();
+            self.chunk.emit(OpCode::Yield, 0);
+        }
+    }
+
     pub(super) fn assign(&mut self, name: String) {
         self.advance();
         self.expr();

compiler/src/modules/parser/types.rs

@@ -43,6 +43,8 @@ pub enum OpCode {
     InPlaceAdd,
     /* Unary plus: calls `__pos__`, coerces bool to int, else identity on numbers. */
     Pos,
+    /* Pushes the value `yield from` produces: the exhausted subiterator's return / StopIteration value. */
+    LoadYieldFrom,
 }
 
 // Python builtin name -> (specialised OpCode, `leaves_value_on_stack`).

compiler/src/modules/vm/dispatch.rs

@@ -575,6 +575,7 @@ impl<'a> VM<'a> {
             OpCode::DictUpdate | OpCode::SetUpdate | OpCode::ListExtend => self.handle_spread_merge(opcode)?,
 
             OpCode::Yield => self.handle_yield()?,
+            OpCode::LoadYieldFrom => self.push(self.yield_from_value),
             OpCode::LoadEllipsis => {
                 let v = self.heap.alloc(HeapObj::Ellipsis)?;
                 self.push(v);
@@ -807,6 +808,8 @@ impl<'a> VM<'a> {
                 self.yielded = false;
                 self.push(result);
             } else {
+                // `yield from gen` evaluates to the subgenerator's return value.
+                self.yield_from_value = result;
                 self.iter_stack.pop();
                 *ip = op as usize;
             }
@@ -818,11 +821,22 @@ impl<'a> VM<'a> {
             match self.try_call_dunder(iter, "__next__", &[], chunk, slots) {
                 Ok(Some(item)) => { self.push(item); }
                 Ok(None) => {
+                    self.yield_from_value = Val::none();
                     self.iter_stack.pop();
                     if op as usize > n { return Err(cold_runtime("jump target out of bounds")); }
                     *ip = op as usize;
                 }
                 Err(VmErr::Raised(m)) if m == "StopIteration" || m.starts_with("StopIteration:") => {
+                    // `raise StopIteration(v)` carries `v` as the yield-from value via the lifted instance.
+                    self.yield_from_value = if m.starts_with("StopIteration:") {
+                        match self.pending.exc_val {
+                            Some(e) if e.is_heap() => match self.heap.get(e) {
+                                HeapObj::ExcInstance(_, args) => args.first().copied().unwrap_or(Val::none()),
+                                _ => Val::none(),
+                            },
+                            _ => Val::none(),
+                        }
+                    } else { Val::none() };
                     self.iter_stack.pop();
                     if op as usize > n { return Err(cold_runtime("jump target out of bounds")); }
                     *ip = op as usize;
@@ -834,6 +848,7 @@ impl<'a> VM<'a> {
         match self.iter_stack.last_mut().and_then(|f| f.next_item()) {
             Some(item) => self.push(item),
             None => {
+                self.yield_from_value = Val::none();
                 self.iter_stack.pop();
                 if op as usize > n { return Err(cold_runtime("jump target out of bounds")); }
                 *ip = op as usize;

compiler/src/modules/vm/gc.rs

@@ -12,6 +12,7 @@ impl<'a> VM<'a> {
         for &v in &self.event_queue { self.heap.mark(v); }
         // The handled exception and any pending finally return value outlive their stack slots.
         if let Some(v) = self.pending.exc_val { self.heap.mark(v); }
+        self.heap.mark(self.yield_from_value);
         if let Some(v) = self.handling_exc { self.heap.mark(v); }
         for u in &self.unwind_stack { if let Unwind::Return(v) = u { self.heap.mark(*v); } }
         // Scheduler holds parked coroutines (and their `WaitingForChildren` task lists) across `top_loop` resumes; mark them so the saved state isn't swept under us.

compiler/src/modules/vm/mod.rs

@@ -130,6 +130,8 @@ pub struct VM<'a> {
     /* Overrides `exec`'s captured `exc_base`. Set by `resume_coroutine` to the level *before* restored exception frames so dispatch's handler search includes them; consumed once at exec entry. */
     pub(crate) pending_exec_exc_base: Option<usize>,
     pub(crate) yielded: bool,
+    /* Return value of the most recently exhausted iterator; read by `LoadYieldFrom` so `x = yield from it` evaluates to the subiterator's StopIteration value. */
+    pub(crate) yield_from_value: Val,
     pub(crate) resume_ip: usize,
     pub output: Vec<String>,
     /* True when the last `output` entry is an unterminated line (print(end="") left it open). */
@@ -183,6 +185,7 @@ impl<'a> VM<'a> {
             pending_sync_frames: Vec::new(),
             pending_exec_exc_base: None,
             yielded: false,
+            yield_from_value: Val::none(),
             resume_ip: 0,
             strict_input: false,
             output: Vec::new(),

compiler/tests/cases/parser.json

@@ -1264,14 +1264,14 @@
     "src": "yield from x", 
     "constants": [], 
     "names": ["x_0"], 
-    "instructions": [["LoadName",0], ["GetIter",0], ["ForIter",6], ["Yield",0], ["PopTop",0], ["Jump",2], ["LoadNone",0], ["PopTop",0], ["ReturnValue",0]],
+    "instructions": [["LoadName",0], ["GetIter",0], ["ForIter",6], ["Yield",0], ["PopTop",0], ["Jump",2], ["LoadYieldFrom",0], ["PopTop",0], ["ReturnValue",0]],
     "annotations": {}
   },
   {
     "src": "yield from range(3)",
     "constants": ["3"],
     "names": [],
-    "instructions": [["LoadConst",0], ["CallRange",1], ["GetIter",0], ["ForIter",7], ["Yield",0], ["PopTop",0], ["Jump",3], ["LoadNone",0], ["PopTop",0], ["ReturnValue",0]],
+    "instructions": [["LoadConst",0], ["CallRange",1], ["GetIter",0], ["ForIter",7], ["Yield",0], ["PopTop",0], ["Jump",3], ["LoadYieldFrom",0], ["PopTop",0], ["ReturnValue",0]],
     "annotations": {}
   },
   {

compiler/tests/cases/vm.json

@@ -3488,5 +3488,45 @@
   {"src": "print(frozenset())", "output": ["frozenset()"]},
   {"src": "print(dict())", "output": ["{}"]},
   {"src": "print(repr(bytes()))", "output": ["b''"]},
-  {"src": "f = float\nprint(f())", "output": ["0.0"]}
+  {"src": "f = float\nprint(f())", "output": ["0.0"]},
+  {"src": "def gen():\n    yield 1\ndef gen2():\n    print((yield from gen()))\nprint(list(gen2()))", "output": ["None", "[1]"]},
+  {"src": "def gen():\n    yield 1\n    yield 2\ndef outer():\n    r = yield from gen()\n    print('r', r)\nprint(list(outer()))", "output": ["r None", "[1, 2]"]},
+  {"src": "def outer():\n    print((yield from [10, 20, 30]))\nprint(list(outer()))", "output": ["None", "[10, 20, 30]"]},
+  {"src": "def outer():\n    print((yield from range(3)))\nprint(list(outer()))", "output": ["None", "[0, 1, 2]"]},
+  {"src": "def outer():\n    print((yield from 'ab'))\nprint(list(outer()))", "output": ["None", "['a', 'b']"]},
+  {"src": "def outer():\n    print((yield from ()))\nprint(list(outer()))", "output": ["None", "[]"]},
+  {"src": "def gen():\n    yield 1\ndef outer():\n    x = (yield from gen()) or 'none'\n    print(x)\nprint(list(outer()))", "output": ["none", "[1]"]},
+  {"src": "def inner():\n    yield 1\n    yield 2\ndef outer():\n    yield 0\n    yield from inner()\n    yield 3\nprint(list(outer()))", "output": ["[0, 1, 2, 3]"]},
+  {"src": "def outer():\n    print((yield from {1: 'a', 2: 'b'}))\nprint(list(outer()))", "output": ["None", "[1, 2]"]},
+  {"src": "def gen():\n    yield 1\ndef outer():\n    res = yield from gen()\n    print(res is None)\nprint(list(outer()))", "output": ["True", "[1]"]},
+  {"src": "class MyIter:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration(42)\ndef gen4():\n    global ret\n    ret = yield from MyIter()\nret = None\nprint(list(gen4()))\nprint(ret)", "output": ["[]", "42"]},
+  {"src": "class It:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration(7)\ndef g():\n    v = yield from It()\n    print('got', v)\nprint(list(g()))", "output": ["got 7", "[]"]},
+  {"src": "class It:\n    def __init__(self):\n        self.n = 0\n    def __iter__(self):\n        return self\n    def __next__(self):\n        self.n += 1\n        if self.n > 2:\n            raise StopIteration(self.n)\n        return self.n\ndef g():\n    v = yield from It()\n    print('ret', v)\nprint(list(g()))", "output": ["ret 3", "[1, 2]"]},
+  {"src": "class It:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration('done')\ndef g():\n    print((yield from It()))\nprint(list(g()))", "output": ["done", "[]"]},
+  {"src": "class It:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration\ndef g():\n    v = yield from It()\n    print(v is None)\nprint(list(g()))", "output": ["True", "[]"]},
+  {"src": "class It:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration([1, 2, 3])\ndef g():\n    v = yield from It()\n    print(v)\nprint(list(g()))", "output": ["[1, 2, 3]", "[]"]},
+  {"src": "class It:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration(99)\ndef g():\n    x = yield from It()\n    yield x\nprint(list(g()))", "output": ["[99]"]},
+  {"src": "class Once:\n    def __init__(self):\n        self.done = False\n    def __iter__(self):\n        return self\n    def __next__(self):\n        if self.done:\n            raise StopIteration(-1)\n        self.done = True\n        return 0\ndef g():\n    r = yield from Once()\n    print('r', r)\nprint(list(g()))", "output": ["r -1", "[0]"]},
+  {"src": "class It:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration(True)\ndef g():\n    v = yield from It()\n    print(v, type(v))\nprint(list(g()))", "output": ["True <class 'bool'>", "[]"]},
+  {"src": "class It:\n    def __iter__(self):\n        return self\n    def __next__(self):\n        raise StopIteration(3.5)\ndef g():\n    v = yield from It()\n    print(v + 1)\nprint(list(g()))", "output": ["4.5", "[]"]},
+  {"src": "def gen():\n    yield 1\n    raise ValueError\ndef gen2():\n    try:\n        print((yield from gen()))\n    except ValueError:\n        print('caught')\nprint(list(gen2()))", "output": ["caught", "[1]"]},
+  {"src": "def gen():\n    yield 1\n    raise KeyError('k')\ndef g():\n    try:\n        yield from gen()\n    except KeyError:\n        print('key')\nprint(list(g()))", "output": ["key", "[1]"]},
+  {"src": "def gen():\n    raise RuntimeError\n    yield\ndef g():\n    try:\n        yield from gen()\n    except RuntimeError:\n        print('rt')\nprint(list(g()))", "output": ["rt", "[]"]},
+  {"src": "def gen():\n    yield 1\n    yield 2\n    raise ValueError('boom')\ndef g():\n    try:\n        yield from gen()\n    except ValueError as e:\n        print('got', e)\nprint(list(g()))", "output": ["got boom", "[1, 2]"]},
+  {"src": "def sub():\n    yield 1\n    raise ZeroDivisionError\ndef g():\n    try:\n        yield from sub()\n    except ZeroDivisionError:\n        print('zde')\n    yield 9\nprint(list(g()))", "output": ["zde", "[1, 9]"]},
+  {"src": "def sub():\n    raise TypeError\n    yield\ndef g():\n    try:\n        yield from sub()\n    except Exception:\n        print('generic')\nprint(list(g()))", "output": ["generic", "[]"]},
+  {"src": "def sub():\n    yield 1\n    raise IndexError\ndef g():\n    try:\n        yield from sub()\n    except IndexError:\n        print('idx')\n    finally:\n        print('fin')\nprint(list(g()))", "output": ["idx", "fin", "[1]"]},
+  {"src": "def sub():\n    yield 1\n    raise ValueError('inner')\ndef mid():\n    yield from sub()\ndef g():\n    try:\n        yield from mid()\n    except ValueError as e:\n        print('outer', e)\nprint(list(g()))", "output": ["outer inner", "[1]"]},
+  {"src": "def sub():\n    yield 1\n    raise ValueError\ndef g():\n    try:\n        yield from sub()\n    except ValueError:\n        yield 'recovered'\nprint(list(g()))", "output": ["[1, 'recovered']"]},
+  {"src": "def sub():\n    raise ValueError('x')\n    yield\ndef g():\n    caught = False\n    try:\n        yield from sub()\n    except ValueError:\n        caught = True\n    print(caught)\nprint(list(g()))", "output": ["True", "[]"]},
+  {"src": "def gen():\n    return 1\n    yield\nf = gen()\ndef run():\n    print((yield from f))\ntry:\n    next(run())\nexcept StopIteration:\n    print('SI')", "output": ["1", "SI"]},
+  {"src": "def sub():\n    yield 1\n    return 99\ndef g():\n    r = yield from sub()\n    print('r', r)\nprint(list(g()))", "output": ["r 99", "[1]"]},
+  {"src": "def sub():\n    return 'end'\n    yield\ndef g():\n    v = yield from sub()\n    print(v)\nprint(list(g()))", "output": ["end", "[]"]},
+  {"src": "def sub():\n    yield 1\n    yield 2\n    return 3\ndef g():\n    total = yield from sub()\n    print('total', total)\nprint(list(g()))", "output": ["total 3", "[1, 2]"]},
+  {"src": "def sub():\n    if False:\n        yield\n    return 5\ndef g():\n    print((yield from sub()))\nprint(list(g()))", "output": ["5", "[]"]},
+  {"src": "def sub():\n    yield 1\n    return\ndef g():\n    v = yield from sub()\n    print(v is None)\nprint(list(g()))", "output": ["True", "[1]"]},
+  {"src": "def sub():\n    return 10\n    yield\ndef mid():\n    x = yield from sub()\n    return x + 1\ndef g():\n    y = yield from mid()\n    print(y)\nprint(list(g()))", "output": ["11", "[]"]},
+  {"src": "def sub():\n    yield 'a'\n    return 'b'\ndef g():\n    r = yield from sub()\n    yield r\nprint(list(g()))", "output": ["['a', 'b']"]},
+  {"src": "def gen():\n    return 42\n    yield\ndef run():\n    v = yield from gen()\n    print('v', v)\ntry:\n    next(run())\nexcept StopIteration:\n    print('stop')", "output": ["v 42", "stop"]},
+  {"src": "def sub():\n    yield 1\n    return 2\ndef g():\n    a = yield from sub()\n    b = yield from sub()\n    print(a, b)\nprint(list(g()))", "output": ["2 2", "[1, 1]"]}
 ]