M6f: ordered stages on root/sort tuple steps (joins/index 6 & 12)

spec/joins_spec.lua

@@ -234,3 +234,37 @@ describe("M6e: nested tuple-stream detection (focus under sort/predicate)", func
     }, run("Account.Order#$o.Product^(ProductID).{ 'Product': `Product Name`, 'Order Index': $o }", dataset("dataset5")))
   end)
 end)
+
+describe("M6f: ordered stages on root/sort tuple steps", function()
+  local NUMS = { 3, 1, 4, 1, 5, 9 }
+
+  it("filter before #: $[[1..4]]#$pos[$pos>=2] re-indexes the survivors", function()
+    assert.are.same({ 1, 5 }, run("$[[1..4]]#$pos[$pos>=2]", NUMS))
+  end)
+
+  it("sort before #: $^($)#$pos[$pos<3] indexes the sorted sequence", function()
+    assert.are.same({ 1, 1, 3 }, run("$^($)#$pos[$pos<3]", NUMS))
+  end)
+
+  it("intermediate: $[[1..4]]#$pos collapses to the filtered values", function()
+    assert.are.same({ 1, 4, 1, 5 }, run("$[[1..4]]#$pos", NUMS))
+  end)
+
+  it("intermediate: $^($)#$pos collapses to the sorted values", function()
+    assert.are.same({ 1, 1, 3, 4, 5, 9 }, run("$^($)#$pos", NUMS))
+  end)
+
+  it("regression: natural-order $#$pos[$pos<3] still keeps the first three", function()
+    assert.are.same({ 3, 1, 4 }, run("$#$pos[$pos<3]", NUMS))
+  end)
+end)
+
+describe("M6f: sort index binds only on a raw (not tuple-bound) stream", function()
+  local NUMS = { 3, 1, 4, 1, 5, 9 }
+  it("sort-on-raw binds the index (index/6 still works)", function()
+    assert.are.same({ 1, 1, 3 }, run("$^($)#$pos[$pos<3]", NUMS))
+  end)
+  it("sort after a prior #-binding does NOT bind (double-index → undefined)", function()
+    assert.is_nil(run("$#$a^($)#$b[$b<2]", NUMS))
+  end)
+end)

spec/jsonata-suite/baseline.lua

@@ -840,13 +840,15 @@ return {
   ["joins/index/1"] = true,
   ["joins/index/10"] = true,
   ["joins/index/11"] = true,
+  ["joins/index/12"] = true,
   ["joins/index/13"] = true,
   ["joins/index/14"] = true,
   ["joins/index/15"] = true,
   ["joins/index/2"] = true,
   ["joins/index/3"] = true,
   ["joins/index/4"] = true,
   ["joins/index/5"] = true,
+  ["joins/index/6"] = true,
   ["joins/index/7"] = true,
   ["joins/index/8"] = true,
   ["joins/index/9"] = true,

src/jsonata/evaluator.lua

@@ -292,6 +292,21 @@ local function apply_predicates(seq, predicates, env, tuple_mode)
   return current
 end
 
+-- Apply an ordered stages list (jsonata evaluateStages) to a tuple stream:
+-- a filter runs the predicate; an index renumbers ALL surviving tuples 0-based.
+local function apply_stages(tuples, stages, env)
+  for _, stage in ipairs(stages) do
+    if stage.type == "filter" then
+      tuples = apply_predicates(tuples, { stage.expr }, env, true)
+    else -- "index"
+      for j = 1, #tuples do
+        tuples[j][stage.value] = j - 1
+      end
+    end
+  end
+  return tuples
+end
+
 -- Reorder a whole context sequence by one or more sort terms (the ^ operator).
 -- comp_after(a, b) is true when a should sort AFTER b, matching jsonata's
 -- evaluateSortExpression: per term, evaluate the key in each element's context;
@@ -659,6 +674,9 @@ local function eval_path_tuple(node, input, env, want_tuples)
         tuples[j][steps[1].focus] = tuples[j]["@"]
       end
     end
+    if steps[1].stages then
+      tuples = apply_stages(tuples, steps[1].stages, env)
+    end
     if steps[1].predicate then
       tuples = apply_predicates(tuples, steps[1].predicate, env, true)
     end
@@ -673,6 +691,27 @@ local function eval_path_tuple(node, input, env, want_tuples)
     local step = steps[i]
     if step.type == "sort" then
       tuples = eval_sort_step(tuples, step.terms, env, true)
+      -- jsonata binds a sort step's index ONLY when sorting a raw (not yet
+      -- tuple-bound) stream — its evaluateTupleStep sort case binds expr.index
+      -- in the `tupleBindings === undefined` branch only. If a prior step
+      -- already bound a focus/index (any tuple key beyond "@"), the index is
+      -- not bound, so e.g. `$#$a^($)#$b[$b<2]` yields undefined, not [1,1].
+      if step.index then
+        local raw = true
+        if tuples[1] then
+          for k in pairs(tuples[1]) do
+            if k ~= "@" then
+              raw = false
+              break
+            end
+          end
+        end
+        if raw then
+          for j = 1, #tuples do
+            tuples[j][step.index] = j - 1
+          end
+        end
+      end
     elseif step.type == "group" then
       -- jsonata propagates the tuple bindings ($e/$c/$i) into a `{` group-by
       -- (reduceTupleStream): pass the tuples through, not collapsed @-values.
@@ -731,7 +770,9 @@ local function eval_path_tuple(node, input, env, want_tuples)
       end
       tuples = next_tuples
     end
-    if step.predicate then
+    if step.stages then
+      tuples = apply_stages(tuples, step.stages, env)
+    elseif step.predicate then
       tuples = apply_predicates(tuples, step.predicate, env, true)
     end
   end

src/jsonata/parser.lua

@@ -687,15 +687,29 @@ local function mark_binding(step, node)
     step = step.steps[#step.steps]
   end
   if node.value == "@" then
-    if step.predicate ~= nil or step.keepArray then
+    if step.predicate ~= nil or step.stages ~= nil or step.keepArray then
       errors.raise("S0215", { position = node.position, token = "@" })
     end
     if step.type == "sort" then
       errors.raise("S0216", { position = node.position, token = "@" })
     end
     step.focus = node.rhs.value
-  else
-    step.index = node.rhs.value
+  else -- "#"
+    if step.stages then
+      step.stages[#step.stages + 1] = { type = "index", value = node.rhs.value }
+    elseif step.predicate then
+      -- a filter preceded this index: re-express the step's filters plus this
+      -- index as an ordered stages list so the index numbers the post-filter
+      -- survivors (jsonata processAST '#': move predicate -> stages, push index)
+      step.stages = {}
+      for _, f in ipairs(step.predicate) do
+        step.stages[#step.stages + 1] = { type = "filter", expr = f }
+      end
+      step.predicate = nil
+      step.stages[#step.stages + 1] = { type = "index", value = node.rhs.value }
+    else
+      step.index = node.rhs.value
+    end
   end
   step.tuple = true
 end