From cbb48f932aaaab6f760182d655f81c68c3d54a2c Mon Sep 17 00:00:00 2001 From: fl Date: Thu, 25 Jun 2026 10:51:45 +0800 Subject: [PATCH 1/3] feat(M6f): ordered stages on root/sort tuple steps (joins/index 6 & 12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mark_binding's # branch now moves a preceding filter into an ordered .stages list before pushing the index stage (jsonata move-on-#), so an index numbers the post-filter/post-sort survivors. The evaluator's apply_stages runs the stages in order in eval_path_tuple's seeding and main loop, and the sort branch binds its #-index. Gated on .stages/sort-.index — non-tuple paths and existing joins are untouched. Defers library-joins 7,8,10 (flat-stages cross-product rework). Co-Authored-By: Claude Opus 4.8 --- spec/joins_spec.lua | 24 ++++++++++++++++++++++++ src/jsonata/evaluator.lua | 27 ++++++++++++++++++++++++++- src/jsonata/parser.lua | 20 +++++++++++++++++--- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/spec/joins_spec.lua b/spec/joins_spec.lua index 9698148..c34263f 100644 --- a/spec/joins_spec.lua +++ b/spec/joins_spec.lua @@ -234,3 +234,27 @@ describe("M6e: nested tuple-stream detection (focus under sort/predicate)", func }, run("Account.Order#$o.Product^(ProductID).{ 'Product': `Product Name`, 'Order Index': $o }", dataset("dataset5"))) end) end) + +describe("M6f: ordered stages on root/sort tuple steps", function() + local NUMS = { 3, 1, 4, 1, 5, 9 } + + it("filter before #: $[[1..4]]#$pos[$pos>=2] re-indexes the survivors", function() + assert.are.same({ 1, 5 }, run("$[[1..4]]#$pos[$pos>=2]", NUMS)) + end) + + it("sort before #: $^($)#$pos[$pos<3] indexes the sorted sequence", function() + assert.are.same({ 1, 1, 3 }, run("$^($)#$pos[$pos<3]", NUMS)) + end) + + it("intermediate: $[[1..4]]#$pos collapses to the filtered values", function() + assert.are.same({ 1, 4, 1, 5 }, run("$[[1..4]]#$pos", NUMS)) + end) + + it("intermediate: $^($)#$pos collapses to the sorted values", function() + assert.are.same({ 1, 1, 3, 4, 5, 9 }, run("$^($)#$pos", NUMS)) + end) + + it("regression: natural-order $#$pos[$pos<3] still keeps the first three", function() + assert.are.same({ 3, 1, 4 }, run("$#$pos[$pos<3]", NUMS)) + end) +end) diff --git a/src/jsonata/evaluator.lua b/src/jsonata/evaluator.lua index 3cfff44..5809882 100644 --- a/src/jsonata/evaluator.lua +++ b/src/jsonata/evaluator.lua @@ -292,6 +292,21 @@ local function apply_predicates(seq, predicates, env, tuple_mode) return current end +-- Apply an ordered stages list (jsonata evaluateStages) to a tuple stream: +-- a filter runs the predicate; an index renumbers ALL surviving tuples 0-based. +local function apply_stages(tuples, stages, env) + for _, stage in ipairs(stages) do + if stage.type == "filter" then + tuples = apply_predicates(tuples, { stage.expr }, env, true) + else -- "index" + for j = 1, #tuples do + tuples[j][stage.value] = j - 1 + end + end + end + return tuples +end + -- Reorder a whole context sequence by one or more sort terms (the ^ operator). -- comp_after(a, b) is true when a should sort AFTER b, matching jsonata's -- evaluateSortExpression: per term, evaluate the key in each element's context; @@ -659,6 +674,9 @@ local function eval_path_tuple(node, input, env, want_tuples) tuples[j][steps[1].focus] = tuples[j]["@"] end end + if steps[1].stages then + tuples = apply_stages(tuples, steps[1].stages, env) + end if steps[1].predicate then tuples = apply_predicates(tuples, steps[1].predicate, env, true) end @@ -673,6 +691,11 @@ local function eval_path_tuple(node, input, env, want_tuples) local step = steps[i] if step.type == "sort" then tuples = eval_sort_step(tuples, step.terms, env, true) + if step.index then + for j = 1, #tuples do + tuples[j][step.index] = j - 1 + end + end elseif step.type == "group" then -- jsonata propagates the tuple bindings ($e/$c/$i) into a `{` group-by -- (reduceTupleStream): pass the tuples through, not collapsed @-values. @@ -731,7 +754,9 @@ local function eval_path_tuple(node, input, env, want_tuples) end tuples = next_tuples end - if step.predicate then + if step.stages then + tuples = apply_stages(tuples, step.stages, env) + elseif step.predicate then tuples = apply_predicates(tuples, step.predicate, env, true) end end diff --git a/src/jsonata/parser.lua b/src/jsonata/parser.lua index e179cdb..b9b9f19 100644 --- a/src/jsonata/parser.lua +++ b/src/jsonata/parser.lua @@ -687,15 +687,29 @@ local function mark_binding(step, node) step = step.steps[#step.steps] end if node.value == "@" then - if step.predicate ~= nil or step.keepArray then + if step.predicate ~= nil or step.stages ~= nil or step.keepArray then errors.raise("S0215", { position = node.position, token = "@" }) end if step.type == "sort" then errors.raise("S0216", { position = node.position, token = "@" }) end step.focus = node.rhs.value - else - step.index = node.rhs.value + else -- "#" + if step.stages then + step.stages[#step.stages + 1] = { type = "index", value = node.rhs.value } + elseif step.predicate then + -- a filter preceded this index: re-express the step's filters plus this + -- index as an ordered stages list so the index numbers the post-filter + -- survivors (jsonata processAST '#': move predicate -> stages, push index) + step.stages = {} + for _, f in ipairs(step.predicate) do + step.stages[#step.stages + 1] = { type = "filter", expr = f } + end + step.predicate = nil + step.stages[#step.stages + 1] = { type = "index", value = node.rhs.value } + else + step.index = node.rhs.value + end end step.tuple = true end From 51a32ca24cb7c9489ed9ad91f2be09421961abfb Mon Sep 17 00:00:00 2001 From: fl Date: Thu, 25 Jun 2026 10:55:21 +0800 Subject: [PATCH 2/3] =?UTF-8?q?test(M6f):=20regen=20official-suite=20basel?= =?UTF-8?q?ine=20=E2=80=94=20joins/index=206=20&=2012,=20zero=20regression?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 --- spec/jsonata-suite/baseline.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/jsonata-suite/baseline.lua b/spec/jsonata-suite/baseline.lua index 3a0bb41..c1aaee8 100644 --- a/spec/jsonata-suite/baseline.lua +++ b/spec/jsonata-suite/baseline.lua @@ -840,6 +840,7 @@ return { ["joins/index/1"] = true, ["joins/index/10"] = true, ["joins/index/11"] = true, + ["joins/index/12"] = true, ["joins/index/13"] = true, ["joins/index/14"] = true, ["joins/index/15"] = true, @@ -847,6 +848,7 @@ return { ["joins/index/3"] = true, ["joins/index/4"] = true, ["joins/index/5"] = true, + ["joins/index/6"] = true, ["joins/index/7"] = true, ["joins/index/8"] = true, ["joins/index/9"] = true, From 0183939e8beff8ba0c0017337f4a9b865c773f8a Mon Sep 17 00:00:00 2001 From: fl Date: Thu, 25 Jun 2026 11:04:02 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(M6f):=20bind=20sort=20index=20only=20on?= =?UTF-8?q?=20a=20raw=20stream=20=E2=80=94=20oracle=20fidelity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review found $#$a^($)#$b[$b<2] returned [1,1] vs jsonata undefined: jsonata's evaluateTupleStep binds a sort step's index ONLY when sorting a non-tuple-bound stream (the `tupleBindings === undefined` branch). Gate the sort-index binding on the incoming tuples carrying no key beyond "@"; index/6 ($^($)#$pos, a raw stream) still binds, the double-index case now yields undefined. Co-Authored-By: Claude Opus 4.8 --- spec/joins_spec.lua | 10 ++++++++++ src/jsonata/evaluator.lua | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/spec/joins_spec.lua b/spec/joins_spec.lua index c34263f..3647239 100644 --- a/spec/joins_spec.lua +++ b/spec/joins_spec.lua @@ -258,3 +258,13 @@ describe("M6f: ordered stages on root/sort tuple steps", function() assert.are.same({ 3, 1, 4 }, run("$#$pos[$pos<3]", NUMS)) end) end) + +describe("M6f: sort index binds only on a raw (not tuple-bound) stream", function() + local NUMS = { 3, 1, 4, 1, 5, 9 } + it("sort-on-raw binds the index (index/6 still works)", function() + assert.are.same({ 1, 1, 3 }, run("$^($)#$pos[$pos<3]", NUMS)) + end) + it("sort after a prior #-binding does NOT bind (double-index → undefined)", function() + assert.is_nil(run("$#$a^($)#$b[$b<2]", NUMS)) + end) +end) diff --git a/src/jsonata/evaluator.lua b/src/jsonata/evaluator.lua index 5809882..8608236 100644 --- a/src/jsonata/evaluator.lua +++ b/src/jsonata/evaluator.lua @@ -691,9 +691,25 @@ local function eval_path_tuple(node, input, env, want_tuples) local step = steps[i] if step.type == "sort" then tuples = eval_sort_step(tuples, step.terms, env, true) + -- jsonata binds a sort step's index ONLY when sorting a raw (not yet + -- tuple-bound) stream — its evaluateTupleStep sort case binds expr.index + -- in the `tupleBindings === undefined` branch only. If a prior step + -- already bound a focus/index (any tuple key beyond "@"), the index is + -- not bound, so e.g. `$#$a^($)#$b[$b<2]` yields undefined, not [1,1]. if step.index then - for j = 1, #tuples do - tuples[j][step.index] = j - 1 + local raw = true + if tuples[1] then + for k in pairs(tuples[1]) do + if k ~= "@" then + raw = false + break + end + end + end + if raw then + for j = 1, #tuples do + tuples[j][step.index] = j - 1 + end end end elseif step.type == "group" then