Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/compiler/ambient/scriptc-node-fallback.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1393,11 +1393,11 @@ declare module "crypto" {
export function randomUUID(): string;
export function randomBytes(size: number): Buffer;
/* The lowered Hash surface is exactly the COMPOSED chain
* createHash("sha1" | "sha256" | "sha384" | "sha512")
* createHash("md5" | "sha1" | "sha256" | "sha384" | "sha512")
* .update(data).digest("hex" | "base64")
* — fused into one call, the Hash handle never materializes (holding
* one fences). sha1 exists for the RFC 6455 Sec-WebSocket-Accept
* hash. */
* hash, md5 for ETags and cache keys. */
export interface Hash {
update(data: string | Uint8Array): Hash;
digest(encoding: "hex" | "base64"): string;
Expand Down
9 changes: 4 additions & 5 deletions packages/compiler/src/backend/emission/emit-exprs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { OVERFLOW_MEMBER } from "./emit-shapes.js";
import { dynDestrCheckHelper, dynIterNHelper, dynKeyGetHelper } from "./emit-walkers.js";
import { collectFfiRetainedOps, parseFfiCallbackKey } from "../ffi-callbacks.js";
import { genResultThunkFor } from "./emit-async.js";
import { emitReadlineNextLine } from "./emit-readline.js";
import { isStableBytesOperand, newValueMayThrow, streamTypedRefEligible, undefinedArmTag } from "../../ir/analysis.js";

function streamTypedRefCommitAdapter(
Expand Down Expand Up @@ -7013,11 +7014,9 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp {
return { name: "", type: e.type };
}
case "rl.nextLine":
// The native async-iterator slice currently belongs to the Rust
// runtime. Keep the C switch exhaustive while refusing an
// accidental C emission loudly instead of generating a wrong
// promise representation.
throw new InternalCompilerError("C emitter does not implement rl.nextLine yet");
// `for await (const line of rl)` — emit-readline.ts, which
// owns the interned `string | undefined` answer adapter.
return emitReadlineNextLine(E, e, arg(0));
// The StringDecoder trio (scr_bytes.c): pure functions over the
// canonical encoding name + packed-f64 pending state; never
// throw.
Expand Down
95 changes: 95 additions & 0 deletions packages/compiler/src/backend/emission/emit-readline.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/* The C emission of node:readline's ASYNC-ITERATOR slice —
* `for await (const line of rl)`, which the frontend lowers to a
* `rl.nextLine` libCall answering `Promise<string | undefined>`.
*
* Everything else in node:readline emits inline in emit-exprs.ts, because
* every other call is one runtime call with no shape to build.
* `rl.nextLine` is not: the runtime answers ONE line (+1) or NULL for
* "nothing more can arrive", and the promise it settles is a union whose
* two arm TAGS are program data. That needs an interned adapter per union
* shape — the raceAdapterFor stance — so it lives in its own file rather
* than growing emit-async.ts.
*/
import { InternalCompilerError } from "../../errors.js";
import type { IrExpr, IrType } from "../../ir/nodes.js";
import { typeKey } from "../../ir/nodes.js";
import { mangleReadlineNextThunk } from "../mangle.js";
import type { CEmitter } from "./emitter.js";
import { vAdapters } from "./emit-types.js";

/* The interned adapters, per emitter. They hang here rather than on
* CEmitter because this is the only file that reads them, and the
* emitter's own thunk registry is a frozen-size file; a WeakMap keyed by
* the emitter gives the same per-compilation lifetime with no shared
* state between compilations. */
const readlineNextThunks = new WeakMap<CEmitter, Map<string, string>>();

function thunkRegistry(E: CEmitter): Map<string, string> {
let registry = readlineNextThunks.get(E);
if (!registry) {
registry = new Map<string, string>();
readlineNextThunks.set(E, registry);
}
return registry;
}

/** Interned `rl.nextLine` answer adapter, one per result-union typeKey.
*
* The runtime hands back a line (+1) or NULL, and this fulfills the
* `string | undefined` promise the call site created. The closure is an
* ordinary RESOLVE closure (scr_make_resolve_fn): caps[0] holds that
* promise +1 and scr_resolve_ref_impl releases it — the `new Promise`
* machinery, with a readline answer where the executor's `resolve` would
* be. The undefined arm is the interned immortal unit instance (free, and
* releases skip it). */
export function readlineNextThunkFor(E: CEmitter, inner: IrType): string {
if (inner.kind !== "union") {
throw new InternalCompilerError("emitter bug: rl.nextLine result is not a union (frontend must fence)");
}
const def = E.unionsById.get(inner.unionId);
const stringTag = def ? def.arms.findIndex((arm) => arm.kind === "string") : -1;
const undefinedTag = def ? def.arms.findIndex((arm) => arm.kind === "undefinedT") : -1;
if (!def || def.arms.length !== 2 || stringTag < 0 || undefinedTag < 0) {
throw new InternalCompilerError("emitter bug: rl.nextLine result union is not `string | undefined`");
}
const registry = thunkRegistry(E);
const key = typeKey(inner);
const existing = registry.get(key);
if (existing) return existing;
const sym = mangleReadlineNextThunk(registry.size);
registry.set(key, sym);
const v = vAdapters(inner);
E.walkerProtos.push(`static void ${sym}(ScrClosure *sc_self, ScrStr *sc_line);`);
E.walkerDefs.push(
`static void ${sym}(ScrClosure *sc_self, ScrStr *sc_line) {`,
` ScrUnion *sc_u = sc_line`,
` ? scr_union_new_ref(${stringTag}, sc_line, scr_str_retain_v, scr_str_release_v, NULL)`,
` : ${E.unitInstanceRef(inner.unionId, undefinedTag)};`,
` scr_resolve_ref_impl(sc_self, sc_u, ${v.retain}, ${v.release}, ${E.traceArgC(inner)});`,
`}`,
);
return sym;
}

/** `rl.nextLine` itself: a fresh promise, handed to the runtime through
* the resolve closure the adapter above expects. Never throws — a closed
* interface answers undefined, which is how the `for await` loop ends —
* so no pending check follows. Answers the promise temporary's name. */
export function emitReadlineNextLine(
E: CEmitter,
expr: IrExpr & { kind: "libCall" },
handle: string,
): { name: string; type: IrType } {
if (expr.type.kind !== "promise") {
throw new InternalCompilerError("emitter bug: rl.nextLine result is not a promise");
}
// An open interface is a stdin consumer, so the loop must run.
E.usesTimers = true;
const adapter = readlineNextThunkFor(E, expr.type.inner);
const promise = E.newTemp(expr.type, `scr_promise_new()`);
E.line(
`scr_rl_next_line(${handle}, scr_make_resolve_fn(${promise.name}, (void *)&${adapter}), &${adapter});` +
E.srcComment(expr.loc),
);
return promise;
}
5 changes: 5 additions & 0 deletions packages/compiler/src/backend/mangle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,11 @@ export function mangleDnsLookupThunk(n: number): string {
export function mangleFsRenameThunk(n: number): string {
return `sc_fsren_${n}`;
}
/** Emitted readline async-iterator adapter (the `string | undefined`
* union's tags are program data), interned per result-union typeKey. */
export function mangleReadlineNextThunk(n: number): string {
return `sc_rlnext_${n}`;
}
/** Emitted SNI answer-closure thunk (the `(err, ctx?) => void` callback a
* TLS server's SNICallback receives — its unions' tags are program data),
* interned per cb func-type key. */
Expand Down
18 changes: 9 additions & 9 deletions packages/compiler/src/frontend/lowering/lower-builtins.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3986,19 +3986,19 @@ function optionMember(p: ts.ObjectLiteralElementLike): { name: string; value: ts
return { kind: "libCall", fn: "crypto.randomBytesToString", args: [size, enc], type: STRING, loc };
}

/** The SHA family the runtimes carry, for BOTH fused chains (createHash
* and createHmac) and the crypto.hash one-shot. sha1 exists for the RFC
* 6455 Sec-WebSocket-Accept hash; sha384/sha512 are the wider digests
* the token/signature idioms want. Every other name fences. */
const LOWERED_DIGEST_ALGORITHMS = ["sha1", "sha256", "sha384", "sha512"] as const;
/** The digests the runtimes carry, for BOTH fused chains (createHash and
* createHmac) and the crypto.hash one-shot. sha1 is the RFC 6455
* Sec-WebSocket-Accept hash, sha384/sha512 the wider token digests, md5 the
* ETag/cache-key checksum both runtimes write out by hand. Others fence. */
const LOWERED_DIGEST_ALGORITHMS = ["md5", "sha1", "sha256", "sha384", "sha512"] as const;

function isLoweredDigestAlgorithm(value: string): boolean {
return (LOWERED_DIGEST_ALGORITHMS as readonly string[]).includes(value);
}

const DIGEST_ALGORITHM_HINT =
'sha1, sha256, sha384, and sha512 are the lowered algorithms: createHash("sha256") ' +
"(sha1 exists for the RFC 6455 Sec-WebSocket-Accept hash)";
'md5, sha1, sha256, sha384, and sha512 are the lowered algorithms: createHash("sha256") ' +
"(sha1 exists for the RFC 6455 Sec-WebSocket-Accept hash, md5 for ETags and cache keys)";

/** The composed hash chain — `createHash("sha256").update(data).digest("hex")`
* — fused into ONE libCall: the Hash handle never materializes (no Hash
Expand Down Expand Up @@ -4113,7 +4113,7 @@ function optionMember(p: ts.ObjectLiteralElementLike): { name: string; value: ts
"createHmac with this algorithm",
chCall,
'the lowered shape is createHmac("sha256", key) — two arguments, a literal ' +
"algorithm (sha1, sha256, sha384, or sha512) and a string or Buffer key " +
"algorithm (md5, sha1, sha256, sha384, or sha512) and a string or Buffer key " +
"(KeyObjects have no lowering)",
);
}
Expand Down Expand Up @@ -4241,7 +4241,7 @@ function optionMember(p: ts.ObjectLiteralElementLike): { name: string; value: ts
L.noLowering(
"crypto.hash with this algorithm",
algorithmNode,
"sha1, sha256, sha384, and sha512 are the lowered one-shot algorithms",
"md5, sha1, sha256, sha384, and sha512 are the lowered one-shot algorithms",
);
}
const dataNode = expr.arguments[1]!;
Expand Down
6 changes: 3 additions & 3 deletions packages/compiler/src/frontend/lowering/surfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1365,14 +1365,14 @@ export const BUILTIN_MODULE_FENCE_HINTS: Record<string, Record<string, string |
createHash:
"the one lowered shape is the composed chain " +
'createHash("sha256").update(data).digest("hex") — the Hash handle itself has no lowering ' +
"(sha1, sha256, sha384, and sha512 are the lowered algorithms)",
"(md5, sha1, sha256, sha384, and sha512 are the lowered algorithms)",
hash:
"the lowered one-shot shapes hash SHA-1/SHA-256/SHA-384/SHA-512 strings or Buffer/Uint8Array inputs to default hex, " +
"the lowered one-shot shapes hash MD5/SHA-1/SHA-256/SHA-384/SHA-512 strings or Buffer/Uint8Array inputs to default hex, " +
'with explicit base64 also supported for strings; other algorithms and output options have no lowering yet',
createHmac:
"the one lowered shape is the composed chain " +
'createHmac("sha256", key).update(data).digest("hex") — the Hmac handle itself has no lowering ' +
"(sha1, sha256, sha384, and sha512 over string or Buffer keys; KeyObject keys have none)",
"(md5, sha1, sha256, sha384, and sha512 over string or Buffer keys; KeyObject keys have none)",
...Object.fromEntries(
[
"generateKeyPair", "generateKeyPairSync", "generateKey", "generateKeySync",
Expand Down
6 changes: 3 additions & 3 deletions packages/compiler/surface-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -576,15 +576,15 @@
"name": "crypto.createHash",
"status": "unsupported",
"code": "SC2020",
"note": "the one lowered shape is the composed chain createHash(\"sha256\").update(data).digest(\"hex\") — the Hash handle itself has no lowering (sha1, sha256, sha384, and sha512 are the lowered algorithms)"
"note": "the one lowered shape is the composed chain createHash(\"sha256\").update(data).digest(\"hex\") — the Hash handle itself has no lowering (md5, sha1, sha256, sha384, and sha512 are the lowered algorithms)"
},
{
"id": "node-builtin.crypto.createHmac",
"kind": "node-builtin",
"name": "crypto.createHmac",
"status": "unsupported",
"code": "SC2020",
"note": "the one lowered shape is the composed chain createHmac(\"sha256\", key).update(data).digest(\"hex\") — the Hmac handle itself has no lowering (sha1, sha256, sha384, and sha512 over string or Buffer keys; KeyObject keys have none)"
"note": "the one lowered shape is the composed chain createHmac(\"sha256\", key).update(data).digest(\"hex\") — the Hmac handle itself has no lowering (md5, sha1, sha256, sha384, and sha512 over string or Buffer keys; KeyObject keys have none)"
},
{
"id": "node-builtin.crypto.createPrivateKey",
Expand Down Expand Up @@ -688,7 +688,7 @@
"name": "crypto.hash",
"status": "unsupported",
"code": "SC2020",
"note": "the lowered one-shot shapes hash SHA-1/SHA-256/SHA-384/SHA-512 strings or Buffer/Uint8Array inputs to default hex, with explicit base64 also supported for strings; other algorithms and output options have no lowering yet"
"note": "the lowered one-shot shapes hash MD5/SHA-1/SHA-256/SHA-384/SHA-512 strings or Buffer/Uint8Array inputs to default hex, with explicit base64 also supported for strings; other algorithms and output options have no lowering yet"
},
{
"id": "node-builtin.crypto.hkdf",
Expand Down
148 changes: 148 additions & 0 deletions packages/compiler/test/emit-c-readline-next-line.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/* `for await (const line of rl)` in the C runtime, against Node.
*
* The corpus differential lanes close stdin immediately, so the only
* shape they can pin is "empty input, loop ends" (corpus 2794). The
* interesting halves of an async iterator are the ones that need real
* bytes on fd 0: lines already buffered when the loop asks, a partial
* last line, a close() landing mid-iteration, and a `question`
* interleaved with the iterator — where Node's own split shows up (onend
* emits 'line' DIRECTLY, so the iterator hears the leftover partial line
* and a pending question never does).
*
* Node IS the expectation here, exactly like the differential lanes: each
* program runs under Node and as a compiled binary over the same stdin
* bytes, and stdout must match byte for byte.
*/
import { spawn } from "node:child_process";
import { mkdtemp, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { NODE_COMPAT_MATRIX } from "../src/index.js";
import { primaryOracleExecutable } from "../../../tests/harness/node-matrix.js";
import { compile } from "../src/index.js";

// A SEMANTIC oracle: pinned to the compat matrix primary, never the host
// (tests/harness/node-matrix.ts explains why).
const oracleExecutable = primaryOracleExecutable(NODE_COMPAT_MATRIX);

interface RunResult {
stdout: string;
exitCode: number | null;
}

function run(file: string, args: string[], input: string): Promise<RunResult> {
return new Promise((settle, reject) => {
const child = spawn(file, args, { stdio: ["pipe", "pipe", "inherit"] });
let stdout = "";
child.stdout.setEncoding("utf8").on("data", (chunk: string) => { stdout += chunk; });
child.on("error", reject);
child.on("close", (exitCode) => settle({ stdout, exitCode }));
child.stdin.end(input);
});
}

/** Compiles `source` through the C backend and returns Node's output and
* the binary's over the same stdin bytes. */
async function bothLanes(name: string, source: string, input: string): Promise<[RunResult, RunResult]> {
const dir = await mkdtemp(join(tmpdir(), "scriptc-c-readline-"));
const entry = join(dir, `${name}.ts`);
await writeFile(entry, source, "utf8");
const result = await compile(entry, {
outDir: dir,
outPath: join(dir, name),
optimization: "dev",
});
expect(
result.ok,
result.ok ? entry : result.diagnostics.map((diagnostic) => diagnostic.message).join("; "),
).toBe(true);
if (!result.ok) throw new Error("unreachable: the compile assertion above failed");
return await Promise.all([
run(oracleExecutable, ["--experimental-strip-types", entry], input),
run(result.binaryPath, [], input),
]);
}

const ITERATE = `import { createInterface } from "node:readline";

async function main(): Promise<void> {
const lines = createInterface({ input: process.stdin, crlfDelay: Infinity });
for await (const line of lines) {
console.log("line", JSON.stringify(line));
}
console.log("done");
}

void main();
`;

const ITERATE_WITH_CLOSE_LISTENER = `import { createInterface } from "node:readline";

async function main(): Promise<void> {
const lines = createInterface({ input: process.stdin, crlfDelay: Infinity });
lines.on("close", () => { console.log("close"); });
for await (const line of lines) {
console.log("line", JSON.stringify(line));
}
console.log("done");
}

void main();
`;

const ITERATE_THEN_STOP = `import { createInterface } from "node:readline";

async function main(): Promise<void> {
const lines = createInterface({ input: process.stdin, crlfDelay: Infinity });
let seen = 0;
for await (const line of lines) {
seen += 1;
console.log("line", JSON.stringify(line));
if (seen === 2) {
lines.close();
console.log("closed");
}
}
console.log("done", seen);
}

void main();
`;

test.each([
["several whole lines", "one\\ntwo\\nthree\\n"],
// The last line has no terminator: Node's onend emits it as a 'line'
// anyway, so the iterator sees it before the loop ends.
["a partial last line", "one\\ntwo\\nthree"],
["one line, no terminator", "solo"],
// Empty input is corpus 2794's shape, kept here beside its neighbours.
["no input at all", ""],
["only terminators", "\\n\\n\\n"],
["CRLF terminators", "one\\r\\ntwo\\r\\n"],
["a held CR at the end", "one\\ntwo\\r"],
])("C readline for-await matches Node: %s", async (name, input) => {
const [node, native] = await bothLanes("iterate", ITERATE, input);
expect(native.stdout).toBe(node.stdout);
expect(native.exitCode).toBe(node.exitCode);
});

test("C readline for-await orders the close event like Node", async () => {
const [node, native] = await bothLanes(
"iterate_close_listener",
ITERATE_WITH_CLOSE_LISTENER,
"one\ntwo\nthree",
);
expect(native.stdout).toBe(node.stdout);
expect(native.exitCode).toBe(node.exitCode);
});

test("C readline for-await ends when the loop body closes the interface", async () => {
const [node, native] = await bothLanes(
"iterate_then_stop",
ITERATE_THEN_STOP,
"one\ntwo\nthree\nfour\n",
);
expect(native.stdout).toBe(node.stdout);
expect(native.exitCode).toBe(node.exitCode);
});
Loading