From 64958d474f0e9a72adf7d429a96697292d48d631 Mon Sep 17 00:00:00 2001 From: filipeforattini Date: Wed, 2 Sep 2026 08:36:14 -0300 Subject: [PATCH 1/2] =?UTF-8?q?feat(crypto):=20carrega=20MD5=20no=20runtim?= =?UTF-8?q?e=20Rust,=20sem=20depend=C3=AAncia=20nova?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ring` não tem MD5 e nunca vai ter — é uma biblioteca de primitivas modernas e MD5 está quebrado para todo uso de segurança. Node carrega MD5 mesmo assim, e o ecossistema publicado depende disso: ETag, chave de cache, fingerprint de conteúdo. Com o digest ausente, `createHash("md5")` cercava no frontend e a ilha respondia `undefined` no bridge, o que deixava o caso `crypto-shims` da suíte npm vermelho na lane rust. MD5 (RFC 1321) escrito à mão em `packages/runtime-rust/src/md5.rs` (140 linhas, sob o `#![forbid(unsafe_code)]` do crate): nenhuma dependência nova entra por causa de um checksum legado. O runtime C já soletra o mesmo digest em `scr_lib.c` — esse é o arquivo de referência semântica contra o qual este foi escrito, então as duas lanes concordam byte a byte. HMAC-MD5 (RFC 2104, bloco 64) vem junto porque `createHmac("md5", …)` existe no Node, não porque seja aconselhável. A ligação em `crypto.rs` não é um `if` no meio do caminho: as duas tabelas de algoritmo viraram enums (`CryptoDigest`, `CryptoHmac`) com um braço `Md5` e um braço `Ring`. Um único lugar decide qual implementação roda, então a cadeia estática fundida, o one-shot `crypto.hash` e o bridge da ilha concordam sobre a tabela por construção, em vez de por três listas que precisam ser mantidas em sincronia. O frontend destranca `md5` junto (`LOWERED_DIGEST_ALGORITHMS`), porque o runtime C já respondia md5 em `scr_crypto_digest_raw` — a cerca era a única coisa que faltava lá. Os hints de recusa, o `.d.ts` ambiente e o manifesto de superfície acompanham; nenhuma entrada mudou de classe, só as três notas. Provas: - `tests/corpus/2870-crypto-md5-digests.ts` (novo): md5 sobre string, Buffer e bytes crus, digests hex e base64, o one-shot, e HMAC-MD5 com chave curta, chave ASCII e chave de 80 bytes (mais longa que o bloco, então substituída pelo próprio digest). Node é o oráculo. Verde na lane C e na lane rust. - `cargo test`: os vetores do apêndice A.5 do RFC 1321 fixam o digest E o padding (o vetor alfanumérico de 62 bytes é o que precisa do segundo bloco), mais as três fronteiras de padding em torno de um bloco (55, 56, 64 bytes) e os casos 1, 2 e 6 do RFC 2202 para o HMAC. 141 testes passam. - `cargo clippy -- -D warnings`: limpo (o digest usa `as_chunks`, não `chunks_exact`, que o clippy 1.98 recusa com tamanho constante). --- .../ambient/scriptc-node-fallback.d.ts | 4 +- .../src/frontend/lowering/lower-builtins.ts | 18 +-- .../src/frontend/lowering/surfaces.ts | 6 +- packages/compiler/surface-manifest.json | 6 +- packages/runtime-rust/src/crypto.rs | 97 ++++++++---- packages/runtime-rust/src/island_host_io.rs | 8 +- packages/runtime-rust/src/lib.rs | 1 + packages/runtime-rust/src/md5.rs | 139 ++++++++++++++++++ packages/runtime-rust/src/tests/crypto.rs | 117 +++++++++++++++ packages/runtime/src/scr_lib.c | 10 +- packages/runtime/src/scr_runtime.h | 6 +- tests/corpus/2870-crypto-md5-digests.ts | 53 +++++++ 12 files changed, 408 insertions(+), 57 deletions(-) create mode 100644 packages/runtime-rust/src/md5.rs create mode 100644 tests/corpus/2870-crypto-md5-digests.ts diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index ee8793d20..e5343ef73 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -1393,11 +1393,11 @@ declare module "crypto" { export function randomUUID(): string; export function randomBytes(size: number): Buffer; /* The lowered Hash surface is exactly the COMPOSED chain - * createHash("sha1" | "sha256" | "sha384" | "sha512") + * createHash("md5" | "sha1" | "sha256" | "sha384" | "sha512") * .update(data).digest("hex" | "base64") * — fused into one call, the Hash handle never materializes (holding * one fences). sha1 exists for the RFC 6455 Sec-WebSocket-Accept - * hash. */ + * hash, md5 for ETags and cache keys. */ export interface Hash { update(data: string | Uint8Array): Hash; digest(encoding: "hex" | "base64"): string; diff --git a/packages/compiler/src/frontend/lowering/lower-builtins.ts b/packages/compiler/src/frontend/lowering/lower-builtins.ts index c7067fad2..e09cf4911 100644 --- a/packages/compiler/src/frontend/lowering/lower-builtins.ts +++ b/packages/compiler/src/frontend/lowering/lower-builtins.ts @@ -3986,19 +3986,19 @@ function optionMember(p: ts.ObjectLiteralElementLike): { name: string; value: ts return { kind: "libCall", fn: "crypto.randomBytesToString", args: [size, enc], type: STRING, loc }; } -/** The SHA family the runtimes carry, for BOTH fused chains (createHash - * and createHmac) and the crypto.hash one-shot. sha1 exists for the RFC - * 6455 Sec-WebSocket-Accept hash; sha384/sha512 are the wider digests - * the token/signature idioms want. Every other name fences. */ - const LOWERED_DIGEST_ALGORITHMS = ["sha1", "sha256", "sha384", "sha512"] as const; +/** The digests the runtimes carry, for BOTH fused chains (createHash and + * createHmac) and the crypto.hash one-shot. sha1 is the RFC 6455 + * Sec-WebSocket-Accept hash, sha384/sha512 the wider token digests, md5 the + * ETag/cache-key checksum both runtimes write out by hand. Others fence. */ + const LOWERED_DIGEST_ALGORITHMS = ["md5", "sha1", "sha256", "sha384", "sha512"] as const; function isLoweredDigestAlgorithm(value: string): boolean { return (LOWERED_DIGEST_ALGORITHMS as readonly string[]).includes(value); } const DIGEST_ALGORITHM_HINT = - 'sha1, sha256, sha384, and sha512 are the lowered algorithms: createHash("sha256") ' + - "(sha1 exists for the RFC 6455 Sec-WebSocket-Accept hash)"; + 'md5, sha1, sha256, sha384, and sha512 are the lowered algorithms: createHash("sha256") ' + + "(sha1 exists for the RFC 6455 Sec-WebSocket-Accept hash, md5 for ETags and cache keys)"; /** The composed hash chain — `createHash("sha256").update(data).digest("hex")` * — fused into ONE libCall: the Hash handle never materializes (no Hash @@ -4113,7 +4113,7 @@ function optionMember(p: ts.ObjectLiteralElementLike): { name: string; value: ts "createHmac with this algorithm", chCall, 'the lowered shape is createHmac("sha256", key) — two arguments, a literal ' + - "algorithm (sha1, sha256, sha384, or sha512) and a string or Buffer key " + + "algorithm (md5, sha1, sha256, sha384, or sha512) and a string or Buffer key " + "(KeyObjects have no lowering)", ); } @@ -4241,7 +4241,7 @@ function optionMember(p: ts.ObjectLiteralElementLike): { name: string; value: ts L.noLowering( "crypto.hash with this algorithm", algorithmNode, - "sha1, sha256, sha384, and sha512 are the lowered one-shot algorithms", + "md5, sha1, sha256, sha384, and sha512 are the lowered one-shot algorithms", ); } const dataNode = expr.arguments[1]!; diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index 37ad0b833..56834f829 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -1365,14 +1365,14 @@ export const BUILTIN_MODULE_FENCE_HINTS: Record JsString { string(&output) } -/// The four lowered digest algorithms, as a LOOKUP: `None` is "this +/// A digest this runtime carries, split by WHO computes it: `ring` for +/// the SHA family, this crate's own `md5.rs` for MD5, which `ring` +/// deliberately does not carry. +#[derive(Clone, Copy)] +enum CryptoDigest { + Md5, + Ring(&'static ring::digest::Algorithm), +} + +impl CryptoDigest { + /// The raw digest bytes. ONE place decides which implementation runs, + /// so every entry point below — the fused static chain, the one-shot, + /// the island bridge — agrees on the table by construction. + fn digest(self, data: &[u8]) -> Vec { + match self { + Self::Md5 => md5_digest(data).to_vec(), + Self::Ring(algorithm) => ring::digest::digest(algorithm, data).as_ref().to_vec(), + } + } +} + +/// The five lowered digest algorithms, as a LOOKUP: `None` is "this /// runtime has no such digest". /// /// The static lane never reaches the `None` arm — the frontend fences /// every other literal — but the island does: `createHash(alg)` takes a -/// runtime string, so the island needs to ask rather than assert. Node's -/// `md5` is deliberately absent (ring does not carry it), which is why -/// asking has to be possible at all. -fn crypto_digest_algorithm_opt(algorithm: &JsString) -> Option<&'static ring::digest::Algorithm> { +/// runtime string, so the island needs to ask rather than assert. +fn crypto_digest_algorithm_opt(algorithm: &JsString) -> Option { match algorithm.as_ref() { - "sha1" => Some(&ring::digest::SHA1_FOR_LEGACY_USE_ONLY), - "sha256" => Some(&ring::digest::SHA256), - "sha384" => Some(&ring::digest::SHA384), - "sha512" => Some(&ring::digest::SHA512), + "md5" => Some(CryptoDigest::Md5), + "sha1" => Some(CryptoDigest::Ring(&ring::digest::SHA1_FOR_LEGACY_USE_ONLY)), + "sha256" => Some(CryptoDigest::Ring(&ring::digest::SHA256)), + "sha384" => Some(CryptoDigest::Ring(&ring::digest::SHA384)), + "sha512" => Some(CryptoDigest::Ring(&ring::digest::SHA512)), _ => None, } } -/// The four lowered digest algorithms. The frontend fences every other +/// The five lowered digest algorithms. The frontend fences every other /// literal, so an unknown name here is a compiler invariant break. -fn crypto_digest_algorithm(algorithm: &JsString) -> &'static ring::digest::Algorithm { +fn crypto_digest_algorithm(algorithm: &JsString) -> CryptoDigest { crypto_digest_algorithm_opt(algorithm) .unwrap_or_else(|| unreachable!("scriptc invariant: unsupported hash algorithm reached the runtime")) } @@ -78,8 +98,8 @@ fn crypto_digest_algorithm(algorithm: &JsString) -> &'static ring::digest::Algor /// an unknown name must ANSWER here, never throw. pub fn crypto_digest_raw(algorithm: &JsString, data: &JsBytes) -> Option> { let algorithm = crypto_digest_algorithm_opt(algorithm)?; - let digest = crypto_with_bytes(data, |data| ring::digest::digest(algorithm, data)); - Some(bytes_from_vec(digest.as_ref().to_vec())) + let digest = crypto_with_bytes(data, |data| algorithm.digest(data)); + Some(bytes_from_vec(digest)) } /// `host.hmac(alg, key, bytes)`, with the same `None` fence as @@ -91,15 +111,14 @@ pub fn crypto_hmac_raw( ) -> Option> { let algorithm = crypto_hmac_algorithm_opt(algorithm)?; let tag = crypto_with_bytes(key, |key| { - let key = ring::hmac::Key::new(algorithm, key); - crypto_with_bytes(data, |data| ring::hmac::sign(&key, data)) + crypto_with_bytes(data, |data| algorithm.sign(key, data)) }); - Some(bytes_from_vec(tag.as_ref().to_vec())) + Some(bytes_from_vec(tag)) } fn crypto_hash_digest(algorithm: &JsString, data: &[u8], encoding: &JsString) -> JsString { - let digest = ring::digest::digest(crypto_digest_algorithm(algorithm), data); - decode_bytes(digest.as_ref(), encoding.as_ref()) + let digest = crypto_digest_algorithm(algorithm).digest(data); + decode_bytes(&digest, encoding.as_ref()) } /// Reads a Buffer/typed-array handle's bytes. Two handles can be read at @@ -111,18 +130,40 @@ fn crypto_with_bytes(data: &JsBytes, body: impl FnOnce(&[u8]) -> T) -> T }) } -/// The HMAC counterpart of `crypto_digest_algorithm_opt`. -fn crypto_hmac_algorithm_opt(algorithm: &JsString) -> Option { +/// The HMAC counterpart of `CryptoDigest`. `ring::hmac` picks the block +/// size from its digest; the MD5 arm carries its own RFC 2104 wrapping +/// (block 64, like sha1/sha256) because `ring` has no MD5 to hand it to. +#[derive(Clone, Copy)] +enum CryptoHmac { + Md5, + Ring(ring::hmac::Algorithm), +} + +impl CryptoHmac { + fn sign(self, key: &[u8], data: &[u8]) -> Vec { + match self { + Self::Md5 => md5_hmac(key, data).to_vec(), + Self::Ring(algorithm) => { + ring::hmac::sign(&ring::hmac::Key::new(algorithm, key), data).as_ref().to_vec() + } + } + } +} + +/// The HMAC counterpart of `crypto_digest_algorithm_opt` — the two tables +/// carry the same names. +fn crypto_hmac_algorithm_opt(algorithm: &JsString) -> Option { match algorithm.as_ref() { - "sha1" => Some(ring::hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY), - "sha256" => Some(ring::hmac::HMAC_SHA256), - "sha384" => Some(ring::hmac::HMAC_SHA384), - "sha512" => Some(ring::hmac::HMAC_SHA512), + "md5" => Some(CryptoHmac::Md5), + "sha1" => Some(CryptoHmac::Ring(ring::hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY)), + "sha256" => Some(CryptoHmac::Ring(ring::hmac::HMAC_SHA256)), + "sha384" => Some(CryptoHmac::Ring(ring::hmac::HMAC_SHA384)), + "sha512" => Some(CryptoHmac::Ring(ring::hmac::HMAC_SHA512)), _ => None, } } -fn crypto_hmac_algorithm(algorithm: &JsString) -> ring::hmac::Algorithm { +fn crypto_hmac_algorithm(algorithm: &JsString) -> CryptoHmac { crypto_hmac_algorithm_opt(algorithm) .unwrap_or_else(|| unreachable!("scriptc invariant: unsupported HMAC algorithm reached the runtime")) } @@ -136,10 +177,8 @@ fn crypto_hmac_digest( data: &[u8], encoding: &JsString, ) -> JsString { - let tag = crypto_with_bytes(key, |key| { - ring::hmac::sign(&ring::hmac::Key::new(crypto_hmac_algorithm(algorithm), key), data) - }); - decode_bytes(tag.as_ref(), encoding.as_ref()) + let tag = crypto_with_bytes(key, |key| crypto_hmac_algorithm(algorithm).sign(key, data)); + decode_bytes(&tag, encoding.as_ref()) } pub fn crypto_hmac_digest_string( diff --git a/packages/runtime-rust/src/island_host_io.rs b/packages/runtime-rust/src/island_host_io.rs index ab70c6eb7..35108678d 100644 --- a/packages/runtime-rust/src/island_host_io.rs +++ b/packages/runtime-rust/src/island_host_io.rs @@ -7,7 +7,7 @@ * FIRST, every call here delegates to the SAME runtime primitive the * static lane lowers to: the island's `fs.readFileSync` and a compiled * `fs.readFileSync` are both `fs_read_file_bytes`, its `createHash` and a - * compiled `createHash` are both ring through `crypto_digest_raw`. The + * compiled `createHash` are both `crypto_digest_raw`. The * island is a different engine, not a different runtime. * * SECOND, these primitives THROW, and a scriptc throw is an unwinding @@ -357,9 +357,9 @@ fn island_host_fs_constants( /// /// `undefined` is a RETURN, never a throw: the shared crypto shim probes /// with an empty input (`env.digest(alg, new Uint8Array(0)) === undefined`) -/// and raises Node's own "Digest method not supported" itself. Node's -/// `md5` takes that path here — ring does not carry it — so the island -/// refuses it out loud instead of answering wrongly. +/// and raises Node's own "Digest method not supported" itself. `md5` +/// ANSWERS — `md5.rs` carries it, since ring does not — so the fence is +/// reached only by a name neither runtime has (`sha3-256`, `blake2b512`). fn island_host_digest( _this: &JsValue, arguments: &[JsValue], diff --git a/packages/runtime-rust/src/lib.rs b/packages/runtime-rust/src/lib.rs index b507821f4..6bcf7f981 100644 --- a/packages/runtime-rust/src/lib.rs +++ b/packages/runtime-rust/src/lib.rs @@ -58,6 +58,7 @@ include!("bytes.rs"); include!("bytes_encoding.rs"); include!("zlib.rs"); include!("text_decoder.rs"); +include!("md5.rs"); include!("crypto.rs"); include!("collections.rs"); include!("event_emitter.rs"); diff --git a/packages/runtime-rust/src/md5.rs b/packages/runtime-rust/src/md5.rs new file mode 100644 index 000000000..44989e152 --- /dev/null +++ b/packages/runtime-rust/src/md5.rs @@ -0,0 +1,139 @@ +/* ── MD5 (RFC 1321) ──────────────────────────────────────────────────── + * + * `ring` deliberately carries no MD5 — it is a modern-primitives library + * and MD5 is broken for every security purpose. Node's `crypto` carries + * it anyway, and so must this runtime: the published ecosystem hashes + * ETags, cache keys and content fingerprints with it, so an island that + * cannot answer `createHash("md5")` cannot run npm code. This is the + * whole reason the digest is spelled out here instead of pulled in: the + * runtime takes no new dependency for a legacy checksum. + * + * The C runtime already spells the same digest out in `scr_lib.c` + * (`scr_md5_digest`) — that file is the semantic reference this one is + * matched against, so the two lanes agree byte for byte. + * + * MD5 is a checksum here, never a security primitive: nothing in this + * file is constant time, and callers must not use it for authentication. + * (`md5_hmac` exists because Node's `createHmac("md5", …)` exists, not + * because HMAC-MD5 is advisable.) + */ + +/// The 64 round constants, `floor(2^32 · |sin(i + 1)|)` for `i` in 0..64 +/// (RFC 1321 §3.4). +const MD5_K: [u32; 64] = [ + 0xd76a_a478, 0xe8c7_b756, 0x2420_70db, 0xc1bd_ceee, 0xf57c_0faf, 0x4787_c62a, + 0xa830_4613, 0xfd46_9501, 0x6980_98d8, 0x8b44_f7af, 0xffff_5bb1, 0x895c_d7be, + 0x6b90_1122, 0xfd98_7193, 0xa679_438e, 0x49b4_0821, 0xf61e_2562, 0xc040_b340, + 0x265e_5a51, 0xe9b6_c7aa, 0xd62f_105d, 0x0244_1453, 0xd8a1_e681, 0xe7d3_fbc8, + 0x21e1_cde6, 0xc337_07d6, 0xf4d5_0d87, 0x455a_14ed, 0xa9e3_e905, 0xfcef_a3f8, + 0x676f_02d9, 0x8d2a_4c8a, 0xfffa_3942, 0x8771_f681, 0x6d9d_6122, 0xfde5_380c, + 0xa4be_ea44, 0x4bde_cfa9, 0xf6bb_4b60, 0xbebf_bc70, 0x289b_7ec6, 0xeaa1_27fa, + 0xd4ef_3085, 0x0488_1d05, 0xd9d4_d039, 0xe6db_99e5, 0x1fa2_7cf8, 0xc4ac_5665, + 0xf429_2244, 0x432a_ff97, 0xab94_23a7, 0xfc93_a039, 0x655b_59c3, 0x8f0c_cc92, + 0xffef_f47d, 0x8584_5dd1, 0x6fa8_7e4f, 0xfe2c_e6e0, 0xa301_4314, 0x4e08_11a1, + 0xf753_7e82, 0xbd3a_f235, 0x2ad7_d2bb, 0xeb86_d391, +]; + +/// The per-round left-rotation amounts (RFC 1321 §3.4), four repeating +/// quadruples, one group of sixteen per round function. +const MD5_R: [u32; 64] = [ + 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, + 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, + 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, + 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, +]; + +/// The MD5 initial state (RFC 1321 §3.3), little-endian words. +const MD5_INITIAL_STATE: [u32; 4] = [0x6745_2301, 0xefcd_ab89, 0x98ba_dcfe, 0x1032_5476]; + +/// The size of one MD5 block, and the HMAC block size that goes with it. +const MD5_BLOCK: usize = 64; + +/// One 64-byte compression, folded into `state`. +/// +/// Every add wraps — MD5 is defined mod 2^32 — and the message words are +/// LITTLE-endian, which is the one place MD5 differs in shape from the +/// SHA family sitting beside it in the C runtime. +fn md5_block(state: &mut [u32; 4], block: &[u8; MD5_BLOCK]) { + let mut message = [0u32; 16]; + for (word, chunk) in message.iter_mut().zip(block.as_chunks::<4>().0) { + *word = u32::from_le_bytes(*chunk); + } + + let [mut a, mut b, mut c, mut d] = *state; + for (round, (&constant, &rotation)) in MD5_K.iter().zip(MD5_R.iter()).enumerate() { + let (mixed, index) = match round / 16 { + 0 => ((b & c) | (!b & d), round), + 1 => ((d & b) | (!d & c), (5 * round + 1) % 16), + 2 => (b ^ c ^ d, (3 * round + 5) % 16), + _ => (c ^ (b | !d), (7 * round) % 16), + }; + let sum = a + .wrapping_add(mixed) + .wrapping_add(constant) + .wrapping_add(message[index]); + a = d; + d = c; + c = b; + b = b.wrapping_add(sum.rotate_left(rotation)); + } + + state[0] = state[0].wrapping_add(a); + state[1] = state[1].wrapping_add(b); + state[2] = state[2].wrapping_add(c); + state[3] = state[3].wrapping_add(d); +} + +/// The MD5 digest of `data`, as its 16 raw bytes. +/// +/// The padding is the RFC's: one `0x80` byte, zeroes, then the message +/// length in BITS as a little-endian u64. That tail needs either one or +/// two more blocks depending on how close the `0x80` lands to the length +/// field, which is why the scratch buffer is two blocks wide. +fn md5_digest(data: &[u8]) -> [u8; 16] { + let mut state = MD5_INITIAL_STATE; + let (blocks, remainder) = data.as_chunks::(); + for block in blocks { + md5_block(&mut state, block); + } + + let mut tail = [0u8; MD5_BLOCK * 2]; + tail[..remainder.len()].copy_from_slice(remainder); + tail[remainder.len()] = 0x80; + let padded = if remainder.len() + 1 + 8 <= MD5_BLOCK { MD5_BLOCK } else { MD5_BLOCK * 2 }; + let bits = (data.len() as u64).wrapping_mul(8); + tail[padded - 8..padded].copy_from_slice(&bits.to_le_bytes()); + for block in tail[..padded].as_chunks::().0 { + md5_block(&mut state, block); + } + + let mut digest = [0u8; 16]; + for (out, word) in digest.as_chunks_mut::<4>().0.iter_mut().zip(state) { + *out = word.to_le_bytes(); + } + digest +} + +/// HMAC-MD5 (RFC 2104) over a 64-byte block, as its 16 raw tag bytes. +/// +/// `ring::hmac` has no MD5 algorithm to hand this to, so the construction +/// is spelled out too: a key longer than the block is replaced by its own +/// digest, a shorter one is zero-padded, and the tag is +/// `MD5(K^opad ‖ MD5(K^ipad ‖ data))`. +fn md5_hmac(key: &[u8], data: &[u8]) -> [u8; 16] { + let mut block_key = [0u8; MD5_BLOCK]; + if key.len() > MD5_BLOCK { + block_key[..16].copy_from_slice(&md5_digest(key)); + } else { + block_key[..key.len()].copy_from_slice(key); + } + + let mut inner = Vec::with_capacity(MD5_BLOCK + data.len()); + inner.extend(block_key.iter().map(|byte| byte ^ 0x36)); + inner.extend_from_slice(data); + + let mut outer = Vec::with_capacity(MD5_BLOCK + 16); + outer.extend(block_key.iter().map(|byte| byte ^ 0x5c)); + outer.extend_from_slice(&md5_digest(&inner)); + md5_digest(&outer) +} diff --git a/packages/runtime-rust/src/tests/crypto.rs b/packages/runtime-rust/src/tests/crypto.rs index eb7c6a393..a5042d392 100644 --- a/packages/runtime-rust/src/tests/crypto.rs +++ b/packages/runtime-rust/src/tests/crypto.rs @@ -55,6 +55,123 @@ ); } + /// MD5 is spelled out by hand in `md5.rs` (ring carries none), so it + /// gets the RFC's own acceptance suite rather than a spot check: RFC + /// 1321 A.5 pins the digest AND the padding, since the 62-byte + /// alphanumeric vector is the one that needs a second block. + #[test] + fn crypto_md5_digests_match_rfc_1321_vectors() { + let hex = string("hex"); + let md5 = string("md5"); + let digest = |input: &str| crypto_hash_digest_string(&md5, &string(input), &hex); + assert_eq!(digest("").as_ref(), "d41d8cd98f00b204e9800998ecf8427e"); + assert_eq!(digest("a").as_ref(), "0cc175b9c0f1b6a831c399e269772661"); + assert_eq!(digest("abc").as_ref(), "900150983cd24fb0d6963f7d28e17f72"); + assert_eq!( + digest("message digest").as_ref(), + "f96b697d7cb7938d525a2f31aaf161d0" + ); + assert_eq!( + digest("abcdefghijklmnopqrstuvwxyz").as_ref(), + "c3fcd3d76192e4007dfb496cca67e13b" + ); + assert_eq!( + digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789").as_ref(), + "d174ab98d277d9f5a5611c2c9f419d9f" + ); + assert_eq!( + digest(&"1234567890".repeat(8)).as_ref(), + "57edf4a22be3c955ac49da2e2107b67a" + ); + + // The three padding boundaries around one block: 55 bytes fits + // the 0x80 and the length field, 56 pushes the length into a + // second block, 64 needs a whole second block of padding. + assert_eq!( + digest(&"x".repeat(55)).as_ref(), + "04364420e25c512fd958a70738aa8f72" + ); + assert_eq!( + digest(&"x".repeat(56)).as_ref(), + "668a72d5ba17f08e62dabcafad6db14b" + ); + assert_eq!( + digest(&"x".repeat(64)).as_ref(), + "c1bb4f81d892b2d57947682aeb252456" + ); + + // Bytes and base64, the other two entry shapes. + assert_eq!( + crypto_hash_digest_bytes(&md5, &bytes_from_vec(b"abc".to_vec()), &hex).as_ref(), + "900150983cd24fb0d6963f7d28e17f72", + ); + assert_eq!( + crypto_hash_digest_bytes( + &md5, + &bytes_from_vec(vec![0, 1, 2, 253, 254, 255]), + &string("base64"), + ) + .as_ref(), + "5yuGRWwZHDJ149VcCrLnVg==", + ); + + // The island bridge answers md5 rather than fencing it, which is + // what lets npm's ETag and cache-key code run here. + let raw = crypto_digest_raw(&md5, &bytes_from_vec(b"abc".to_vec())) + .expect("md5 must be a carried digest"); + assert_eq!(bytes_len(&raw), 16.0); + assert!(crypto_digest_raw(&string("sha3-256"), &bytes_from_vec(vec![])).is_none()); + } + + /// HMAC-MD5 (RFC 2202) — the construction is hand-written too, so the + /// short, exact-block and over-long key cases each get a vector. + #[test] + fn crypto_hmac_md5_matches_rfc_2202_vectors() { + let hex = string("hex"); + let md5 = string("md5"); + assert_eq!( + crypto_hmac_digest_string( + &md5, + &bytes_from_vec(vec![0x0b; 16]), + &string("Hi There"), + &hex, + ) + .as_ref(), + "9294727a3638bb1c13f48ef8158bfc9d", + ); + assert_eq!( + crypto_hmac_digest_string( + &md5, + &bytes_from_vec(b"Jefe".to_vec()), + &string("what do ya want for nothing?"), + &hex, + ) + .as_ref(), + "750c783e6ab0b503eaa86e310a5db738", + ); + // Test case 6: an 80-byte key, longer than the 64-byte block, so + // the key is replaced by its own digest first. + assert_eq!( + crypto_hmac_digest_bytes( + &md5, + &bytes_from_vec(vec![0xaa; 80]), + &bytes_from_vec( + b"Test Using Larger Than Block-Size Key - Hash Key First".to_vec() + ), + &hex, + ) + .as_ref(), + "6b1ab7fe4bd7bf8f0b62e6ce61b9d0cd", + ); + let tag = crypto_hmac_raw( + &md5, + &bytes_from_vec(b"key".to_vec()), + &bytes_from_vec(b"msg".to_vec()), + ) + .expect("md5 must be a carried HMAC"); + assert_eq!(bytes_len(&tag), 16.0); + } + #[test] fn crypto_hmac_digests_match_rfc_4231_vectors() { // RFC 4231 test case 1: a 20-byte 0x0b key over "Hi There". diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 850167763..f1b0986da 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -3689,9 +3689,11 @@ static ScrStr *scr_digest_encode(const unsigned char *d, size_t n, const ScrStr return scr_str_new(buf, o); } -/* ── MD5 (RFC 1321) — island npm code only (the static frontend fences - * every non-SHA algorithm literal; published packages hash cache keys and - * etags with md5, so the island's createHash carries it). ─────────── */ +/* ── MD5 (RFC 1321) — the ETag/cache-key checksum the ecosystem still + * spells everywhere, reached from BOTH the island's createHash and the + * fused static chains. Broken for every security purpose, and nothing + * here is constant time: it is a checksum, never an authenticator. The + * Rust runtime spells the same digest out in md5.rs, byte for byte. ── */ static const uint32_t scr_md5_k[64] = { 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, @@ -3923,7 +3925,7 @@ static ScrStr *scr_hash_digest_raw(const ScrStr *alg, const unsigned char *data, unsigned char d[64]; char name[16]; scr_digest_alg_name(alg, name); - /* sha1/sha256/sha384/sha512 — every other literal is frontend-fenced. */ + /* md5/sha1/sha256/sha384/sha512 — every other literal is frontend-fenced. */ size_t n = scr_crypto_digest_raw(name, data, len, d); return scr_digest_encode(d, n, enc); } diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 3d7384542..80cdf376f 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -2628,10 +2628,10 @@ bool scr_children_wait(double max_wait_ms); ScrStr *scr_crypto_random_uuid(void); ScrStr *scr_crypto_random_string(double n, ScrStr *enc); /* +1, or throws */ /* The composed createHash(alg).update(data).digest(enc) chain, fused by - * the compiler (no Hash handle exists). alg is "sha1" | "sha256" | - * "sha384" | "sha512" and enc "hex" | "base64" — compile-time literals, + * the compiler (no Hash handle exists). alg is "md5" | "sha1" | "sha256" + * | "sha384" | "sha512" and enc "hex" | "base64" — compile-time literals, * frontend-fenced (sha1 exists for the RFC 6455 Sec-WebSocket-Accept - * hash). Strings hash their UTF-8 bytes (Node's default input encoding; + * hash, md5 for ETags and cache keys). Strings hash their UTF-8 bytes (Node's default input encoding; * ScrStr storage IS utf8), the bytes form a Buffer/typed array's raw * bytes. Borrowed; +1 string. Never throw. */ ScrStr *scr_crypto_hash_digest_str(ScrStr *alg, ScrStr *data, ScrStr *enc); diff --git a/tests/corpus/2870-crypto-md5-digests.ts b/tests/corpus/2870-crypto-md5-digests.ts new file mode 100644 index 000000000..ecec251fa --- /dev/null +++ b/tests/corpus/2870-crypto-md5-digests.ts @@ -0,0 +1,53 @@ +// MD5 through every lowered crypto surface: the fused createHash chain +// over strings and Buffers, both digest encodings, the crypto.hash +// one-shot, and HMAC-MD5 (RFC 2104 block 64) over string and Buffer keys. +// The first three digests are the RFC 1321 A.5 test-suite vectors, which +// pin the padding: "" and "abc" take one block, the 62-byte alphanumeric +// string takes two (its 0x80 lands past the length field). +import { createHash, createHmac, hash } from "node:crypto"; + +const ALPHANUMERIC = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + +console.log(createHash("md5").update("").digest("hex")); +console.log(createHash("md5").update("abc").digest("hex")); +console.log(createHash("md5").update(ALPHANUMERIC).digest("hex")); + +// A 64-byte input: exactly one block of message, so the padding needs a +// whole SECOND block on its own. +console.log(createHash("md5").update("x".repeat(64)).digest("hex")); +console.log(createHash("md5").update("x".repeat(119)).digest("hex")); +console.log(createHash("md5").update("x".repeat(120)).digest("hex")); + +// Non-ASCII hashes its UTF-8 bytes, which is Node's default input encoding. +console.log(createHash("md5").update("café ☕").digest("hex")); +console.log(createHash("md5").update("").digest("base64")); +console.log(createHash("md5").update("scriptc").digest("base64")); + +// Buffer input, and a Buffer holding raw bytes rather than text. +console.log(createHash("md5").update(Buffer.from("abc", "utf8")).digest("hex")); +console.log( + createHash("md5").update(Buffer.from([0, 1, 2, 253, 254, 255])).digest("hex"), +); +console.log(createHash("md5").update(Buffer.from("ff00ff", "hex")).digest("base64")); + +// The one-shot, which shares the runtime's digest table with the chain. +console.log(hash("md5", "abc")); +console.log(hash("md5", Buffer.from("abc", "utf8"))); + +// HMAC-MD5, RFC 2202 test cases 1, 2 and 6: a 16-byte 0x0b key, a short +// ASCII key, and an 80-byte key — longer than the 64-byte block, so the +// key is replaced by its own digest. +console.log( + createHmac("md5", Buffer.alloc(16, 0x0b)).update("Hi There").digest("hex"), +); +console.log( + createHmac("md5", "Jefe").update("what do ya want for nothing?").digest("hex"), +); +console.log( + createHmac("md5", Buffer.alloc(80, 0xaa)) + .update("Test Using Larger Than Block-Size Key - Hash Key First") + .digest("hex"), +); +console.log(createHmac("md5", "key").update(Buffer.from("msg")).digest("base64")); +console.log(createHmac("md5", Buffer.alloc(64, 0x41)).update("").digest("hex")); From 3ad234643c6471915881a9e08814674acd6d928e Mon Sep 17 00:00:00 2001 From: filipeforattini Date: Wed, 2 Sep 2026 08:36:54 -0300 Subject: [PATCH 2/2] feat(readline): porta rl.nextLine para o runtime C MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `for await (const line of rl)` só existia no backend Rust; o emissor C recusava de propósito ("the native async-iterator slice currently belongs to the Rust runtime"), o que deixava a fixture 2794 vermelha na lane C e, por queda no fallback C, também na LLVM. Uma implementação em C resolve as duas. O plano de duas partes está no corpo do PR #14; as duas partes entram aqui. ## (a) o modo waiter em scr_readline.c O slot de callback pendente de `question` tinha quase a forma certa, mas faltava o modo "próxima linha OU fim": `scr_rl_settle_close` DESCARTAVA um callback pendente (a `question` do Node de fato nunca responde), enquanto `nextLine` precisa resolver `undefined` no fim, e resolver de novo em toda chamada seguinte para o laço terminar. O que o plano não previa, e as sondas contra o Node mostraram, é que `nextLine` não é só "question que também responde undefined" — é o único consumidor que SOBREVIVE às suas linhas. O `for await` do Node é um iterador `EventEmitter.on('line')`, ou seja, um LISTENER: uma linha parseada enquanto o corpo do laço roda (entre um await settlar e o `nextLine` seguinte) fica bufferizada, não se perde. Uma `question` não tem essa fila — entre duas perguntas nada escuta, que é exatamente por que uma linha não reclamada continua caindo numa interface só de `question`. Então a interface ganhou uma fila de linhas, ligada na primeira vez que um programa pede `nextLine`. A segunda descoberta é o EOF, e ela explica uma assimetria que parecia bug do nosso lado. Com `printf 'a\nb' | node`, o `for await` entrega `"b"` e uma `question` pendente NÃO recebe nada. O motivo está no `lib/internal/readline/interface.js`: o `onend` emite `'line'` DIRETAMENTE, enquanto o caminho por chunk passa por `[kOnLine]` — e só `[kOnLine]` responde uma `question`. Ou seja, a linha parcial final é do iterador e não da pergunta. O `\r` retido continua sendo a exceção: ele é um terminador de verdade cuja decisão `\r\n` expirou, veio pelo caminho comum, e a `question` responde. ## (b) o adaptador de promise no emissor C Seguindo o precedente `raceAdapterFor`, mas com a máquina de `resolve` que já existe: o call site cria `scr_promise_new()` e entrega ao runtime uma closure de resolve comum (`scr_make_resolve_fn`), cujo caps[0] segura a promise +1 e cujo `scr_resolve_ref_impl` a libera. O adaptador internado por union monta o braço `string` da linha respondida, ou o braço `undefined` (a instância unitária imortal internada) no fim, e cumpre. Uma tag de arm é dado do programa, então a variabilidade inteira é uma thunk por forma de union. Isso vive em `emit-readline.ts`, arquivo novo, e não dentro de `emit-async.ts`/`emitter.ts`: os dois estão no teto de dívida congelado do `check-rust-file-lines`, e engordá-los é justamente o que aquele gate proíbe. O teto de `emit-exprs.ts` desce de 7.964 para 7.963 pela extração. ## Provas - 2794-readline-async-iterator: **verde na lane C** e **verde na LLVM** (que cai no fallback C). Era o 13º vermelho do differential. - `packages/compiler/test/emit-c-readline-next-line.test.ts` (novo): as lanes do corpus fecham o stdin na hora, então a única forma que elas fixam é "entrada vazia, o laço termina". As metades interessantes de um iterador assíncrono precisam de bytes reais no fd 0, e este teste fornece: linhas inteiras, linha final parcial (a que só o iterador vê), uma linha sem terminador nenhum, entrada vazia, só terminadores, CRLF, `\r` retido no fim, a ordem do evento 'close', e um `close()` chamado de dentro do corpo do laço. Node é a expectativa em todos. 9/9 passam. - `pnpm lint` (line caps + island bootstrap + eslint): 0 erros. --- .../src/backend/emission/emit-exprs.ts | 9 +- .../src/backend/emission/emit-readline.ts | 95 +++++++++ packages/compiler/src/backend/mangle.ts | 5 + .../test/emit-c-readline-next-line.test.ts | 148 ++++++++++++++ packages/runtime/src/scr_readline.c | 181 ++++++++++++++++-- packages/runtime/src/scr_runtime.h | 11 +- scripts/check-rust-file-lines.mjs | 2 +- 7 files changed, 423 insertions(+), 28 deletions(-) create mode 100644 packages/compiler/src/backend/emission/emit-readline.ts create mode 100644 packages/compiler/test/emit-c-readline-next-line.test.ts diff --git a/packages/compiler/src/backend/emission/emit-exprs.ts b/packages/compiler/src/backend/emission/emit-exprs.ts index f9221c5d0..0bcc290f7 100644 --- a/packages/compiler/src/backend/emission/emit-exprs.ts +++ b/packages/compiler/src/backend/emission/emit-exprs.ts @@ -10,6 +10,7 @@ import { OVERFLOW_MEMBER } from "./emit-shapes.js"; import { dynDestrCheckHelper, dynIterNHelper, dynKeyGetHelper } from "./emit-walkers.js"; import { collectFfiRetainedOps, parseFfiCallbackKey } from "../ffi-callbacks.js"; import { genResultThunkFor } from "./emit-async.js"; +import { emitReadlineNextLine } from "./emit-readline.js"; import { isStableBytesOperand, newValueMayThrow, streamTypedRefEligible, undefinedArmTag } from "../../ir/analysis.js"; function streamTypedRefCommitAdapter( @@ -7013,11 +7014,9 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp { return { name: "", type: e.type }; } case "rl.nextLine": - // The native async-iterator slice currently belongs to the Rust - // runtime. Keep the C switch exhaustive while refusing an - // accidental C emission loudly instead of generating a wrong - // promise representation. - throw new InternalCompilerError("C emitter does not implement rl.nextLine yet"); + // `for await (const line of rl)` — emit-readline.ts, which + // owns the interned `string | undefined` answer adapter. + return emitReadlineNextLine(E, e, arg(0)); // The StringDecoder trio (scr_bytes.c): pure functions over the // canonical encoding name + packed-f64 pending state; never // throw. diff --git a/packages/compiler/src/backend/emission/emit-readline.ts b/packages/compiler/src/backend/emission/emit-readline.ts new file mode 100644 index 000000000..24ff79893 --- /dev/null +++ b/packages/compiler/src/backend/emission/emit-readline.ts @@ -0,0 +1,95 @@ +/* The C emission of node:readline's ASYNC-ITERATOR slice — + * `for await (const line of rl)`, which the frontend lowers to a + * `rl.nextLine` libCall answering `Promise`. + * + * Everything else in node:readline emits inline in emit-exprs.ts, because + * every other call is one runtime call with no shape to build. + * `rl.nextLine` is not: the runtime answers ONE line (+1) or NULL for + * "nothing more can arrive", and the promise it settles is a union whose + * two arm TAGS are program data. That needs an interned adapter per union + * shape — the raceAdapterFor stance — so it lives in its own file rather + * than growing emit-async.ts. + */ +import { InternalCompilerError } from "../../errors.js"; +import type { IrExpr, IrType } from "../../ir/nodes.js"; +import { typeKey } from "../../ir/nodes.js"; +import { mangleReadlineNextThunk } from "../mangle.js"; +import type { CEmitter } from "./emitter.js"; +import { vAdapters } from "./emit-types.js"; + +/* The interned adapters, per emitter. They hang here rather than on + * CEmitter because this is the only file that reads them, and the + * emitter's own thunk registry is a frozen-size file; a WeakMap keyed by + * the emitter gives the same per-compilation lifetime with no shared + * state between compilations. */ +const readlineNextThunks = new WeakMap>(); + +function thunkRegistry(E: CEmitter): Map { + let registry = readlineNextThunks.get(E); + if (!registry) { + registry = new Map(); + readlineNextThunks.set(E, registry); + } + return registry; +} + +/** Interned `rl.nextLine` answer adapter, one per result-union typeKey. + * + * The runtime hands back a line (+1) or NULL, and this fulfills the + * `string | undefined` promise the call site created. The closure is an + * ordinary RESOLVE closure (scr_make_resolve_fn): caps[0] holds that + * promise +1 and scr_resolve_ref_impl releases it — the `new Promise` + * machinery, with a readline answer where the executor's `resolve` would + * be. The undefined arm is the interned immortal unit instance (free, and + * releases skip it). */ +export function readlineNextThunkFor(E: CEmitter, inner: IrType): string { + if (inner.kind !== "union") { + throw new InternalCompilerError("emitter bug: rl.nextLine result is not a union (frontend must fence)"); + } + const def = E.unionsById.get(inner.unionId); + const stringTag = def ? def.arms.findIndex((arm) => arm.kind === "string") : -1; + const undefinedTag = def ? def.arms.findIndex((arm) => arm.kind === "undefinedT") : -1; + if (!def || def.arms.length !== 2 || stringTag < 0 || undefinedTag < 0) { + throw new InternalCompilerError("emitter bug: rl.nextLine result union is not `string | undefined`"); + } + const registry = thunkRegistry(E); + const key = typeKey(inner); + const existing = registry.get(key); + if (existing) return existing; + const sym = mangleReadlineNextThunk(registry.size); + registry.set(key, sym); + const v = vAdapters(inner); + E.walkerProtos.push(`static void ${sym}(ScrClosure *sc_self, ScrStr *sc_line);`); + E.walkerDefs.push( + `static void ${sym}(ScrClosure *sc_self, ScrStr *sc_line) {`, + ` ScrUnion *sc_u = sc_line`, + ` ? scr_union_new_ref(${stringTag}, sc_line, scr_str_retain_v, scr_str_release_v, NULL)`, + ` : ${E.unitInstanceRef(inner.unionId, undefinedTag)};`, + ` scr_resolve_ref_impl(sc_self, sc_u, ${v.retain}, ${v.release}, ${E.traceArgC(inner)});`, + `}`, + ); + return sym; +} + +/** `rl.nextLine` itself: a fresh promise, handed to the runtime through + * the resolve closure the adapter above expects. Never throws — a closed + * interface answers undefined, which is how the `for await` loop ends — + * so no pending check follows. Answers the promise temporary's name. */ +export function emitReadlineNextLine( + E: CEmitter, + expr: IrExpr & { kind: "libCall" }, + handle: string, +): { name: string; type: IrType } { + if (expr.type.kind !== "promise") { + throw new InternalCompilerError("emitter bug: rl.nextLine result is not a promise"); + } + // An open interface is a stdin consumer, so the loop must run. + E.usesTimers = true; + const adapter = readlineNextThunkFor(E, expr.type.inner); + const promise = E.newTemp(expr.type, `scr_promise_new()`); + E.line( + `scr_rl_next_line(${handle}, scr_make_resolve_fn(${promise.name}, (void *)&${adapter}), &${adapter});` + + E.srcComment(expr.loc), + ); + return promise; +} diff --git a/packages/compiler/src/backend/mangle.ts b/packages/compiler/src/backend/mangle.ts index 9e574190c..5ec7d45b3 100644 --- a/packages/compiler/src/backend/mangle.ts +++ b/packages/compiler/src/backend/mangle.ts @@ -212,6 +212,11 @@ export function mangleDnsLookupThunk(n: number): string { export function mangleFsRenameThunk(n: number): string { return `sc_fsren_${n}`; } +/** Emitted readline async-iterator adapter (the `string | undefined` + * union's tags are program data), interned per result-union typeKey. */ +export function mangleReadlineNextThunk(n: number): string { + return `sc_rlnext_${n}`; +} /** Emitted SNI answer-closure thunk (the `(err, ctx?) => void` callback a * TLS server's SNICallback receives — its unions' tags are program data), * interned per cb func-type key. */ diff --git a/packages/compiler/test/emit-c-readline-next-line.test.ts b/packages/compiler/test/emit-c-readline-next-line.test.ts new file mode 100644 index 000000000..11f76f9dc --- /dev/null +++ b/packages/compiler/test/emit-c-readline-next-line.test.ts @@ -0,0 +1,148 @@ +/* `for await (const line of rl)` in the C runtime, against Node. + * + * The corpus differential lanes close stdin immediately, so the only + * shape they can pin is "empty input, loop ends" (corpus 2794). The + * interesting halves of an async iterator are the ones that need real + * bytes on fd 0: lines already buffered when the loop asks, a partial + * last line, a close() landing mid-iteration, and a `question` + * interleaved with the iterator — where Node's own split shows up (onend + * emits 'line' DIRECTLY, so the iterator hears the leftover partial line + * and a pending question never does). + * + * Node IS the expectation here, exactly like the differential lanes: each + * program runs under Node and as a compiled binary over the same stdin + * bytes, and stdout must match byte for byte. + */ +import { spawn } from "node:child_process"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { NODE_COMPAT_MATRIX } from "../src/index.js"; +import { primaryOracleExecutable } from "../../../tests/harness/node-matrix.js"; +import { compile } from "../src/index.js"; + +// A SEMANTIC oracle: pinned to the compat matrix primary, never the host +// (tests/harness/node-matrix.ts explains why). +const oracleExecutable = primaryOracleExecutable(NODE_COMPAT_MATRIX); + +interface RunResult { + stdout: string; + exitCode: number | null; +} + +function run(file: string, args: string[], input: string): Promise { + return new Promise((settle, reject) => { + const child = spawn(file, args, { stdio: ["pipe", "pipe", "inherit"] }); + let stdout = ""; + child.stdout.setEncoding("utf8").on("data", (chunk: string) => { stdout += chunk; }); + child.on("error", reject); + child.on("close", (exitCode) => settle({ stdout, exitCode })); + child.stdin.end(input); + }); +} + +/** Compiles `source` through the C backend and returns Node's output and + * the binary's over the same stdin bytes. */ +async function bothLanes(name: string, source: string, input: string): Promise<[RunResult, RunResult]> { + const dir = await mkdtemp(join(tmpdir(), "scriptc-c-readline-")); + const entry = join(dir, `${name}.ts`); + await writeFile(entry, source, "utf8"); + const result = await compile(entry, { + outDir: dir, + outPath: join(dir, name), + optimization: "dev", + }); + expect( + result.ok, + result.ok ? entry : result.diagnostics.map((diagnostic) => diagnostic.message).join("; "), + ).toBe(true); + if (!result.ok) throw new Error("unreachable: the compile assertion above failed"); + return await Promise.all([ + run(oracleExecutable, ["--experimental-strip-types", entry], input), + run(result.binaryPath, [], input), + ]); +} + +const ITERATE = `import { createInterface } from "node:readline"; + +async function main(): Promise { + const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); + for await (const line of lines) { + console.log("line", JSON.stringify(line)); + } + console.log("done"); +} + +void main(); +`; + +const ITERATE_WITH_CLOSE_LISTENER = `import { createInterface } from "node:readline"; + +async function main(): Promise { + const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); + lines.on("close", () => { console.log("close"); }); + for await (const line of lines) { + console.log("line", JSON.stringify(line)); + } + console.log("done"); +} + +void main(); +`; + +const ITERATE_THEN_STOP = `import { createInterface } from "node:readline"; + +async function main(): Promise { + const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); + let seen = 0; + for await (const line of lines) { + seen += 1; + console.log("line", JSON.stringify(line)); + if (seen === 2) { + lines.close(); + console.log("closed"); + } + } + console.log("done", seen); +} + +void main(); +`; + +test.each([ + ["several whole lines", "one\\ntwo\\nthree\\n"], + // The last line has no terminator: Node's onend emits it as a 'line' + // anyway, so the iterator sees it before the loop ends. + ["a partial last line", "one\\ntwo\\nthree"], + ["one line, no terminator", "solo"], + // Empty input is corpus 2794's shape, kept here beside its neighbours. + ["no input at all", ""], + ["only terminators", "\\n\\n\\n"], + ["CRLF terminators", "one\\r\\ntwo\\r\\n"], + ["a held CR at the end", "one\\ntwo\\r"], +])("C readline for-await matches Node: %s", async (name, input) => { + const [node, native] = await bothLanes("iterate", ITERATE, input); + expect(native.stdout).toBe(node.stdout); + expect(native.exitCode).toBe(node.exitCode); +}); + +test("C readline for-await orders the close event like Node", async () => { + const [node, native] = await bothLanes( + "iterate_close_listener", + ITERATE_WITH_CLOSE_LISTENER, + "one\ntwo\nthree", + ); + expect(native.stdout).toBe(node.stdout); + expect(native.exitCode).toBe(node.exitCode); +}); + +test("C readline for-await ends when the loop body closes the interface", async () => { + const [node, native] = await bothLanes( + "iterate_then_stop", + ITERATE_THEN_STOP, + "one\ntwo\nthree\nfour\n", + ); + expect(native.stdout).toBe(node.stdout); + expect(native.exitCode).toBe(node.exitCode); +}); diff --git a/packages/runtime/src/scr_readline.c b/packages/runtime/src/scr_readline.c index ef06e8d84..575006071 100644 --- a/packages/runtime/src/scr_readline.c +++ b/packages/runtime/src/scr_readline.c @@ -1,4 +1,4 @@ -/* node:readline — the question/close slice over the stdin unit +/* node:readline — the question/nextLine/close slice over the stdin unit * (scr_events.c), linked exactly when the program uses it (the events * gating; rl.* libCalls imply events). * @@ -17,13 +17,29 @@ * listeners SYNCHRONOUSLY — Node's close() emits inline, which is why * the portless prompt's close-listener resolve wins over the question * callback's (oracle-pinned). - * - stdin EOF closes every open interface the same way: the buffered - * partial line is DISCARDED (Node), then 'close' fires. - * - question() after close() throws Node's ERR_USE_AFTER_CLOSE message. + * - nextLine() is `for await (const line of rl)`: it answers the next + * line, or UNDEFINED once the interface is closed or stdin ended, and + * goes on answering undefined at every later call so the loop ends + * (Node's iterator keeps returning {done: true}). This is the one + * consumer that OUTLIVES its lines: the moment a program asks for one, + * a 'line' listener exists in Node's model, so an unclaimed line stops + * dropping and QUEUES for the next ask. + * - stdin EOF closes every open interface: a held \r terminates its line + * and answers a pending question; a leftover PARTIAL line reaches the + * async iterator but never a question — Node's onend emits 'line' + * directly rather than through [kOnLine], and only [kOnLine] answers a + * question (probed against Node 24: `printf 'a\nb' | node` delivers "b" + * to `for await` and nothing to a pending `question`). Then 'close' + * fires. + * - question() after close() throws Node's ERR_USE_AFTER_CLOSE message; + * nextLine() after close() answers undefined instead — a closed + * iterator is done, not an error. * - An interface created AFTER stdin ended is DEAD (pinned against * Node): a question still writes its prompt, but nothing ever answers * and 'close' never fires — Node's process exits with the question - * pending, and the loop exhausts the same way here. */ + * pending, and the loop exhausts the same way here. A nextLine on a + * dead interface answers undefined at once (Node's iterator over an + * already-ended input is immediately done). */ #include "scr_runtime.h" #include @@ -41,6 +57,13 @@ typedef struct ScrRl { bool dead; /* created after stdin ended: close fires at question() */ ScrClosure *q_cb; /* pending question's callback (owned), or NULL */ void (*q_fn)(ScrClosure *, ScrStr *); + ScrClosure *nl_cb; /* pending nextLine waiter (owned), or NULL */ + void (*nl_fn)(ScrClosure *, ScrStr *); /* a NULL line means undefined */ + bool iterating; /* a nextLine has been asked: a 'line' listener exists in + * Node's model, so an unclaimed line QUEUES below rather + * than dropping */ + ScrStr **lines; /* queued lines (owned) waiting for the next nextLine */ + size_t n_lines, cap_lines; ScrClosure **close_cbs; /* owned zero-arg listeners */ size_t n_close, cap_close; char *buf; /* undelivered stdin bytes */ @@ -62,8 +85,47 @@ static ScrRl *scr_rl_find(double id) { return NULL; } +/* ── the async iterator's line queue ────────────────────────────────── + * Node's `for await (const line of rl)` is an EventEmitter.on('line') + * iterator: it is a LISTENER, so a line parsed while the loop body is + * running (between one await settling and the next nextLine) is buffered + * rather than lost. A question has no such queue — nothing listens + * between two questions, which is why an unclaimed line still drops for + * a question-only interface. */ + +/* Hands a parsed line to the pending nextLine waiter, or queues it. + * Takes the line's +1 either way. */ +static void scr_rl_deliver_line(ScrRl *rl, ScrStr *line) { + if (rl->nl_cb) { + ScrClosure *cb = rl->nl_cb; + void (*fn)(ScrClosure *, ScrStr *) = rl->nl_fn; + rl->nl_cb = NULL; /* consumed BEFORE the callback runs (once) */ + fn(cb, line); /* the adapter owns the +1 line */ + scr_closure_release(cb); + return; + } + if (rl->n_lines == rl->cap_lines) { + rl->cap_lines = rl->cap_lines ? rl->cap_lines * 2 : 4; + rl->lines = realloc(rl->lines, rl->cap_lines * sizeof *rl->lines); + if (!rl->lines) scr_rl_oom(); + } + rl->lines[rl->n_lines++] = line; +} + +/* The oldest queued line (+1 moves out), or NULL when none is waiting. */ +static ScrStr *scr_rl_take_queued(ScrRl *rl) { + if (rl->n_lines == 0) return NULL; + ScrStr *line = rl->lines[0]; + rl->n_lines--; + memmove(rl->lines, rl->lines + 1, rl->n_lines * sizeof *rl->lines); + return line; +} + /* Fires the close listeners (snapshot; synchronous, like Node's emit) and - * releases everything the interface holds. */ + * releases everything the interface holds. A pending nextLine answers + * UNDEFINED — the iterator is done, where a pending question simply never + * hears back. The queue survives: Node's iterator flushes the lines it + * already heard before reporting done. */ static void scr_rl_settle_close(ScrRl *rl) { if (rl->closed) return; rl->closed = true; @@ -92,6 +154,16 @@ static void scr_rl_settle_close(ScrRl *rl) { free(rl->buf); rl->buf = NULL; rl->len = rl->cap = 0; + /* The close listeners run first (Node emits 'close' inline), then the + * parked iterator learns it is done — the fulfillment is a microtask + * either way, so the listeners' output still comes first. */ + if (rl->nl_cb) { + ScrClosure *cb = rl->nl_cb; + void (*fn)(ScrClosure *, ScrStr *) = rl->nl_fn; + rl->nl_cb = NULL; + fn(cb, NULL); /* undefined: the iterator is done */ + scr_closure_release(cb); + } } /* One complete line off the front of the buffer: *adv is the byte count @@ -122,7 +194,9 @@ static void scr_rl_drain(ScrRl *rl) { ScrStr *line = NULL; ScrClosure *cb = rl->q_cb; void (*fn)(ScrClosure *, ScrStr *) = rl->q_fn; - if (cb) line = scr_str_new(rl->buf, line_len); + /* A line is only built for a consumer: a question, the async + * iterator, or the iterator's queue. */ + if (cb || rl->iterating) line = scr_str_new(rl->buf, line_len); memmove(rl->buf, rl->buf + adv, rl->len - adv); rl->len -= adv; if (cb) { @@ -130,8 +204,14 @@ static void scr_rl_drain(ScrRl *rl) { fn(cb, line); /* the adapter owns the +1 line */ scr_closure_release(cb); if (scr_exc_pending()) return; + continue; + } + if (rl->iterating) { + scr_rl_deliver_line(rl, line); /* the waiter, or the queue */ + if (scr_exc_pending()) return; + continue; } - /* No pending question: the line drops, exactly Node's unheard 'line'. */ + /* No consumer at all: the line drops, exactly Node's unheard 'line'. */ } } @@ -153,24 +233,40 @@ static void scr_rl_data_adapter(ScrClosure *cb, ScrBytes *chunk) { } } -/* stdin EOF: every open interface closes — the buffered partial line is - * DISCARDED (Node; a trailing held \r still terminates its line first). */ +/* stdin EOF: every open interface closes. The leftover bytes are one + * last line — but for WHOM differs, and the split is Node's own: onend + * emits 'line' DIRECTLY, where the per-chunk path goes through + * [kOnLine], and only [kOnLine] answers a question. So the async + * iterator hears the partial line and a pending question does not. + * The one exception is a HELD \r ("a\r" then EOF): the \r is a real line + * terminator whose \r\n decision expired, so that line came through the + * ordinary path and a question does answer it. */ static void scr_rl_end_thunk(ScrClosure *cb) { (void)cb; scr_rl_data_cb = NULL; /* the stdin unit dropped its listeners itself */ for (ScrRl *rl = scr_rls; rl; rl = rl->next) { if (rl->closed || rl->dead) continue; - if (rl->len > 0 && rl->buf[rl->len - 1] == '\r' && rl->q_cb) { - /* "a\r" then EOF: the \r terminates the line (Node's crlfDelay - * expiry collapsed to EOF time). */ - ScrStr *line = scr_str_new(rl->buf, rl->len - 1); - ScrClosure *qcb = rl->q_cb; - void (*fn)(ScrClosure *, ScrStr *) = rl->q_fn; - rl->q_cb = NULL; - rl->len = 0; - fn(qcb, line); - scr_closure_release(qcb); - if (scr_exc_pending()) return; + if (rl->len > 0) { + bool held_cr = rl->buf[rl->len - 1] == '\r'; + size_t line_len = held_cr ? rl->len - 1 : rl->len; + if (held_cr && rl->q_cb) { + /* "a\r" then EOF: the \r terminates the line (Node's crlfDelay + * expiry collapsed to EOF time). */ + ScrStr *line = scr_str_new(rl->buf, line_len); + ScrClosure *qcb = rl->q_cb; + void (*fn)(ScrClosure *, ScrStr *) = rl->q_fn; + rl->q_cb = NULL; + rl->len = 0; + fn(qcb, line); + scr_closure_release(qcb); + if (scr_exc_pending()) return; + } else if (rl->iterating) { + ScrStr *line = scr_str_new(rl->buf, line_len); + rl->len = 0; + scr_rl_deliver_line(rl, line); + if (scr_exc_pending()) return; + } + /* Neither: the partial line is DISCARDED, exactly Node. */ } scr_rl_settle_close(rl); if (scr_exc_pending()) return; @@ -186,6 +282,14 @@ static void scr_rl_cleanup_atexit(void) { scr_closure_release(rl->q_cb); rl->q_cb = NULL; } + if (rl->nl_cb) { + scr_closure_release(rl->nl_cb); + rl->nl_cb = NULL; + } + for (size_t i = 0; i < rl->n_lines; i++) scr_str_release(rl->lines[i]); + free(rl->lines); + rl->lines = NULL; + rl->n_lines = rl->cap_lines = 0; for (size_t i = 0; i < rl->n_close; i++) scr_closure_release(rl->close_cbs[i]); free(rl->close_cbs); rl->close_cbs = NULL; @@ -253,6 +357,41 @@ void scr_rl_question(double id, const ScrStr *query, ScrClosure *cb /*moves*/, scr_rl_drain(rl); /* an already-buffered line answers immediately */ } +/* `for await (const line of rl)`: the next line, or NULL for undefined. + * + * Unlike question, this NEVER throws on a closed interface — a done + * iterator answers undefined, and answers it again at every later call, + * which is what ends the loop. The three prompt answers, in order: a line + * the queue already holds, a line already buffered in the byte window + * (scr_rl_drain, exactly question's immediate answer), or undefined when + * nothing more can arrive. Otherwise the waiter parks and the loop keeps + * fd 0 alive, because the interface is still an open stdin consumer. */ +void scr_rl_next_line(double id, ScrClosure *cb /*moves*/, + void (*fn)(ScrClosure *, ScrStr *)) { + ScrRl *rl = scr_rl_find(id); + if (!rl) { + fn(cb, NULL); + scr_closure_release(cb); + return; + } + rl->iterating = true; /* from here on, unclaimed lines queue */ + ScrStr *queued = scr_rl_take_queued(rl); + if (queued) { + fn(cb, queued); + scr_closure_release(cb); + return; + } + if (rl->closed || rl->dead) { + fn(cb, NULL); + scr_closure_release(cb); + return; + } + if (rl->nl_cb) scr_closure_release(rl->nl_cb); /* re-ask replaces */ + rl->nl_cb = cb; + rl->nl_fn = fn; + scr_rl_drain(rl); /* an already-buffered line answers immediately */ +} + void scr_rl_close(double id) { ScrRl *rl = scr_rl_find(id); if (!rl) return; diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 80cdf376f..06639f89b 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -4109,10 +4109,19 @@ bool scr_stdin_ended(void); * "readline was closed" on a closed interface (may-throw); close fires * the 'close' listeners SYNCHRONOUSLY (Node's inline emit) and detaches * the stdin consumer; onClose registers a zero-arg listener (moves). - * Answer adapters: thunk0 ignores the line, thunk_str passes it. */ + * Answer adapters: thunk0 ignores the line, thunk_str passes it. + * + * nextLine is `for await (const line of rl)`: the same adapter shape, + * except the line is NULL for JS's undefined — the iterator's done — and + * a closed interface ANSWERS undefined instead of throwing, at that call + * and at every later one, which is what ends the loop. The emitter's + * adapter owns the +1 line and fulfills the `string | undefined` promise + * it captured. Never throws. */ double scr_rl_create(void); void scr_rl_question(double id, const ScrStr *query, ScrClosure *cb /*moves*/, void (*fn)(ScrClosure *, ScrStr *)); +void scr_rl_next_line(double id, ScrClosure *cb /*moves*/, + void (*fn)(ScrClosure *, ScrStr *)); void scr_rl_close(double id); void scr_rl_on_close(double id, ScrClosure *cb /*moves*/); void scr_rl_answer_thunk0(ScrClosure *cb, ScrStr *answer); diff --git a/scripts/check-rust-file-lines.mjs b/scripts/check-rust-file-lines.mjs index 56e88063e..d7ab7c3af 100644 --- a/scripts/check-rust-file-lines.mjs +++ b/scripts/check-rust-file-lines.mjs @@ -13,7 +13,7 @@ const legacyOversizedFiles = new Map([ ["packages/compiler/src/backend/cc.test.ts", 4_104], ["packages/compiler/src/backend/cc.ts", 6_087], ["packages/compiler/src/backend/emission/emit-async.ts", 1_279], - ["packages/compiler/src/backend/emission/emit-exprs.ts", 7_964], + ["packages/compiler/src/backend/emission/emit-exprs.ts", 7_963], ["packages/compiler/src/backend/emission/emit-walkers.ts", 2_062], ["packages/compiler/src/backend/emission/emitter.ts", 2_134], ["packages/compiler/src/backend/llvm/dyn.ts", 3_105],