diff --git a/CHANGELOG.md b/CHANGELOG.md index 17a4385..a07f422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## June 2026: robustness badges + +- Each parser page gains a Robustness section with six per-parser badges mined from the parser's own source and behavior, so a chooser can weigh crash-safety alongside speed and coverage. +- Static panic discipline: a new `featurescan` crate parses each parser's library source with `syn` and counts panic-inducing constructs (panic!, unreachable!, unimplemented!, todo!, unwrap, expect, indexing), excluding tests, benches, and test-helper files, and reads the crate's own lint policy so a parser that bans those lints by design is shown as banned. The counts are a code-smell proxy, not a crash proof. +- Empirical panic rate: grading now tells a caught panic apart from an honest error, so each parser page reports how often it actually panics on the real corpus instead of returning an error. qusql-parse is the only parser that panics on real input (a fraction of a percent), and turso_parser's many static unreachable! macros never fire, which is exactly why the static and empirical signals are shown side by side. +- Recursion depth: a child-process probe measures how deeply each parser nests input before it either rejects with a clean recursion-limit error or overflows the stack and aborts the process. Among the pure-Rust parsers only sqlparser-rs (limit 48) and sqlite3-parser (no call recursion) are depth-guarded, while polyglot-sql overflows at depth 232. +- Unsafe surface (count plus whether the crate forbids unsafe), direct dependency count, and whether the AST derives serde round out the badges. +- The feature scan and depth probe run as part of `cargo regen`, and their committed JSON snapshots are baked into the site at build time, so the wasm build stays free of network access. + ## June 2026: real engines, batch axis, and the time machine - Validity is now graded against the real database engines (PostgreSQL, SQLite, MySQL, ClickHouse, DuckDB, SQL Server), run once locally in Docker via testcontainers by the `oracle` crate, with the labels committed under `oracle/labels` so grading and CI need no Docker. Library oracles are gone. diff --git a/Cargo.toml b/Cargo.toml index 88fd557..871089f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "viz", "web", "membench", "oracle", "timemachine"] +members = [".", "viz", "web", "membench", "oracle", "timemachine", "featurescan"] default-members = ["."] resolver = "2" diff --git a/benches/batch_parsing.rs b/benches/batch_parsing.rs index 01528b4..68c6650 100644 --- a/benches/batch_parsing.rs +++ b/benches/batch_parsing.rs @@ -14,7 +14,7 @@ //! parses in that dialect), so the two numbers are directly comparable. //! //! Only parsers with a multi-statement entry point take part (see -//! `BenchParser::can_batch`); `databend-common-ast` parses one statement per +//! `BenchParser::can_batch`). `databend-common-ast` parses one statement per //! call and is simply skipped here. //! //! Output (under `target/batch_dist/`), self-contained for now (not yet wired @@ -157,7 +157,7 @@ fn main() { return; } - // Acceptance checks are panic-guarded; suppress the default panic message so + // Acceptance checks are panic-guarded. Suppress the default panic message so // a caught panic does not spam stderr. std::panic::set_hook(Box::new(|_| {})); diff --git a/featurescan/Cargo.toml b/featurescan/Cargo.toml new file mode 100644 index 0000000..9d24d1b --- /dev/null +++ b/featurescan/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "featurescan" +version = "0.1.0" +edition = "2021" +description = "Static source-feature scan of the benchmarked SQL parsers (panic discipline, unsafe, lint policy)" +license = "MIT" +publish = false +default-run = "featurescan" + +# Offline analysis crate. It locates each benchmarked parser's source on disk via +# `cargo metadata` (git checkouts and registry crates alike), parses the library +# `src/` with `syn`, and records panic-inducing constructs, unsafe usage, and the +# crate's own lint policy. The committed result (featurescan/data/featurescan.json) +# is baked into the web metadata, so the wasm build stays free of network access. +# Not a default workspace member, so `cargo build` never pulls these deps. + +[dependencies] +cargo_metadata = "0.19" +syn = { version = "2", features = ["full", "visit"] } +# span-locations gives line numbers on spans, so test-region lines can be +# subtracted from the LOC used for per-KLOC density. +proc-macro2 = { version = "1", features = ["span-locations"] } +quote = "1" +walkdir = "2" +toml = "0.8" +serde_json = "1" +# Shared schema types (FeatureScan/DepthScan), so the committed JSON the web bakes +# in has a single source of truth. +viz = { path = "../viz" } +# The depth probe runs the real parsers, so it needs the main crate. The static +# scan binary does not use it, but a workspace member with one heavy dep is fine +# since featurescan is never part of the default build. +sql_ast_benchmark = { path = ".." } + +[[bin]] +name = "featurescan" +path = "src/main.rs" + +[[bin]] +name = "featurescan-depth" +path = "src/depth.rs" diff --git a/featurescan/data/depth.json b/featurescan/data/depth.json new file mode 100644 index 0000000..de1fedc --- /dev/null +++ b/featurescan/data/depth.json @@ -0,0 +1,97 @@ +{ + "note": "Recursion-depth probe (nested parens, 8 MiB worker stack, ceiling 50000). Each (parser, depth) trial runs in a child process. A clean exit means the depth was handled, a signal kill means a stack overflow. `crash_depth` is the smallest overflowing depth (null = never crashed up to the ceiling). `limit_depth` is the smallest depth the parser rejects instead of accepting (its graceful recursion limit, null = accepts up to the boundary). `guarded` = it rejects deep input cleanly and never crashes. Regenerate with `cargo run -p featurescan --bin featurescan-depth`.", + "stack_bytes": 8388608, + "ceil": 50000, + "parsers": [ + { + "parser": "sqlparser-rs", + "dialect": "postgresql", + "guarded": true, + "shape_rejected": false, + "limit_depth": 48, + "crash_depth": null, + "ceil": 50000 + }, + { + "parser": "pg_query.rs", + "dialect": "postgresql", + "guarded": true, + "shape_rejected": false, + "limit_depth": 9994, + "crash_depth": null, + "ceil": 50000 + }, + { + "parser": "pg_query (summary)", + "dialect": "postgresql", + "guarded": true, + "shape_rejected": false, + "limit_depth": 9994, + "crash_depth": null, + "ceil": 50000 + }, + { + "parser": "qusql-parse", + "dialect": "postgresql", + "guarded": false, + "shape_rejected": false, + "limit_depth": null, + "crash_depth": 8578, + "ceil": 50000 + }, + { + "parser": "polyglot-sql", + "dialect": "postgresql", + "guarded": false, + "shape_rejected": false, + "limit_depth": null, + "crash_depth": 232, + "ceil": 50000 + }, + { + "parser": "databend-common-ast", + "dialect": "postgresql", + "guarded": false, + "shape_rejected": false, + "limit_depth": null, + "crash_depth": 421, + "ceil": 50000 + }, + { + "parser": "orql", + "dialect": "oracle", + "guarded": false, + "shape_rejected": true, + "limit_depth": null, + "crash_depth": 5184, + "ceil": 50000 + }, + { + "parser": "sqlglot-rust", + "dialect": "postgresql", + "guarded": false, + "shape_rejected": false, + "limit_depth": null, + "crash_depth": 548, + "ceil": 50000 + }, + { + "parser": "sqlite3-parser", + "dialect": "sqlite", + "guarded": true, + "shape_rejected": false, + "limit_depth": null, + "crash_depth": null, + "ceil": 50000 + }, + { + "parser": "turso_parser", + "dialect": "sqlite", + "guarded": false, + "shape_rejected": false, + "limit_depth": null, + "crash_depth": 2199, + "ceil": 50000 + } + ] +} \ No newline at end of file diff --git a/featurescan/data/featurescan.json b/featurescan/data/featurescan.json new file mode 100644 index 0000000..3b945f5 --- /dev/null +++ b/featurescan/data/featurescan.json @@ -0,0 +1,327 @@ +{ + "note": "Static source scan of each parser's library src/ (panic families, unsafe, lint policy). Counts exclude tests/benches/examples, #[cfg(test)] items, and test-helper files (e.g. test_utils.rs). Macro-body unwraps are not counted (opaque to syn). Counts are a code-smell proxy, not a crash proof. Regenerate with `cargo run -p featurescan`.", + "parsers": [ + { + "parser": "sqlparser-rs", + "package": "sqlparser", + "version": "0.62.0", + "counts": { + "loc": 68907, + "test_lines": 4276, + "code_loc": 64631, + "files": 43, + "parse_failures": 0, + "panic": 3, + "unreachable": 0, + "unimplemented": 0, + "todo": 0, + "assert": 4, + "unwrap": 7, + "expect": 0, + "unwrap_unchecked": 0, + "index": 8, + "unsafe_blocks": 0, + "unsafe_fns": 0, + "unsafe_impls": 0, + "serde_derive": false + }, + "lints": { + "lints": { + "unreachable": "forbid" + }, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 6, + "serde_dep": true + }, + { + "parser": "pg_query.rs", + "package": "pg_query", + "version": "6.1.1", + "counts": { + "loc": 15402, + "test_lines": 0, + "code_loc": 15402, + "files": 13, + "parse_failures": 0, + "panic": 1, + "unreachable": 0, + "unimplemented": 0, + "todo": 0, + "assert": 0, + "unwrap": 38, + "expect": 0, + "unwrap_unchecked": 0, + "index": 9, + "unsafe_blocks": 40, + "unsafe_fns": 2, + "unsafe_impls": 0, + "serde_derive": true + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 5, + "serde_dep": true + }, + { + "parser": "pg_query (summary)", + "package": "pg_query", + "version": "6.1.1", + "counts": { + "loc": 15402, + "test_lines": 0, + "code_loc": 15402, + "files": 13, + "parse_failures": 0, + "panic": 1, + "unreachable": 0, + "unimplemented": 0, + "todo": 0, + "assert": 0, + "unwrap": 38, + "expect": 0, + "unwrap_unchecked": 0, + "index": 9, + "unsafe_blocks": 40, + "unsafe_fns": 2, + "unsafe_impls": 0, + "serde_derive": true + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 5, + "serde_dep": true + }, + { + "parser": "qusql-parse", + "package": "qusql-parse", + "version": "0.8.0", + "counts": { + "loc": 35323, + "test_lines": 1711, + "code_loc": 33612, + "files": 43, + "parse_failures": 0, + "panic": 1, + "unreachable": 0, + "unimplemented": 0, + "todo": 0, + "assert": 0, + "unwrap": 48, + "expect": 6, + "unwrap_unchecked": 0, + "index": 11, + "unsafe_blocks": 1, + "unsafe_fns": 0, + "unsafe_impls": 0, + "serde_derive": false + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 0, + "serde_dep": false + }, + { + "parser": "polyglot-sql", + "package": "polyglot-sql", + "version": "0.4.4", + "counts": { + "loc": 239954, + "test_lines": 16786, + "code_loc": 223168, + "files": 74, + "parse_failures": 0, + "panic": 5, + "unreachable": 212, + "unimplemented": 0, + "todo": 0, + "assert": 0, + "unwrap": 312, + "expect": 17, + "unwrap_unchecked": 0, + "index": 1066, + "unsafe_blocks": 0, + "unsafe_fns": 0, + "unsafe_impls": 0, + "serde_derive": true + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 7, + "serde_dep": true + }, + { + "parser": "databend-common-ast", + "package": "databend-common-ast", + "version": "0.2.5", + "counts": { + "loc": 31399, + "test_lines": 271, + "code_loc": 31128, + "files": 81, + "parse_failures": 0, + "panic": 1, + "unreachable": 33, + "unimplemented": 0, + "todo": 0, + "assert": 3, + "unwrap": 27, + "expect": 0, + "unwrap_unchecked": 0, + "index": 46, + "unsafe_blocks": 0, + "unsafe_fns": 0, + "unsafe_impls": 0, + "serde_derive": true + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 26, + "serde_dep": true + }, + { + "parser": "sqlglot-rust", + "package": "sqlglot-rust", + "version": "0.10.0", + "counts": { + "loc": 29937, + "test_lines": 3951, + "code_loc": 25986, + "files": 29, + "parse_failures": 0, + "panic": 0, + "unreachable": 4, + "unimplemented": 0, + "todo": 0, + "assert": 1, + "unwrap": 21, + "expect": 4, + "unwrap_unchecked": 0, + "index": 162, + "unsafe_blocks": 11, + "unsafe_fns": 5, + "unsafe_impls": 2, + "serde_derive": true + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 5, + "serde_dep": true + }, + { + "parser": "sqlite3-parser", + "package": "sqlite3-parser", + "version": "0.16.0", + "counts": { + "loc": 6712, + "test_lines": 584, + "code_loc": 6128, + "files": 12, + "parse_failures": 0, + "panic": 2, + "unreachable": 5, + "unimplemented": 0, + "todo": 0, + "assert": 3, + "unwrap": 5, + "expect": 0, + "unwrap_unchecked": 0, + "index": 59, + "unsafe_blocks": 1, + "unsafe_fns": 0, + "unsafe_impls": 0, + "serde_derive": false + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 7, + "serde_dep": false + }, + { + "parser": "turso_parser", + "package": "turso_parser", + "version": "0.6.1", + "counts": { + "loc": 19732, + "test_lines": 8126, + "code_loc": 11606, + "files": 8, + "parse_failures": 0, + "panic": 0, + "unreachable": 29, + "unimplemented": 0, + "todo": 0, + "assert": 7, + "unwrap": 19, + "expect": 0, + "unwrap_unchecked": 0, + "index": 67, + "unsafe_blocks": 3, + "unsafe_fns": 1, + "unsafe_impls": 0, + "serde_derive": false + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 8, + "serde_dep": true + }, + { + "parser": "orql", + "package": "orql", + "version": "0.1.0", + "counts": { + "loc": 17994, + "test_lines": 481, + "code_loc": 17513, + "files": 40, + "parse_failures": 0, + "panic": 6, + "unreachable": 0, + "unimplemented": 0, + "todo": 0, + "assert": 0, + "unwrap": 6, + "expect": 13, + "unwrap_unchecked": 0, + "index": 23, + "unsafe_blocks": 1, + "unsafe_fns": 0, + "unsafe_impls": 0, + "serde_derive": false + }, + "lints": { + "lints": {}, + "workspace_inherited": false + }, + "forbids_unsafe": false, + "direct_deps": 2, + "serde_dep": false + } + ] +} \ No newline at end of file diff --git a/featurescan/src/depth.rs b/featurescan/src/depth.rs new file mode 100644 index 0000000..6277e37 --- /dev/null +++ b/featurescan/src/depth.rs @@ -0,0 +1,234 @@ +//! Recursion-depth probe: how deep can each parser nest before it stops, and how +//! does it stop, a clean recursion-limit error or a hard stack overflow. +//! +//! A stack overflow aborts the whole process and is uncatchable (even on a worker +//! thread and even by `catch_unwind`), so the depth where a parser overflows +//! cannot be found safely in-process. Instead each (parser, depth) trial runs in a +//! CHILD PROCESS: if the child exits with a status code it handled that depth +//! (accepted, rejected, or a caught panic). If it is killed by a signal it +//! overflowed. The parent binary-searches two boundaries per parser: the graceful +//! limit (smallest depth the parser rejects instead of accepting) and the crash +//! depth (smallest depth that overflows). A parser that rejects deep input with a +//! clean error and never crashes is "depth-guarded". +//! +//! The probe shape is nested parentheses around a literal (`SELECT (((1)))`), +//! valid SQL at any depth and the classic stack-overflow trigger for +//! recursive-descent parsers. The child worker uses a fixed 8 MiB stack (a typical +//! default), so the crash depths are comparable and reflect a normal environment, +//! not the 512 MiB grading threads. + +use std::process::{Command, Stdio}; + +use sql_ast_benchmark::datasets::Dialect; +use sql_ast_benchmark::{BenchParser, ParseOutcome}; +use viz::{DepthReport, DepthScan}; + +/// Worker stack for the child trial. Fixed so crash depths are comparable. +const PROBE_STACK: usize = 8 * 1024 * 1024; +/// Highest depth tried. Above any recursion limit, below run-away string sizes. +const CEIL: usize = 50_000; +/// Env var carrying the child trial spec: "|". +const CHILD_ENV: &str = "FEATURESCAN_DEPTH_CHILD"; + +/// Outcome of one depth trial. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Trial { + Accepted, + Rejected, + Panicked, + Unsupported, + /// The child died (stack overflow / abort): this depth is not survivable. + Crash, +} + +/// Representative dialect to probe each parser in (its primary home dialect). +fn rep_dialect(p: BenchParser) -> Dialect { + match p { + BenchParser::Sqlite3 | BenchParser::Turso => Dialect::Sqlite, + BenchParser::Orql => Dialect::Oracle, + _ => Dialect::Postgresql, + } +} + +/// Nested-parens statement at the given depth: `SELECT ((( 1 )))`. +fn nested_sql(depth: usize) -> String { + let mut s = String::with_capacity(depth * 2 + 16); + s.push_str("SELECT "); + for _ in 0..depth { + s.push('('); + } + s.push('1'); + for _ in 0..depth { + s.push(')'); + } + s +} + +fn main() { + if let Ok(spec) = std::env::var(CHILD_ENV) { + run_child(&spec); + return; + } + run_parent(); +} + +/// Child: parse one nested statement on a bounded stack and exit with a code that +/// encodes the outcome. A stack overflow aborts the process before we exit, which +/// the parent reads as a crash. +fn run_child(spec: &str) { + // Caught panics are expected here, so silence the hook so the parent's captured + // stderr stays clean. + std::panic::set_hook(Box::new(|_| {})); + let mut parts = spec.split('|'); + let idx: usize = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); + let depth: usize = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); + let parser = BenchParser::all()[idx]; + let dialect = rep_dialect(parser); + let sql = nested_sql(depth); + + let outcome = std::thread::Builder::new() + .stack_size(PROBE_STACK) + .spawn(move || parser.parse_outcome(&sql, dialect)) + .expect("spawn probe thread") + .join() + .expect("probe thread"); + + let code = match outcome { + ParseOutcome::Accepted => 0, + ParseOutcome::Rejected(_) => 1, + ParseOutcome::Panicked(_) => 2, + ParseOutcome::Unsupported => 3, + }; + std::process::exit(code); +} + +/// Parent: probe every parser and write the committed depth snapshot. +fn run_parent() { + let exe = std::env::current_exe().expect("current exe"); + let parsers = BenchParser::all(); + + let mut reports = Vec::new(); + for (idx, &p) in parsers.iter().enumerate() { + // Cache trials so the two binary searches share results. + let mut cache: std::collections::HashMap = std::collections::HashMap::new(); + let mut trial = |depth: usize| -> Trial { + if let Some(&t) = cache.get(&depth) { + return t; + } + let t = probe(&exe, idx, depth); + cache.insert(depth, t); + t + }; + + let report = analyze(p, &mut trial); + eprintln!("{:22} {}", p.name(), describe(&report)); + reports.push(report); + } + + let snapshot = DepthScan { + note: format!( + "Recursion-depth probe (nested parens, {} MiB worker stack, ceiling {CEIL}). \ + Each (parser, depth) trial runs in a child process. A clean exit means \ + the depth was handled, a signal kill means a stack overflow. \ + `crash_depth` is the smallest overflowing depth (null = never crashed up \ + to the ceiling). `limit_depth` is the smallest depth the parser rejects \ + instead of accepting (its graceful recursion limit, null = accepts up to \ + the boundary). `guarded` = it rejects deep input cleanly and never \ + crashes. Regenerate with `cargo run -p featurescan --bin featurescan-depth`.", + PROBE_STACK / (1024 * 1024) + ), + stack_bytes: PROBE_STACK, + ceil: CEIL, + parsers: reports, + }; + + let out_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("data"); + std::fs::create_dir_all(&out_dir).expect("create data dir"); + let out_path = out_dir.join("depth.json"); + let json = serde_json::to_string_pretty(&snapshot).expect("serialize"); + std::fs::write(&out_path, json).expect("write snapshot"); + eprintln!("wrote {}", out_path.display()); +} + +/// Run one child trial and classify its exit status. +fn probe(exe: &std::path::Path, parser_idx: usize, depth: usize) -> Trial { + let status = Command::new(exe) + .env(CHILD_ENV, format!("{parser_idx}|{depth}")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("spawn child"); + match status.code() { + Some(0) => Trial::Accepted, + Some(1) => Trial::Rejected, + Some(2) => Trial::Panicked, + Some(3) => Trial::Unsupported, + // None = killed by signal (overflow/abort). Any other code is treated as a crash. + _ => Trial::Crash, + } +} + +/// Smallest depth in `(lo, hi]` where `pred` holds, assuming `pred` is monotonic +/// (once true it stays true) with `!pred(lo)` and `pred(hi)`. +fn boundary(mut lo: usize, mut hi: usize, mut pred: impl FnMut(usize) -> bool) -> usize { + while lo + 1 < hi { + let mid = lo + (hi - lo) / 2; + if pred(mid) { + hi = mid; + } else { + lo = mid; + } + } + hi +} + +/// Find the graceful limit and crash depth for one parser. +fn analyze(p: BenchParser, trial: &mut impl FnMut(usize) -> Trial) -> DepthReport { + let dialect = rep_dialect(p); + let top = trial(CEIL); + + // Crash depth: smallest depth that overflows, if any up to the ceiling. + let crash_depth = if top == Trial::Crash { + Some(boundary(1, CEIL, |d| trial(d) == Trial::Crash)) + } else { + None + }; + + // If the parser rejects even shallow nesting, it does not accept this probe + // shape at all, so its graceful recursion limit cannot be read from this shape + // (the rejection is syntactic, not a depth guard). The crash depth is still + // meaningful: the parser recurses through the nesting before failing. + let shape_rejected = trial(1) != Trial::Accepted; + + // Search the survivable range for the graceful limit (first non-accept). + let safe_top = crash_depth.map_or(CEIL, |c| c - 1); + let limit_depth = if shape_rejected { + None + } else if safe_top >= 2 && trial(safe_top) != Trial::Accepted { + Some(boundary(1, safe_top, |d| trial(d) != Trial::Accepted)) + } else { + None + }; + + DepthReport { + parser: p.name().to_string(), + dialect: dialect.dir_name().to_string(), + guarded: crash_depth.is_none(), + shape_rejected, + limit_depth, + crash_depth, + ceil: CEIL, + } +} + +fn describe(r: &DepthReport) -> String { + let limit = if r.shape_rejected { + "shape n/a".to_string() + } else { + r.limit_depth.map_or("none".to_string(), |d| d.to_string()) + }; + match r.crash_depth { + Some(c) => format!("CRASHES at depth {c} (limit: {limit})"), + None => format!("guarded (limit {limit}, no crash up to {})", r.ceil), + } +} diff --git a/featurescan/src/lints.rs b/featurescan/src/lints.rs new file mode 100644 index 0000000..6eb4939 --- /dev/null +++ b/featurescan/src/lints.rs @@ -0,0 +1,140 @@ +//! Reading a crate's own lint policy: the panic-relevant lints it bans by design. +//! +//! Two sources are merged: the `[lints]` table in the crate's `Cargo.toml` and the +//! crate-root inner attributes (`#![deny(...)]` / `#![forbid(...)]`) in `lib.rs` or +//! `main.rs`. A lint set to `deny` or `forbid` is treated as "banned by design", +//! because then a regression fails the build (or `cargo clippy`). + +use std::path::Path; + +use quote::ToTokens; +use viz::LintPolicy; + +/// The panic-relevant lints we report on, keyed by their bare name (the part after +/// `clippy::`, or the rust lint name for `unsafe_code`). +pub const RELEVANT: &[&str] = &[ + "unwrap_used", + "expect_used", + "panic", + "unreachable", + "todo", + "unimplemented", + "indexing_slicing", + "arithmetic_side_effects", + "integer_arithmetic", + "panic_in_result_fn", + "unsafe_code", +]; + +/// Collect lint policy from the crate manifest and its crate root. +pub fn collect(manifest_path: &Path, src_dir: &Path) -> LintPolicy { + let mut info = LintPolicy::default(); + read_manifest_lints(manifest_path, &mut info); + for root in ["lib.rs", "main.rs"] { + read_root_attrs(&src_dir.join(root), &mut info); + } + info +} + +fn record(info: &mut LintPolicy, name: &str, level: &str) { + let bare = name.rsplit("::").next().unwrap_or(name); + if RELEVANT.contains(&bare) { + // Keep the strongest level seen (forbid > deny > warn > allow). + let strength = |l: &str| match l { + "forbid" => 3, + "deny" => 2, + "warn" => 1, + _ => 0, + }; + let entry = info.lints.entry(bare.to_string()).or_default(); + if strength(level) >= strength(entry) { + *entry = level.to_string(); + } + } +} + +fn read_manifest_lints(manifest_path: &Path, info: &mut LintPolicy) { + let Ok(text) = std::fs::read_to_string(manifest_path) else { + return; + }; + let Ok(value) = text.parse::() else { + return; + }; + let Some(lints) = value.get("lints") else { + return; + }; + if lints.get("workspace").and_then(toml::Value::as_bool) == Some(true) { + info.workspace_inherited = true; + } + for group in ["clippy", "rust"] { + let Some(table) = lints.get(group).and_then(toml::Value::as_table) else { + continue; + }; + for (name, spec) in table { + let level = match spec { + toml::Value::String(s) => Some(s.clone()), + toml::Value::Table(t) => t + .get("level") + .and_then(toml::Value::as_str) + .map(str::to_string), + _ => None, + }; + if let Some(level) = level { + record(info, name, &level); + } + } + } +} + +fn read_root_attrs(path: &Path, info: &mut LintPolicy) { + let Ok(text) = std::fs::read_to_string(path) else { + return; + }; + let Ok(file) = syn::parse_file(&text) else { + return; + }; + for attr in &file.attrs { + let level = if attr.path().is_ident("forbid") { + "forbid" + } else if attr.path().is_ident("deny") { + "deny" + } else if attr.path().is_ident("warn") { + "warn" + } else if attr.path().is_ident("allow") { + "allow" + } else { + continue; + }; + let _ = attr.parse_nested_meta(|meta| { + let name = meta + .path + .to_token_stream() + .to_string() + .replace(char::is_whitespace, ""); + record(info, &name, level); + Ok(()) + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strongest_level_wins() { + let mut info = LintPolicy::default(); + record(&mut info, "clippy::unwrap_used", "warn"); + record(&mut info, "clippy::unwrap_used", "deny"); + record(&mut info, "clippy::unwrap_used", "warn"); + assert!(info.is_banned("unwrap_used")); + assert_eq!(info.lints.get("unwrap_used").unwrap(), "deny"); + } + + #[test] + fn ignores_irrelevant_lints() { + let mut info = LintPolicy::default(); + record(&mut info, "clippy::needless_return", "deny"); + assert!(info.lints.is_empty()); + } +} diff --git a/featurescan/src/main.rs b/featurescan/src/main.rs new file mode 100644 index 0000000..764febb --- /dev/null +++ b/featurescan/src/main.rs @@ -0,0 +1,111 @@ +//! Static source-feature scan of the benchmarked SQL parsers. +//! +//! Locates each parser's source on disk via `cargo metadata`, scans its library +//! `src/` for panic-inducing constructs and unsafe usage, reads its lint policy, +//! and writes a committed snapshot at `featurescan/data/featurescan.json` that the +//! web metadata bakes in. Run with `cargo run -p featurescan`. + +mod lints; +mod scan; + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use cargo_metadata::MetadataCommand; +use viz::{FeatureScan, ParserFeatures}; + +/// (display name as used by the web metadata, crate package name). +const PARSERS: &[(&str, &str)] = &[ + ("sqlparser-rs", "sqlparser"), + ("pg_query.rs", "pg_query"), + // The summary mode is the same libpg_query crate, so it shares the scan. + ("pg_query (summary)", "pg_query"), + ("qusql-parse", "qusql-parse"), + ("polyglot-sql", "polyglot-sql"), + ("databend-common-ast", "databend-common-ast"), + ("sqlglot-rust", "sqlglot-rust"), + ("sqlite3-parser", "sqlite3-parser"), + ("turso_parser", "turso_parser"), + ("orql", "orql"), +]; + +fn main() { + let metadata = MetadataCommand::new() + .exec() + .expect("cargo metadata failed"); + + let by_name: BTreeMap<&str, &cargo_metadata::Package> = metadata + .packages + .iter() + .map(|p| (p.name.as_str(), p)) + .collect(); + + let mut reports = Vec::new(); + for (display, package_name) in PARSERS { + let Some(pkg) = by_name.get(package_name) else { + eprintln!("warning: package `{package_name}` not in cargo metadata, skipping"); + continue; + }; + let manifest_path: &Path = pkg.manifest_path.as_std_path(); + let src_dir = manifest_path + .parent() + .map(|p| p.join("src")) + .unwrap_or_else(|| PathBuf::from("src")); + + let counts = scan::scan_src(&src_dir); + let lint_info = lints::collect(manifest_path, &src_dir); + let forbids_unsafe = lint_info.is_banned("unsafe_code"); + + let direct_deps = pkg + .dependencies + .iter() + .filter(|d| d.kind == cargo_metadata::DependencyKind::Normal) + .count(); + let serde_dep = pkg + .dependencies + .iter() + .any(|d| d.name == "serde" || d.name == "serde_derive"); + + eprintln!( + "{display:22} v{} : {} files ({} unparsed), unwrap={} expect={} panic={} unreachable={} todo={} unsafe={}", + pkg.version, + counts.files, + counts.parse_failures, + counts.unwrap, + counts.expect, + counts.panic, + counts.unreachable, + counts.todo, + counts.unsafe_blocks + counts.unsafe_fns + counts.unsafe_impls, + ); + + reports.push(ParserFeatures { + parser: (*display).to_string(), + package: (*package_name).to_string(), + version: pkg.version.to_string(), + counts, + lints: lint_info, + forbids_unsafe, + direct_deps, + serde_dep, + }); + } + + let snapshot = FeatureScan { + note: "Static source scan of each parser's library src/ (panic families, \ + unsafe, lint policy). Counts exclude tests/benches/examples, \ + #[cfg(test)] items, and test-helper files (e.g. test_utils.rs). \ + Macro-body unwraps are not counted (opaque to syn). Counts are a \ + code-smell proxy, not a crash proof. Regenerate with `cargo run -p \ + featurescan`." + .to_string(), + parsers: reports, + }; + + let out_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("data"); + std::fs::create_dir_all(&out_dir).expect("create data dir"); + let out_path = out_dir.join("featurescan.json"); + let json = serde_json::to_string_pretty(&snapshot).expect("serialize snapshot"); + std::fs::write(&out_path, json).expect("write snapshot"); + eprintln!("wrote {}", out_path.display()); +} diff --git a/featurescan/src/scan.rs b/featurescan/src/scan.rs new file mode 100644 index 0000000..c78df89 --- /dev/null +++ b/featurescan/src/scan.rs @@ -0,0 +1,270 @@ +//! AST-level counting of panic-inducing constructs and unsafe usage in a crate's +//! library source, using `syn`. +//! +//! Limitations, stated plainly because the numbers are a code-smell proxy and not +//! a crash proof: +//! - Only library `src/` is scanned. `tests/`, `benches/`, `examples/`, inline +//! `#[cfg(test)]` / `#[test]` items, and well-known test-helper files (e.g. a +//! `test_utils.rs` exposed as a non-gated `pub mod` for integration tests) are +//! skipped, so test-only unwraps do not count. +//! - Macro bodies are opaque token streams to `syn`, so an `unwrap()` written +//! inside a `format!`/`vec!` argument is not counted. This undercounts slightly. +//! - Index expressions count every `a[i]`, including provably-safe constant indices. + +use std::path::Path; + +use quote::ToTokens; +use syn::visit::{self, Visit}; +use viz::FeatureCounts; +use walkdir::WalkDir; + +/// Scan every `.rs` file under `src_dir`, accumulating construct counts into the +/// shared [`FeatureCounts`] schema. +pub fn scan_src(src_dir: &Path) -> FeatureCounts { + let mut counts = FeatureCounts::default(); + for entry in WalkDir::new(src_dir).into_iter().filter_map(Result::ok) { + let path = entry.path(); + if path.extension().is_none_or(|e| e != "rs") { + continue; + } + if is_test_path(path) { + continue; + } + let Ok(text) = std::fs::read_to_string(path) else { + continue; + }; + counts.files += 1; + counts.loc += text.lines().count(); + match syn::parse_file(&text) { + Ok(file) => { + let mut scanner = Scanner { + counts: &mut counts, + }; + scanner.visit_file(&file); + } + Err(_) => counts.parse_failures += 1, + } + } + counts.code_loc = counts.loc.saturating_sub(counts.test_lines); + counts +} + +/// Number of source lines a node spans (requires proc-macro2 span-locations). +fn span_lines(node: &impl syn::spanned::Spanned) -> usize { + let span = node.span(); + span.end().line.saturating_sub(span.start().line) + 1 +} + +/// True if a `src/` file is a test fixture or test-helper rather than library code: +/// it sits under a `tests`/`benches`/`examples` directory, or its name is a known +/// test-helper file. These often are not `#[cfg(test)]`-gated (a `pub mod test_utils` +/// shared with integration tests), so the AST cfg filter alone would not skip them. +fn is_test_path(path: &Path) -> bool { + let in_test_dir = path.components().any(|c| { + let s = c.as_os_str().to_string_lossy(); + s == "tests" || s == "test" || s == "benches" || s == "examples" + }); + let test_stem = matches!( + path.file_stem().and_then(|s| s.to_str()), + Some("tests" | "test_utils" | "test_util" | "test_helpers" | "test_helper" | "testing") + ); + in_test_dir || test_stem +} + +struct Scanner<'a> { + counts: &'a mut FeatureCounts, +} + +impl<'ast> Visit<'ast> for Scanner<'_> { + fn visit_item_mod(&mut self, node: &'ast syn::ItemMod) { + if has_cfg_test(&node.attrs) { + self.counts.test_lines += span_lines(node); + return; + } + visit::visit_item_mod(self, node); + } + + fn visit_item_fn(&mut self, node: &'ast syn::ItemFn) { + if is_test_item(&node.attrs) { + self.counts.test_lines += span_lines(node); + return; + } + if node.sig.unsafety.is_some() { + self.counts.unsafe_fns += 1; + } + visit::visit_item_fn(self, node); + } + + fn visit_impl_item_fn(&mut self, node: &'ast syn::ImplItemFn) { + if is_test_item(&node.attrs) { + self.counts.test_lines += span_lines(node); + return; + } + if node.sig.unsafety.is_some() { + self.counts.unsafe_fns += 1; + } + visit::visit_impl_item_fn(self, node); + } + + fn visit_item_impl(&mut self, node: &'ast syn::ItemImpl) { + if is_test_item(&node.attrs) { + self.counts.test_lines += span_lines(node); + return; + } + if node.unsafety.is_some() { + self.counts.unsafe_impls += 1; + } + visit::visit_item_impl(self, node); + } + + fn visit_expr_unsafe(&mut self, node: &'ast syn::ExprUnsafe) { + self.counts.unsafe_blocks += 1; + visit::visit_expr_unsafe(self, node); + } + + fn visit_macro(&mut self, node: &'ast syn::Macro) { + if let Some(seg) = node.path.segments.last() { + match seg.ident.to_string().as_str() { + "panic" => self.counts.panic += 1, + "unreachable" => self.counts.unreachable += 1, + "unimplemented" => self.counts.unimplemented += 1, + "todo" => self.counts.todo += 1, + "assert" | "assert_eq" | "assert_ne" => self.counts.assert += 1, + _ => {} + } + } + visit::visit_macro(self, node); + } + + fn visit_expr_method_call(&mut self, node: &'ast syn::ExprMethodCall) { + // Disambiguate the std panicking methods from user-defined combinators of + // the same name (e.g. a parser's `self.expect(Token)` that returns Result). + // `Option/Result::unwrap` takes no args, and `expect` takes a single string + // message. syn has no type info, so the argument shape is the best signal. + match node.method.to_string().as_str() { + "unwrap" | "unwrap_err" if node.args.is_empty() => self.counts.unwrap += 1, + "expect" | "expect_err" if is_message_call(&node.args) => self.counts.expect += 1, + "unwrap_unchecked" | "unwrap_err_unchecked" if node.args.is_empty() => { + self.counts.unwrap_unchecked += 1; + } + _ => {} + } + visit::visit_expr_method_call(self, node); + } + + fn visit_expr_index(&mut self, node: &'ast syn::ExprIndex) { + self.counts.index += 1; + visit::visit_expr_index(self, node); + } + + fn visit_attribute(&mut self, node: &'ast syn::Attribute) { + if node.path().is_ident("derive") { + let _ = node.parse_nested_meta(|meta| { + if meta + .path + .segments + .last() + .is_some_and(|s| s.ident == "Serialize") + { + self.counts.serde_derive = true; + } + Ok(()) + }); + } + visit::visit_attribute(self, node); + } +} + +/// True if a method-call argument list looks like an `Option/Result::expect` +/// message: exactly one argument that is a string-ish expression. This excludes +/// user-defined `expect(Token)` combinators while accepting `.expect("msg")`, +/// `.expect(&format!(...))`, and similar. +fn is_message_call(args: &syn::punctuated::Punctuated) -> bool { + args.len() == 1 && is_message_expr(&args[0]) +} + +fn is_message_expr(expr: &syn::Expr) -> bool { + match expr { + syn::Expr::Lit(lit) => matches!(lit.lit, syn::Lit::Str(_)), + syn::Expr::Reference(r) => is_message_expr(&r.expr), + syn::Expr::Macro(m) => m + .mac + .path + .segments + .last() + .is_some_and(|s| matches!(s.ident.to_string().as_str(), "format" | "concat")), + _ => false, + } +} + +/// True if these attributes mark the item as test-only (`#[test]` or `#[cfg(test)]`, +/// including nested forms like `#[cfg(all(test, ...))]`). +fn is_test_item(attrs: &[syn::Attribute]) -> bool { + attrs.iter().any(|a| a.path().is_ident("test")) || has_cfg_test(attrs) +} + +/// True if any attribute is a `cfg` whose predicate mentions `test`. +fn has_cfg_test(attrs: &[syn::Attribute]) -> bool { + attrs.iter().any(|a| { + a.path().is_ident("cfg") && { + // Stringify so nested `all(test, ...)` / `any(test, ...)` are caught. + let tokens = a.meta.to_token_stream().to_string(); + tokens.contains("test") + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counts_constructs_and_skips_tests() { + let src = r#" + fn real() { + let a = foo().unwrap(); + let b = bar().expect("msg"); + let c = arr[3]; + // A user-defined combinator named like the std methods: must NOT count. + self.expect(TokenType::RParen); + let d = self.unwrap(node, extra); + if x { panic!("no") } + unreachable!(); + todo!(); + unsafe { do_thing(); } + } + + #[derive(Clone, Serialize)] + struct Ast; + + #[cfg(test)] + mod tests { + fn helper() { + let _ = thing().unwrap(); + let _ = other().expect("nope"); + } + } + + #[test] + fn a_test() { + let _ = z().unwrap(); + } + "#; + let file = syn::parse_file(src).expect("fixture parses"); + let mut counts = FeatureCounts::default(); + let mut scanner = Scanner { + counts: &mut counts, + }; + scanner.visit_file(&file); + + // Only the unwraps/expects in `real()` count, not the test module or #[test] fn. + assert_eq!(counts.unwrap, 1, "unwrap"); + assert_eq!(counts.expect, 1, "expect"); + assert_eq!(counts.index, 1, "index"); + assert_eq!(counts.panic, 1, "panic"); + assert_eq!(counts.unreachable, 1, "unreachable"); + assert_eq!(counts.todo, 1, "todo"); + assert_eq!(counts.unsafe_blocks, 1, "unsafe block"); + assert!(counts.serde_derive, "serde derive detected"); + } +} diff --git a/oracle/src/main.rs b/oracle/src/main.rs index 1a6236c..88305c6 100644 --- a/oracle/src/main.rs +++ b/oracle/src/main.rs @@ -2,7 +2,7 @@ //! //! For each reference dialect this brings up the actual engine in Docker, labels //! every corpus statement valid/invalid by the schema-free rule (a syntax/parse -//! error means invalid; no error or a schema/semantic error means it parsed, so +//! error means invalid. No error or a schema/semantic error means it parsed, so //! valid), and writes a committed cache under `oracle/labels/{dir}.tsv.zst` that //! `sql_ast_benchmark::oracle_cache` reads. Run locally with Docker: //! @@ -10,8 +10,8 @@ //! cargo run --release -p oracle -- sqlite # one or more by dir name //! //! Server engines (PostgreSQL, MySQL, ClickHouse, SQL Server) run in -//! testcontainers and connect over a mapped port; SQLite runs as the `sqlite3` -//! CLI in a one-shot container; DuckDB, which has no server and whose CLI errors +//! testcontainers and connect over a mapped port. SQLite runs as the `sqlite3` +//! CLI in a one-shot container. DuckDB, which has no server and whose CLI errors //! carry no line numbers, links the real `libduckdb` in-process via the `duckdb` //! crate. Each uses the engine's parse-only path where one exists. @@ -100,7 +100,7 @@ fn write_cache(dialect: Dialect, stmts: &[String], valid: &[bool]) -> Result<()> } /// PostgreSQL: real server in a container. Each statement runs inside a rolled -/// back transaction (PG has transactional DDL); invalid iff the SQLSTATE is +/// back transaction (PG has transactional DDL). Invalid iff the SQLSTATE is /// `42601` (syntax_error). Schema errors (42P01, 42703) and "cannot run in a /// transaction" (25xxx) are not syntax, so they count as valid (parsed fine). async fn label_postgresql(stmts: &[String]) -> Result> { @@ -146,7 +146,7 @@ async fn label_postgresql(stmts: &[String]) -> Result> { /// MySQL: real server in a container. We use `PREPARE`, MySQL's parse-only path: /// it parses (and name-resolves) without executing, so there are no side effects /// and nothing blocks. Invalid iff `PREPARE` fails with error 1064 -/// (ER_PARSE_ERROR); a missing table/column (1146/1054) or "unsupported in the +/// (ER_PARSE_ERROR). A missing table/column (1146/1054) or "unsupported in the /// prepared-statement protocol" (1295) means it parsed, so it is valid. async fn label_mysql(stmts: &[String]) -> Result> { use mysql_async::prelude::Queryable; @@ -190,7 +190,7 @@ async fn label_mysql(stmts: &[String]) -> Result> { /// ClickHouse: real server in a container, queried over HTTP. `EXPLAIN AST` /// parses only (no execution, no tables needed). Invalid iff the exception code -/// is 62 (SYNTAX_ERROR); any other code (unknown table/identifier, not +/// is 62 (SYNTAX_ERROR). Any other code (unknown table/identifier, not /// implemented) means it parsed, so it is valid. async fn label_clickhouse(stmts: &[String]) -> Result> { use testcontainers_modules::clickhouse::ClickHouse; @@ -289,7 +289,7 @@ async fn label_tsql(stmts: &[String]) -> Result> { /// DuckDB: real engine via the in-process `duckdb` crate (the actual libduckdb). /// DuckDB has no server, and its CLI errors carry no line numbers (so the /// container batch-correlation used for SQLite is unreliable), so we link the -/// real engine directly. `prepare` parses and binds without executing; a +/// real engine directly. `prepare` parses and binds without executing. A /// "Parser Error" is a syntax error (invalid), while a "Binder"/"Catalog Error" /// (unknown table or column) means it parsed, so it is valid. fn label_duckdb(stmts: &[String]) -> Result> { @@ -312,9 +312,9 @@ fn label_duckdb(stmts: &[String]) -> Result> { /// SQLite: real engine via the `sqlite3` CLI in a one-shot container. We feed a /// script of `EXPLAIN ;` (compiles, does not execute, so no side effects) /// and read stderr. `EXPLAIN` resolves names, so "no such table/column" surfaces -/// as a non-syntax error (valid); only a syntax error makes a statement invalid. +/// as a non-syntax error (valid). Only a syntax error makes a statement invalid. fn label_sqlite(stmts: &[String]) -> Result> { - // Script line 1 is `.bail off`; statement i is on line i + 2. + // Script line 1 is `.bail off`. Statement i is on line i + 2. let mut script = String::from(".bail off\n"); for s in stmts { script.push_str("EXPLAIN "); @@ -395,7 +395,7 @@ mod tests { #[test] fn missing_object_errors_are_valid() { - // The statement parsed; it only references objects we did not create. + // The statement parsed. It only references objects we did not create. assert!(!is_sqlite_invalid("no such table: documents")); assert!(!is_sqlite_invalid("no such column: x")); assert!(!is_sqlite_invalid("no such function: my_udf")); diff --git a/src/bin/sqlbench.rs b/src/bin/sqlbench.rs index 54c3a81..d4b9595 100644 --- a/src/bin/sqlbench.rs +++ b/src/bin/sqlbench.rs @@ -12,8 +12,9 @@ //! prints the per-dataset acceptance matrix instead //! of per-dialect reference metrics. //! export write `web/assets/bench.json.zst` for the explorer. -//! regen run the whole data pipeline (timing + memory -//! benches, then export) with one command. +//! regen run the whole data pipeline (feature scan + +//! depth probe + timing + memory benches + +//! time machine, then export) with one command. //! //! The grading logic lives in the library (`report`). This binary is argument //! dispatch plus table formatting. @@ -193,7 +194,7 @@ fn run_coverage() { /// The memory bench installs a custom global allocator, so it must run in its /// own process, separate from the timing bench (which must stay on the default /// allocator for fair numbers) and from export. That is why this shells out to -/// the timing and memory benches rather than calling them in-process; export +/// the timing and memory benches rather than calling them in-process. Export /// runs in-process at the end since it needs no special allocator. fn run_regen() { if let Err(e) = sql_ast_benchmark::datasets::ensure_corpus() { @@ -202,9 +203,34 @@ fn run_regen() { } // Each step writes under target/ (read by export) or, for the time-machine, // straight to web/assets/history.json.zst. The memory passes install a global - // allocator, so they are separate processes; the time-machine memory pass + // allocator, so they are separate processes. The time-machine memory pass // runs before its timing pass, which merges the memory sidecar. - let steps: [(&str, &[&str]); 5] = [ + let steps: [(&str, &[&str]); 7] = [ + // Static source-feature scan and recursion-depth probe, writing the + // committed featurescan/data/*.json the web bakes in. Independent of the + // benches, so run first. + ( + "cargo", + &[ + "run", + "--release", + "-p", + "featurescan", + "--bin", + "featurescan", + ], + ), // featurescan/data/featurescan.json + ( + "cargo", + &[ + "run", + "--release", + "-p", + "featurescan", + "--bin", + "featurescan-depth", + ], + ), // featurescan/data/depth.json ("cargo", &["bench"]), // target/bench_dist/ + target/batch_dist/ ("cargo", &["run", "--release", "-p", "membench"]), // target/mem_dist/ ( @@ -264,7 +290,7 @@ fn usage() -> ! { eprintln!("usage: sqlbench "); eprintln!(" correctness [--per-file] grade parsers over datasets/"); eprintln!(" export write web/assets/bench.json.zst for the site"); - eprintln!(" regen run timing + memory benches, then export"); + eprintln!(" regen run feature scan + depth probe + benches + time machine, then export"); std::process::exit(2); } diff --git a/src/export.rs b/src/export.rs index b84c5bb..c44bb41 100644 --- a/src/export.rs +++ b/src/export.rs @@ -140,6 +140,9 @@ fn metrics(report: &DialectReport) -> Vec { } else { pct(s.accepted_valid, report.valid_total) }, + attempted: s.attempted, + panicked: s.panicked, + panic_pct: pct(s.panicked, s.attempted), }) .collect() } @@ -235,7 +238,7 @@ struct BatchMemRow { /// Whether a batch parse consumed the whole accepted set, so its normalized cost /// can be trusted. A fail-fast parser that errors partway yields `n_parsed` -/// below `n_accepted`; statements with internal `;` only push `n_parsed` higher, +/// below `n_accepted`. Statements with internal `;` only push `n_parsed` higher, /// so `>=` is the right "fully consumed" test. const fn batch_complete(n_parsed: usize, n_accepted: usize) -> bool { n_accepted > 0 && n_parsed >= n_accepted @@ -294,7 +297,7 @@ fn read_batch_mem() -> Vec { /// Merge batch time and batch memory rows for one dialect into per-parser /// `ParserBatch`. A parser appears only if at least one axis parsed the whole -/// accepted set (see [`batch_complete`]); an axis whose batch bailed out early +/// accepted set (see [`batch_complete`]). An axis whose batch bailed out early /// is dropped to `None` so the explorer never shows a misleading number. Pure, /// so the merge and the guard are testable. fn batch_for(dir: &str, perf: &[BatchPerfRow], mem: &[BatchMemRow]) -> Vec { @@ -521,7 +524,7 @@ fn write_failure_tsv(file: &str, rejected: &[String], reasons: &[String]) -> std /// Build the TSV body for a failure download: a `statement\treason` header then /// up to `cap` rows. Each row is the rejected statement, a tab, then the parser's /// error message, with backslashes, tabs, and newlines escaped in both columns so -/// every row stays on a single line. `reasons` is aligned with `rejected`; a +/// every row stays on a single line. `reasons` is aligned with `rejected`. A /// missing reason is written as an empty cell. fn format_failure_tsv(rejected: &[String], reasons: &[String], cap: usize) -> String { fn escape(s: &str) -> String { @@ -704,7 +707,7 @@ mod tests { #[test] fn git_short_runs_without_panicking() { - // In the repo it returns Some(hash); the point is it does not panic and + // In the repo it returns Some(hash). The point is it does not panic and // yields a non-empty string when present. if let Some(h) = git_short() { assert!(!h.is_empty()); diff --git a/src/lib.rs b/src/lib.rs index 58337da..bdac671 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -115,53 +115,70 @@ fn panic_reason(p: &(dyn std::any::Any + Send)) -> String { .unwrap_or_else(|| "panicked".to_string()) } -/// Run a parse closure under panic protection, turning a panic into an error -/// message. Several parsers use `todo!()`/`panic!` on unimplemented paths, so -/// this keeps a worker alive and still yields a reason for the rejection. -fn catch_parse( - f: impl FnOnce() -> Result<(), String> + std::panic::UnwindSafe, -) -> Result<(), String> { - match std::panic::catch_unwind(f) { - Ok(r) => r, - Err(p) => Err(panic_reason(&*p)), +/// The outcome of one parse attempt, distinguishing a panic from an honest error. +/// +/// `try_parse`/`accepts` collapse `Rejected` and `Panicked` into the same +/// `Some(Err(..))`, which is right for correctness grading (a panic is still a +/// non-acceptance), but the empirical panic-rate metric needs them apart: how +/// often does a parser actually throw on real input instead of returning an error. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ParseOutcome { + /// The parser does not model this dialect (was `None` from `try_parse`). + Unsupported, + /// Parsed successfully. + Accepted, + /// Rejected with the parser's own error message. + Rejected(String), + /// Aborted with a caught panic, carrying the panic message. + Panicked(String), +} + +/// Run a parse closure under panic protection, classifying the result into a +/// [`ParseOutcome`] that distinguishes a clean rejection from a caught panic. +/// Several parsers use `todo!()`/`panic!`/`unreachable!` on unimplemented paths, +/// so this keeps the worker alive and records that the abort was a panic, which +/// the empirical panic-rate metric counts separately from honest `Err` returns. +fn catch_outcome(f: impl FnOnce() -> Result<(), String>) -> ParseOutcome { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) { + Ok(Ok(())) => ParseOutcome::Accepted, + Ok(Err(e)) => ParseOutcome::Rejected(e), + Err(p) => ParseOutcome::Panicked(panic_reason(&*p)), } } -fn qusql_try(sql: &str, d: SQLDialect) -> Result<(), String> { - catch_parse(|| { - let opts = ParseOptions::new() - .dialect(d) - .arguments(qusql_parse::SQLArguments::Dollar); - let mut issues = Issues::new(sql); - let _ = parse_statements(sql, &mut issues, &opts); - issues - .get() - .iter() - .find(|i| i.level == Level::Error) - .map_or(Ok(()), |e| Err(e.message.to_string())) - }) +// The `*_raw` helpers do the bare parse with NO panic protection, so the single +// `catch_outcome` wrapping them can tell a panic from an honest `Err`. They are +// only ever called inside `catch_outcome`. + +fn qusql_raw(sql: &str, d: SQLDialect) -> Result<(), String> { + let opts = ParseOptions::new() + .dialect(d) + .arguments(qusql_parse::SQLArguments::Dollar); + let mut issues = Issues::new(sql); + let _ = parse_statements(sql, &mut issues, &opts); + issues + .get() + .iter() + .find(|i| i.level == Level::Error) + .map_or(Ok(()), |e| Err(e.message.to_string())) } -fn databend_try(sql: &str, d: DatabendDialect) -> Result<(), String> { - catch_parse(|| { - let tokens = databend_tokenize(sql).map_err(|e| e.to_string())?; - databend_parse(&tokens, d) - .map(|_| ()) - .map_err(|e| e.to_string()) - }) +fn databend_raw(sql: &str, d: DatabendDialect) -> Result<(), String> { + let tokens = databend_tokenize(sql).map_err(|e| e.to_string())?; + databend_parse(&tokens, d) + .map(|_| ()) + .map_err(|e| e.to_string()) } -fn sqlite3_try(sql: &str) -> Result<(), String> { - catch_parse(|| { - let mut parser = sqlite3_parser::lexer::sql::Parser::new(sql.as_bytes()); - loop { - match parser.next() { - Ok(Some(_)) => {} - Ok(None) => return Ok(()), - Err(e) => return Err(e.to_string()), - } +fn sqlite3_raw(sql: &str) -> Result<(), String> { + let mut parser = sqlite3_parser::lexer::sql::Parser::new(sql.as_bytes()); + loop { + match parser.next() { + Ok(Some(_)) => {} + Ok(None) => return Ok(()), + Err(e) => return Err(e.to_string()), } - }) + } } fn sqlite3_reprint(sql: &str) -> Option { @@ -184,17 +201,15 @@ fn sqlite3_reprint(sql: &str) -> Option { .unwrap_or(None) } -fn turso_try(sql: &str) -> Result<(), String> { - catch_parse(|| { - let mut parser = turso_parser::parser::Parser::new(sql.as_bytes()); - loop { - match parser.next_cmd() { - Ok(Some(_)) => {} - Ok(None) => return Ok(()), - Err(e) => return Err(e.to_string()), - } +fn turso_raw(sql: &str) -> Result<(), String> { + let mut parser = turso_parser::parser::Parser::new(sql.as_bytes()); + loop { + match parser.next_cmd() { + Ok(Some(_)) => {} + Ok(None) => return Ok(()), + Err(e) => return Err(e.to_string()), } - }) + } } fn turso_reprint(sql: &str) -> Option { @@ -396,43 +411,67 @@ impl BenchParser { /// parser does not model the dialect, `Some(Ok(()))` = accepted, and /// `Some(Err(msg))` = rejected with the parser's own error message (or a /// panic message for parsers that abort on edge-case input). Panic-protected - /// like [`Self::accepts`], so it never unwinds the caller. + /// like [`Self::accepts`], so it never unwinds the caller. A panic and an + /// honest `Err` both map to `Some(Err(..))` here. Use [`Self::parse_outcome`] + /// when the distinction matters (the empirical panic-rate metric). #[must_use] pub fn try_parse(self, sql: &str, dialect: Dialect) -> Option> { + match self.parse_outcome(sql, dialect) { + ParseOutcome::Unsupported => None, + ParseOutcome::Accepted => Some(Ok(())), + ParseOutcome::Rejected(e) | ParseOutcome::Panicked(e) => Some(Err(e)), + } + } + + /// Parse `sql` in `dialect`, distinguishing a clean rejection from a caught + /// panic. Every parser is wrapped in one `catch_unwind` so a `panic!` / + /// `unreachable!` / `unwrap` on an unimplemented path becomes + /// [`ParseOutcome::Panicked`] rather than aborting the worker or hiding as a + /// plain rejection. This is the empirical counterpart to the static panic + /// scan: it counts the panics a parser actually throws on the real corpus. + #[must_use] + pub fn parse_outcome(self, sql: &str, dialect: Dialect) -> ParseOutcome { match self { - Self::Sqlparser => Some( + Self::Sqlparser => catch_outcome(|| { SqlparserParser::parse_sql(&*sqlparser_dialect(dialect), sql) .map(|_| ()) - .map_err(|e| e.to_string()), - ), - Self::PgQuery => (dialect == Dialect::Postgresql) - .then(|| pg_query::parse(sql).map(|_| ()).map_err(|e| e.to_string())), - Self::PgQuerySummary => (dialect == Dialect::Postgresql).then(|| { + .map_err(|e| e.to_string()) + }), + Self::PgQuery if dialect == Dialect::Postgresql => { + catch_outcome(|| pg_query::parse(sql).map(|_| ()).map_err(|e| e.to_string())) + } + Self::PgQuerySummary if dialect == Dialect::Postgresql => catch_outcome(|| { pg_query::summary(sql, -1) .map(|_| ()) .map_err(|e| e.to_string()) }), - Self::Qusql => qusql_dialect(dialect).map(|d| qusql_try(sql, d)), - Self::Polyglot => Some(catch_parse(|| { + Self::Qusql => qusql_dialect(dialect).map_or(ParseOutcome::Unsupported, |d| { + catch_outcome(|| qusql_raw(sql, d)) + }), + Self::Polyglot => catch_outcome(|| { polyglot_parse(sql, polyglot_dialect(dialect)) .map(|_| ()) .map_err(|e| e.to_string()) - })), - Self::Databend => databend_dialect_of(dialect).map(|d| databend_try(sql, d)), - Self::Orql => (dialect == Dialect::Oracle).then(|| { - catch_parse(|| { - orql_parser::parse(sql) - .map(|_| ()) - .map_err(|e| e.to_string()) - }) }), - Self::Sqlglot => Some(catch_parse(|| { + Self::Databend => databend_dialect_of(dialect).map_or(ParseOutcome::Unsupported, |d| { + catch_outcome(|| databend_raw(sql, d)) + }), + Self::Orql if dialect == Dialect::Oracle => catch_outcome(|| { + orql_parser::parse(sql) + .map(|_| ()) + .map_err(|e| e.to_string()) + }), + Self::Sqlglot => catch_outcome(|| { sqlglot_rust::parser::parse_statements(sql, sqlglot_dialect(dialect)) .map(|_| ()) .map_err(|e| e.to_string()) - })), - Self::Sqlite3 => (dialect == Dialect::Sqlite).then(|| sqlite3_try(sql)), - Self::Turso => (dialect == Dialect::Sqlite).then(|| turso_try(sql)), + }), + Self::Sqlite3 if dialect == Dialect::Sqlite => catch_outcome(|| sqlite3_raw(sql)), + Self::Turso if dialect == Dialect::Sqlite => catch_outcome(|| turso_raw(sql)), + // Single-dialect parsers asked about a dialect they do not model. + Self::PgQuery | Self::PgQuerySummary | Self::Orql | Self::Sqlite3 | Self::Turso => { + ParseOutcome::Unsupported + } } } @@ -508,10 +547,10 @@ impl BenchParser { /// not model the dialect or has no batch entry point ([`Self::can_batch`]). /// /// Fail-fast parsers (those returning a `Vec` or erroring on the first bad - /// statement) yield `0` if the whole batch fails; streaming parsers + /// statement) yield `0` if the whole batch fails. Streaming parsers /// (`sqlite3-parser`, `turso_parser`) yield the count parsed before the /// first error or EOF. Batches are built from already-accepted statements, - /// so a clean run parses all of them; the count is kept for coverage. + /// so a clean run parses all of them. The count is kept for coverage. #[must_use] pub fn parse_batch(self, sql: &str, dialect: Dialect) -> Option { match self { @@ -698,7 +737,7 @@ impl BenchParser { /// retained together). `None` when the parser has no batch entry point /// ([`Self::can_batch`], so databend), when its memory is invisible to the /// Rust allocator (the `libpg_query` bindings), or when it does not model - /// `dialect`. Called single-threaded from the `membench` binary; under any + /// `dialect`. Called single-threaded from the `membench` binary. Under any /// other binary the counters are zero and it returns `Some((0, 0))`. #[must_use] pub fn measure_mem_batch(self, sql: &str, dialect: Dialect) -> Option<(usize, usize)> { @@ -796,11 +835,11 @@ pub struct ParserId { /// A benchmarked parser, abstracted over the concrete library build. /// -/// [`BenchParser`] implements this for the current builds; the `timemachine` +/// [`BenchParser`] implements this for the current builds. The `timemachine` /// crate implements it for historical versions. The grading, timing, and memory /// drivers operate over `&dyn Parser`, so one implementation serves both. /// -/// Implementors provide the required methods; `accepts`, `measure_mem_batch`, +/// Implementors provide the required methods. `accepts`, `measure_mem_batch`, /// `roundtrips`, and `fidelity` have default implementations built on them /// (mirroring [`BenchParser`]'s inherent methods), so a historical version only /// needs the core parse hooks. @@ -830,6 +869,19 @@ pub trait Parser: Sync { self.try_parse(sql, dialect).map(|r| r.is_ok()) } + /// Parse, distinguishing a caught panic from an honest rejection. The default + /// is built on `try_parse`, which has already folded any panic into `Err`, so + /// it never reports `Panicked`. The current build ([`BenchParser`]) overrides + /// it with a panic-detecting implementation. Historical versions keep the + /// default (their panic rate is not measured, only the current build's is). + fn parse_outcome(&self, sql: &str, dialect: Dialect) -> ParseOutcome { + match self.try_parse(sql, dialect) { + None => ParseOutcome::Unsupported, + Some(Ok(())) => ParseOutcome::Accepted, + Some(Err(e)) => ParseOutcome::Rejected(e), + } + } + /// Whole-script `(peak, retained)`, gated on a batch entry point. fn measure_mem_batch(&self, sql: &str, dialect: Dialect) -> Option<(usize, usize)> { if self.can_batch() { @@ -881,7 +933,7 @@ impl Parser for BenchParser { family: self.name(), version: self.current_version(), // The current build's release date is filled by the timemachine - // registry (which owns the trend's dates); empty here is fine. + // registry (which owns the trend's dates). Empty here is fine. released: "", } } @@ -912,6 +964,9 @@ impl Parser for BenchParser { fn accepts(&self, sql: &str, dialect: Dialect) -> Option { (*self).accepts(sql, dialect) } + fn parse_outcome(&self, sql: &str, dialect: Dialect) -> ParseOutcome { + (*self).parse_outcome(sql, dialect) + } fn measure_mem_batch(&self, sql: &str, dialect: Dialect) -> Option<(usize, usize)> { (*self).measure_mem_batch(sql, dialect) } @@ -934,9 +989,47 @@ pub mod stats; #[cfg(test)] mod tests { - use super::{has_canonical, has_reference, reference_accepts, BenchParser}; + use super::ParseOutcome; + use super::{catch_outcome, has_canonical, has_reference, reference_accepts, BenchParser}; use crate::datasets::Dialect; + #[test] + fn catch_outcome_classifies_accept_reject_and_panic() { + assert_eq!(catch_outcome(|| Ok(())), ParseOutcome::Accepted); + assert_eq!( + catch_outcome(|| Err("bad".to_string())), + ParseOutcome::Rejected("bad".to_string()) + ); + // A panic in the parse closure becomes Panicked, not an abort. + match catch_outcome(|| panic!("boom")) { + ParseOutcome::Panicked(msg) => assert!(msg.contains("boom")), + other => panic!("expected Panicked, got {other:?}"), + } + } + + #[test] + fn parse_outcome_agrees_with_try_parse() { + let p = BenchParser::Sqlparser; + // Accepted and rejected map to the same verdicts try_parse reports. + assert_eq!( + p.parse_outcome("SELECT 1", Dialect::Postgresql), + ParseOutcome::Accepted + ); + assert!(matches!( + p.parse_outcome("SELECT 1 FROM", Dialect::Postgresql), + ParseOutcome::Rejected(_) + )); + // A dialect the parser does not model is Unsupported (try_parse -> None). + assert_eq!( + BenchParser::Sqlite3.parse_outcome("SELECT 1", Dialect::Postgresql), + ParseOutcome::Unsupported + ); + assert_eq!( + BenchParser::Sqlite3.try_parse("SELECT 1", Dialect::Postgresql), + None + ); + } + const ALL_DIALECTS: [Dialect; 13] = [ Dialect::Postgresql, Dialect::Mysql, @@ -1120,7 +1213,7 @@ mod tests { #[test] fn can_batch_excludes_only_databend() { - // databend-common-ast parses one statement per call; every other parser + // databend-common-ast parses one statement per call. Every other parser // has a multi-statement entry point. assert!(!BenchParser::Databend.can_batch()); for p in BenchParser::all() { diff --git a/src/report.rs b/src/report.rs index eb61687..f37e3fd 100644 --- a/src/report.rs +++ b/src/report.rs @@ -33,6 +33,13 @@ pub struct ParserStat { pub roundtrip_ok: usize, /// Reference-fidelity-preserving among accepted-valid. pub fidelity_ok: usize, + /// Statements the parser attempted in this dialect (the panic-rate + /// denominator): every graded statement, since a supporting parser is run on + /// all of them. Zero for a parser that does not model the dialect. + pub attempted: usize, + /// Statements on which the parser threw a caught panic instead of returning a + /// result (the empirical panic-rate numerator). + pub panicked: usize, } impl ParserStat { @@ -41,6 +48,8 @@ impl ParserStat { self.accepted_invalid += other.accepted_invalid; self.roundtrip_ok += other.roundtrip_ok; self.fidelity_ok += other.fidelity_ok; + self.attempted += other.attempted; + self.panicked += other.panicked; } } @@ -111,8 +120,20 @@ pub fn grade_chunk(stmts: &[String], dialect: Dialect, parsers: &[&dyn Parser]) } for (i, &p) in parsers.iter().enumerate() { - if p.accepts(sql, dialect) != Some(true) { - continue; + // A panic is still a non-acceptance (it does not enter the accepted + // tallies), but it is counted separately for the panic-rate metric. + match p.parse_outcome(sql, dialect) { + crate::ParseOutcome::Unsupported => continue, + crate::ParseOutcome::Panicked(_) => { + report.stats[i].attempted += 1; + report.stats[i].panicked += 1; + continue; + } + crate::ParseOutcome::Rejected(_) => { + report.stats[i].attempted += 1; + continue; + } + crate::ParseOutcome::Accepted => report.stats[i].attempted += 1, } if is_valid { report.stats[i].accepted_valid += 1; @@ -419,6 +440,76 @@ mod tests { p } + /// A parser whose outcome is driven by the statement text, so grading of + /// panics can be exercised deterministically without a real parser that + /// happens to panic. "OK" is accepted, "PANIC" panics, anything else rejects. + struct MockParser; + + impl crate::Parser for MockParser { + fn id(&self) -> crate::ParserId { + crate::ParserId { + family: "mock", + version: "0", + released: "", + } + } + fn supports(&self, _d: Dialect) -> bool { + true + } + fn try_parse(&self, sql: &str, dialect: Dialect) -> Option> { + match self.parse_outcome(sql, dialect) { + crate::ParseOutcome::Unsupported => None, + crate::ParseOutcome::Accepted => Some(Ok(())), + crate::ParseOutcome::Rejected(e) | crate::ParseOutcome::Panicked(e) => Some(Err(e)), + } + } + fn parse_outcome(&self, sql: &str, _d: Dialect) -> crate::ParseOutcome { + match sql { + "OK" => crate::ParseOutcome::Accepted, + "PANIC" => crate::ParseOutcome::Panicked("boom".to_string()), + _ => crate::ParseOutcome::Rejected("nope".to_string()), + } + } + fn parse_once(&self, sql: &str, _d: Dialect) -> bool { + sql == "OK" + } + fn parse_batch(&self, _sql: &str, _d: Dialect) -> Option { + None + } + fn can_batch(&self) -> bool { + false + } + fn measure_mem(&self, _sql: &str, _d: Dialect) -> Option<(usize, usize)> { + None + } + fn reprint(&self, _sql: &str, _d: Dialect) -> Option { + None + } + fn can_reprint(&self, _d: Dialect) -> bool { + false + } + } + + #[test] + fn grade_chunk_counts_panics_separately_from_rejections() { + let stmts = vec![ + "OK".to_string(), + "PANIC".to_string(), + "PANIC".to_string(), + "reject".to_string(), + ]; + let mock = MockParser; + let parsers: [&dyn Parser; 1] = [&mock]; + // Multi is a provenance dialect: every statement is treated as valid. + let r = grade_chunk(&stmts, Dialect::Multi, &parsers); + let s = &r.stats[0]; + assert_eq!(s.attempted, 4, "every statement attempted"); + assert_eq!(s.panicked, 2, "two panics counted"); + assert_eq!(s.accepted_valid, 1, "only OK accepted"); + // A panic is a non-acceptance: it must not inflate the accepted tallies. + assert_eq!(s.accepted_invalid, 0); + } + #[test] fn provenance_dialect_treats_everything_as_valid() { let stmts = vec!["SELECT 1".to_string()]; diff --git a/timemachine/src/families/sqlite3.rs b/timemachine/src/families/sqlite3.rs index 4d4d2f7..59baffd 100644 --- a/timemachine/src/families/sqlite3.rs +++ b/timemachine/src/families/sqlite3.rs @@ -1,5 +1,5 @@ //! Historical sqlite3-parser (lemon-rs) versions. SQLite only, a streaming -//! `FallibleIterator` of commands; reprints via each command's `Display`. +//! `FallibleIterator` of commands. Reprints via each command's `Display`. use fallible_iterator::FallibleIterator as _; use sql_ast_benchmark::datasets::Dialect; diff --git a/timemachine/src/families/sqlparser.rs b/timemachine/src/families/sqlparser.rs index 8c2c4a2..8b3f693 100644 --- a/timemachine/src/families/sqlparser.rs +++ b/timemachine/src/families/sqlparser.rs @@ -2,7 +2,7 @@ //! //! Adding a version is three lines: a Cargo alias, one `sqlparser_version!` //! invocation here, and one entry in the registry. Versions that share the -//! public API are covered by the macro; an API break would get its own block. +//! public API are covered by the macro. An API break would get its own block. use sql_ast_benchmark::datasets::Dialect; use sql_ast_benchmark::{Parser, ParserId}; diff --git a/timemachine/src/families/turso.rs b/timemachine/src/families/turso.rs index f565cc3..356410a 100644 --- a/timemachine/src/families/turso.rs +++ b/timemachine/src/families/turso.rs @@ -1,5 +1,5 @@ //! turso_parser (the SQLite parser from Turso). SQLite only, a streaming -//! `next_cmd` loop; reprints via each command's `Display`. Only one stable +//! `next_cmd` loop. Reprints via each command's `Display`. Only one stable //! release is published, so the history is a single point. use sql_ast_benchmark::datasets::Dialect; diff --git a/timemachine/src/run.rs b/timemachine/src/run.rs index 78a5227..b283d0a 100644 --- a/timemachine/src/run.rs +++ b/timemachine/src/run.rs @@ -5,7 +5,7 @@ //! ([`stats::perf_from`], [`stats::dist_from`]) so the history is computed the //! same way as the current snapshot. Timing and memory run as separate binaries //! (the memory one installs a global allocator), each producing part of the -//! history; the timing binary merges in the memory sidecar and writes the final +//! history. The timing binary merges in the memory sidecar and writes the final //! per-family file. use sql_ast_benchmark::batch::join_batch; @@ -159,6 +159,12 @@ fn metrics_of(report: &report::DialectReport, dialect: Dialect) -> ParserMetrics } else { pct(s.accepted_valid, report.valid_total) }, + // The time machine does not measure the empirical panic rate (only the + // current build does, via BenchParser's panic-detecting parse_outcome), so + // it is left unmeasured rather than reported as a misleading zero. + attempted: s.attempted, + panicked: 0, + panic_pct: None, } } diff --git a/viz/src/chart.rs b/viz/src/chart.rs index b810d14..493e962 100644 --- a/viz/src/chart.rs +++ b/viz/src/chart.rs @@ -22,7 +22,7 @@ type Res = Result<(), Box>; /// Pixels reserved on the right of each chart for the legend, sized to the /// widest label so short-label charts (e.g. a single-dialect parser page) do /// not get a wide empty band while long-label charts still fit. The 34px swatch -/// indent precedes the text; ~5px/char approximates 11px sans-serif, plus a +/// indent precedes the text. ~5px/char approximates 11px sans-serif, plus a /// 16px right margin. Clamped so even one short label keeps a sane band. fn legend_width(lines: &[Line]) -> i32 { let max_chars = lines @@ -423,7 +423,7 @@ pub fn trend_lines(title: &str, series: &[TrendSeries], w: u32, h: u32, y_desc: if !xmin.is_finite() || !xmax.is_finite() { return Ok(()); // no data } - // Pad the x range; a single-release family still gets a sane window. + // Pad the x range. A single-release family still gets a sane window. let xpad = ((xmax - xmin) * 0.08).max(0.08); let (xlo, xhi) = (xmin - xpad, xmax + xpad); if !ymin.is_finite() || ymin <= 0.0 { diff --git a/viz/src/lib.rs b/viz/src/lib.rs index 3e5aa9a..1e6c640 100644 --- a/viz/src/lib.rs +++ b/viz/src/lib.rs @@ -17,6 +17,7 @@ pub use chart::{ }; pub use color::{parser_hex, parser_rgb}; pub use schema::{ - Bundle, CoverageFile, CoverageMatrix, DialectData, DialectRun, FamilyHistory, MemDist, - ParserBatch, ParserFailures, ParserMem, ParserMetrics, ParserPerf, VersionRun, + Bundle, CoverageFile, CoverageMatrix, DepthReport, DepthScan, DialectData, DialectRun, + FamilyHistory, FeatureCounts, FeatureScan, LintPolicy, MemDist, ParserBatch, ParserFailures, + ParserFeatures, ParserMem, ParserMetrics, ParserPerf, VersionRun, }; diff --git a/viz/src/schema.rs b/viz/src/schema.rs index b00df9d..c2015b5 100644 --- a/viz/src/schema.rs +++ b/viz/src/schema.rs @@ -1,8 +1,123 @@ //! The committed-snapshot JSON schema, shared by `sqlbench export` (serialize) //! and the `web` viewer (deserialize). +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; +/// Static source-feature scan of one parser's library `src/` (panic-inducing +/// constructs, unsafe usage, lint policy, dependency footprint). Produced by the +/// `featurescan` crate and baked into the web metadata. See that crate for the +/// counting rules and caveats (counts are a code-smell proxy, not a crash proof). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FeatureScan { + pub note: String, + pub parsers: Vec, +} + +/// One parser's static-scan results. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ParserFeatures { + /// Display name, matching the parser-page name. + pub parser: String, + pub package: String, + pub version: String, + pub counts: FeatureCounts, + pub lints: LintPolicy, + /// The crate sets `forbid(unsafe_code)`. + pub forbids_unsafe: bool, + /// Direct, non-dev, non-build dependencies. + pub direct_deps: usize, + /// The crate depends on serde (AST serialization is plausible). + pub serde_dep: bool, +} + +/// Library-source construct counts (panic families, unsafe, LOC). All counts +/// exclude tests, benches, examples, `#[cfg(test)]` items, and test-helper files. +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct FeatureCounts { + pub loc: usize, + pub test_lines: usize, + /// Non-test lines (`loc - test_lines`), the per-KLOC density denominator. + pub code_loc: usize, + pub files: usize, + pub parse_failures: usize, + pub panic: usize, + pub unreachable: usize, + pub unimplemented: usize, + pub todo: usize, + pub assert: usize, + pub unwrap: usize, + pub expect: usize, + pub unwrap_unchecked: usize, + pub index: usize, + pub unsafe_blocks: usize, + pub unsafe_fns: usize, + pub unsafe_impls: usize, + pub serde_derive: bool, +} + +impl FeatureCounts { + /// Hard, unconditional panics: `panic!`, `unreachable!`, `unimplemented!`, `todo!`. + #[must_use] + pub const fn hard_panics(&self) -> usize { + self.panic + self.unreachable + self.unimplemented + self.todo + } + + /// Total unsafe occurrences (blocks, functions, impls). + #[must_use] + pub const fn unsafe_total(&self) -> usize { + self.unsafe_blocks + self.unsafe_fns + self.unsafe_impls + } +} + +/// A parser's own panic-relevant lint policy: which lints it bans by design. +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct LintPolicy { + /// Lint bare name (e.g. `unwrap_used`) -> level (`forbid`/`deny`/`warn`/`allow`). + pub lints: BTreeMap, + /// The crate inherits `[lints]` from a workspace not resolvable from the + /// published package, so any policy there is invisible to the scan. + pub workspace_inherited: bool, +} + +impl LintPolicy { + /// True if the lint is set to `deny` or `forbid` (a build-failing ban). + #[must_use] + pub fn is_banned(&self, lint: &str) -> bool { + matches!( + self.lints.get(lint).map(String::as_str), + Some("deny" | "forbid") + ) + } +} + +/// Recursion-depth probe results across parsers. Produced by `featurescan-depth`. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct DepthScan { + pub note: String, + pub stack_bytes: usize, + pub ceil: usize, + pub parsers: Vec, +} + +/// One parser's recursion-depth result. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct DepthReport { + pub parser: String, + pub dialect: String, + /// Rejects deep input cleanly and never overflows up to the ceiling. + pub guarded: bool, + /// The parser does not accept the probe shape even at depth 1, so its graceful + /// limit cannot be read from this shape (the crash depth is still valid). + pub shape_rejected: bool, + /// Smallest depth rejected instead of accepted (graceful recursion limit). + pub limit_depth: Option, + /// Smallest depth that overflows the stack (None = never, up to the ceiling). + pub crash_depth: Option, + pub ceil: usize, +} + /// Top-level results bundle (one committed `bench.json.zst`). #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Bundle { @@ -46,7 +161,7 @@ pub struct DialectData { /// Whole-script (batch) parse results for one parser in one dialect. /// /// The parser's whole accepted set is concatenated into a single script and -/// parsed in one call; the cost is divided by the statement count. This +/// parsed in one call, and the cost is divided by the statement count. This /// complements the per-statement [`ParserPerf`]/[`ParserMem`], exposing the /// amortization a batch API gains or loses (a grown `Vec` of statements, all /// ASTs held at once). The values are means (total over count), so they compare @@ -106,7 +221,7 @@ pub struct MemDist { /// A preview of the statements one parser rejected in one dialect, plus the /// info needed to offer the full set as a download. The full list is shipped -/// separately as a committed `.tsv.zst` file (see `download`); only a short +/// separately as a committed `.tsv.zst` file (see `download`). Only a short /// preview is embedded in the JSON to keep it small. #[derive(Serialize, Deserialize, Clone, Debug)] pub struct ParserFailures { @@ -152,6 +267,20 @@ pub struct ParserMetrics { pub fidelity_pct: Option, /// Provenance dialects: fraction of the corpus accepted. pub accept_pct: Option, + /// Statements the parser attempted in this dialect (the panic-rate + /// denominator), so per-parser rates can be aggregated across dialects. 0 in + /// older snapshots. + #[serde(default)] + pub attempted: usize, + /// Statements on which the parser threw a caught panic instead of returning a + /// result (0 in older snapshots, and in historical time-machine versions whose + /// panic rate is not measured). + #[serde(default)] + pub panicked: usize, + /// Empirical panic rate: panics as a fraction of statements attempted in this + /// dialect. `None` when nothing was attempted or the value is unmeasured. + #[serde(default)] + pub panic_pct: Option, } /// Timing distribution for one parser in one dialect. diff --git a/web/assets/bench.json.zst b/web/assets/bench.json.zst index 8ffba9c..5665632 100644 Binary files a/web/assets/bench.json.zst and b/web/assets/bench.json.zst differ diff --git a/web/assets/history.json.zst b/web/assets/history.json.zst index f8059ad..16449cb 100644 Binary files a/web/assets/history.json.zst and b/web/assets/history.json.zst differ diff --git a/web/src/components.rs b/web/src/components.rs index 4d1f58b..be79c81 100644 --- a/web/src/components.rs +++ b/web/src/components.rs @@ -6,11 +6,11 @@ use crate::Route; use dioxus::prelude::*; use dioxus_free_icons::icons::fa_brands_icons::{FaGit, FaGithub, FaRust}; use dioxus_free_icons::icons::fa_solid_icons::{ - FaArrowLeftLong, FaArrowsRotate, FaBox, FaBug, FaBuilding, FaCalendarDays, FaChartColumn, - FaChartLine, FaCode, FaCodeCommit, FaCodeFork, FaCopy, FaCube, FaDatabase, FaDownload, - FaFlaskVial, FaHeartPulse, FaMicrochip, FaMobileScreen, FaScaleBalanced, FaServer, - FaShieldHalved, FaStar, FaStopwatch, FaTableCells, FaTag, FaTriangleExclamation, FaUsers, - FaVial, + FaArrowLeftLong, FaArrowsRotate, FaBomb, FaBox, FaBug, FaBuilding, FaCalendarDays, + FaChartColumn, FaChartLine, FaCode, FaCodeCommit, FaCodeFork, FaCopy, FaCube, FaDatabase, + FaDownload, FaFlaskVial, FaHeartPulse, FaLayerGroup, FaMicrochip, FaMobileScreen, + FaScaleBalanced, FaServer, FaShieldHalved, FaSitemap, FaStar, FaStopwatch, FaTableCells, FaTag, + FaTriangleExclamation, FaUsers, FaVial, }; use dioxus_free_icons::Icon; use std::cmp::Ordering; @@ -485,7 +485,7 @@ fn download_js(fig_id: &str, filename: &str, png: bool) -> String { /// A chart figure: the inline SVG plus a caption and PNG/SVG download buttons. /// `id` must be unique per figure on the page (the download script locates the -/// SVG by it); `filename` is the saved file's base name (no extension). +/// SVG by it). `filename` is the saved file's base name (no extension). fn chart_figure(id: &str, svg: &str, aria_label: &str, caption: &str, filename: &str) -> Element { let png_js = download_js(id, filename, true); let svg_js = download_js(id, filename, false); @@ -1431,6 +1431,11 @@ fn parser_meta_pills(parser: &str) -> Element { let Some(m) = parser_meta(parser) else { return rsx! {}; }; + // Source- and behavior-mined badges (panic discipline, empirical panic rate, + // unsafe, recursion depth, deps, serde), shown alongside the repo/crate facts. + let feat = crate::data::parser_features(parser); + let depth = crate::data::parser_depth(parser); + let panic = crate::data::panic_totals(parser); rsx! { div { class: "meta-grid", @@ -1451,13 +1456,210 @@ fn parser_meta_pills(parser: &str) -> Element { {meta_flag(rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaMicrochip } }, "no_std", if m.no_std { "yes".to_string() } else { "no".to_string() }, m.no_std, crate::metadata::no_std_description(m.no_std))} {meta_flag(rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaCube } }, "wasm", if m.wasm { "yes".to_string() } else { "no".to_string() }, m.wasm, crate::metadata::wasm_description(m.wasm))} {meta_flag(rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaRust } }, "impl", if m.pure_rust { "pure Rust".to_string() } else { "C FFI".to_string() }, m.pure_rust, crate::metadata::pure_rust_description(m.pure_rust))} - {meta_flag(rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaShieldHalved } }, "unsafe", if m.unsafe_note.is_empty() { "none".to_string() } else { "uses".to_string() }, m.unsafe_note.is_empty(), &crate::metadata::unsafe_description(m.unsafe_note))} {meta_flag(rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaHeartPulse } }, "maintained", if crate::metadata::maintained(m.last_release) { "active".to_string() } else { "stale".to_string() }, crate::metadata::maintained(m.last_release), &crate::metadata::maintenance_description(m.last_release))} {meta_flag(rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaFlaskVial } }, "miri/san", if m.sanitizers.is_empty() { "no".to_string() } else { m.sanitizers.to_string() }, !m.sanitizers.is_empty(), &crate::metadata::sanitizer_description(m.sanitizers))} + {feat.map_or_else(|| rsx! {}, panic_discipline_pill)} + {empirical_panic_pill(panic)} + {feat.map_or_else(|| rsx! {}, |f| unsafe_pill(f, m.unsafe_note))} + {depth.map_or_else(|| rsx! {}, depth_pill)} + {feat.map_or_else(|| rsx! {}, deps_pill)} + {feat.map_or_else(|| rsx! {}, serde_pill)} } } } +/// Static panic-discipline pill: banned by lint, clean (none present), or a count +/// of panic-inducing constructs. Neutral (informational): the count is a +/// code-smell proxy, not a crash proof, so the empirical pill carries the alarm. +fn panic_discipline_pill(f: &viz::ParserFeatures) -> Element { + let c = &f.counts; + let hard = c.hard_panics(); + let fallible = c.unwrap + c.expect + c.unwrap_unchecked; + let total = hard + fallible; + let banned = f.lints.is_banned("unwrap_used") + || f.lints.is_banned("panic") + || f.lints.is_banned("expect_used"); + let value = if banned { + "banned".to_string() + } else if total == 0 { + "clean".to_string() + } else { + total.to_string() + }; + let desc = if banned { + format!( + "Panic discipline: bans panic-inducing lints by design (a regression fails the build). \ + Static scan still found {total} construct(s) in library source ({hard} hard panics \ + like panic!/unreachable!, {} unwrap, {} expect). Counts exclude tests and are a \ + code-smell proxy, not a crash proof.", + c.unwrap, c.expect + ) + } else if total == 0 { + "Panic discipline: no panic!, unreachable!, unwrap, or expect in library source (excluding tests), though no lint enforces this.".to_string() + } else { + format!( + "Panic discipline: {total} panic-inducing construct(s) in library source: {hard} hard \ + panics (panic!/unreachable!/unimplemented!/todo!), {} unwrap, {} expect, over {} \ + non-test lines. Counts exclude tests and are a code-smell proxy, not a crash proof. \ + See the empirical panic rate for observed behavior.", + c.unwrap, c.expect, c.code_loc + ) + }; + meta_item( + rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaTriangleExclamation } }, + "panic discipline", + value, + desc, + ) +} + +/// Empirical panic-rate pill: how often the parser actually panics on the real +/// corpus rather than returning an error. The real risk signal, flagged red when +/// any panic is observed. +fn empirical_panic_pill(totals: Option<(usize, usize)>) -> Element { + let (value, ok, desc) = match totals { + None => ( + "n/a".to_string(), + true, + "Empirical panic rate not measured in this snapshot.".to_string(), + ), + Some((0, attempted)) => ( + "0".to_string(), + true, + format!( + "Empirical panic rate: 0 panics across {attempted} statements parsed on the real \ + corpus. It returns errors instead of panicking." + ), + ), + Some((panicked, attempted)) => { + let pct = 100.0 * panicked as f64 / attempted as f64; + ( + format!("{panicked} ({pct:.3}%)"), + false, + format!( + "Empirical panic rate: {panicked} of {attempted} statements ({pct:.3}%) made \ + the parser panic (caught) on the real corpus instead of returning an error. \ + This is observed behavior, not a code smell." + ), + ) + } + }; + meta_flag( + rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaBomb } }, + "panics on corpus", + value, + ok, + &desc, + ) +} + +/// Unsafe-surface pill: forbidden, none, or a count of unsafe occurrences. +fn unsafe_pill(f: &viz::ParserFeatures, note: &str) -> Element { + let total = f.counts.unsafe_total(); + let (value, ok) = if f.forbids_unsafe { + ("forbidden".to_string(), true) + } else if total == 0 { + ("none".to_string(), true) + } else { + (total.to_string(), false) + }; + let note_suffix = if note.is_empty() { + String::new() + } else { + format!(" Used for: {note}.") + }; + let desc = if f.forbids_unsafe { + "Unsafe: forbids unsafe code (#![forbid(unsafe_code)]), so the crate cannot contain any unsafe block.".to_string() + } else if total == 0 { + "Unsafe: no unsafe code in library source.".to_string() + } else { + format!( + "Unsafe: {total} unsafe occurrence(s) in library source ({} blocks, {} fns, {} impls).{note_suffix}", + f.counts.unsafe_blocks, f.counts.unsafe_fns, f.counts.unsafe_impls + ) + }; + meta_flag( + rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaShieldHalved } }, + "unsafe", + value, + ok, + &desc, + ) +} + +/// Recursion-depth pill: depth-guarded (clean error, no overflow) or the depth at +/// which deeply nested input overflows the stack and aborts the process. +fn depth_pill(d: &viz::DepthReport) -> Element { + let (value, ok, desc) = match d.crash_depth { + None => { + let v = d + .limit_depth + .map_or_else(|| "guarded".to_string(), |l| format!("guarded ({l})")); + let detail = d.limit_depth.map_or_else( + || format!("handled nesting up to {} without overflowing (it does not recurse on the call stack)", d.ceil), + |l| format!("rejects deeply nested input with a clean error at depth {l} and never overflows the stack (probed to {})", d.ceil), + ); + ( + v, + true, + format!("Recursion depth: depth-guarded, it {detail}, on an 8 MiB stack."), + ) + } + Some(crash) => ( + format!("crashes @{crash}"), + false, + format!( + "Recursion depth: stack-overflows on nested input at depth {crash} (8 MiB stack), \ + aborting the whole process. Deeply nested SQL is a denial-of-service risk unless \ + input depth is bounded." + ), + ), + }; + meta_flag( + rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaLayerGroup } }, + "recursion depth", + value, + ok, + &desc, + ) +} + +/// Dependency-footprint pill: direct (non-dev) dependency count. Neutral. +fn deps_pill(f: &viz::ParserFeatures) -> Element { + meta_item( + rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaSitemap } }, + "deps", + f.direct_deps.to_string(), + format!( + "Dependency footprint: {} direct, non-dev dependencies. Fewer means a lighter, \ + faster-compiling addition to a project.", + f.direct_deps + ), + ) +} + +/// AST-serializability pill: whether the parser derives serde::Serialize on its +/// types, so the parse tree can be serialized. Neutral capability flag. +fn serde_pill(f: &viz::ParserFeatures) -> Element { + let yes = f.counts.serde_derive; + let value = if yes { + "yes".to_string() + } else { + "no".to_string() + }; + let desc = if yes { + "Serializable AST: the crate derives serde::Serialize on its types, so the parse tree can be serialized to JSON and similar." + } else { + "No serde derive found on the crate's types, so the AST is not directly serializable without custom code." + }; + meta_item( + rsx! { Icon { width: 12, height: 12, fill: "currentColor".to_string(), icon: FaDatabase } }, + "serde AST", + value, + desc.to_string(), + ) +} + fn ratio_pct(n: usize, base: usize) -> String { if base == 0 { "N/A".to_string() @@ -1665,7 +1867,7 @@ fn col_help(name: &str) -> Option<&'static str> { } /// A generic click-to-sort data table. `corner` labels the first (header) -/// column; `columns` are the value-column labels. Clicking any header toggles +/// column, and `columns` are the value-column labels. Clicking any header toggles /// ascending / descending on that column. `footer`, if present, is a row /// pinned below the sorted rows (used for the coverage subtotal). #[component] @@ -1891,7 +2093,7 @@ fn memory_table(d: &DialectData) -> Element { "Memory" } p { class: "table-cap", - "Bytes per statement, measured with a counting allocator. \"peak\" is the high-water mark of live memory during the parse, \"retained\" what the produced AST keeps alive afterwards. \"peak mean\" and \"retained mean\" are the per-statement averages, and \"batch peak/stmt\" and \"batch ret/stmt\" are the same over the whole accepted set parsed as one script divided by its statement count, so compare each batch column to the adjacent mean (batch retained is higher when every statement's AST is held at once; blank where not measured or no batch entry point). The libpg_query bindings are omitted (they parse in C, invisible to the Rust allocator)." + "Bytes per statement, measured with a counting allocator. \"peak\" is the high-water mark of live memory during the parse, \"retained\" what the produced AST keeps alive afterwards. \"peak mean\" and \"retained mean\" are the per-statement averages, and \"batch peak/stmt\" and \"batch ret/stmt\" are the same over the whole accepted set parsed as one script divided by its statement count, so compare each batch column to the adjacent mean (batch retained is higher when every statement's AST is held at once, blank where not measured or no batch entry point). The libpg_query bindings are omitted (they parse in C, invisible to the Rust allocator)." } div { class: "charts", {chart_figure(&format!("chart-{}-mempeak-ecdf", d.dir_name), &peak_ecdf, &format!("Empirical CDF of peak memory for {}, one curve per parser.", d.display_name), "Peak live memory per parse, one curve per parser. Further left is leaner (log scale).", &format!("{}-peak-memory-ecdf", d.dir_name))} diff --git a/web/src/data.rs b/web/src/data.rs index cc91379..6836474 100644 --- a/web/src/data.rs +++ b/web/src/data.rs @@ -3,11 +3,11 @@ //! Both the main bundle and the time-machine history are zstd-compressed and //! embedded via `include_bytes!`, then decompressed in wasm with `ruzstd`. //! Embedding (rather than a runtime fetch) keeps the viewer immune to GitHub -//! Pages base-path fetch pitfalls; compressing keeps the wasm payload small +//! Pages base-path fetch pitfalls. Compressing keeps the wasm payload small //! (the bundle is ~25x smaller compressed). use std::sync::OnceLock; -use viz::{Bundle, FamilyHistory}; +use viz::{Bundle, DepthReport, DepthScan, FamilyHistory, FeatureScan, ParserFeatures}; /// The results bundle, zstd-compressed and embedded. static BUNDLE_RAW: &[u8] = include_bytes!("../assets/bench.json.zst"); @@ -15,6 +15,13 @@ static BUNDLE_RAW: &[u8] = include_bytes!("../assets/bench.json.zst"); /// Combined time-machine history for every family, zstd-compressed and embedded. static HISTORY_RAW: &[u8] = include_bytes!("../assets/history.json.zst"); +/// Static source-feature scan (panic discipline, unsafe, lints, deps). Small and +/// committed uncompressed, so embedded as a string and parsed once. +static FEATURESCAN_RAW: &str = include_str!("../../featurescan/data/featurescan.json"); + +/// Recursion-depth probe results, committed uncompressed. +static DEPTH_RAW: &str = include_str!("../../featurescan/data/depth.json"); + /// Decompress an embedded zstd blob to bytes. fn unzstd(raw: &[u8]) -> Vec { let mut decoder = ruzstd::StreamingDecoder::new(raw).expect("embedded blob is valid zstd"); @@ -45,6 +52,46 @@ pub fn history(family: &str) -> Option<&'static FamilyHistory> { histories().iter().find(|h| h.family == family) } +/// The static source-feature scan (parsed once). +fn featurescan() -> &'static FeatureScan { + static CACHE: OnceLock = OnceLock::new(); + CACHE.get_or_init(|| serde_json::from_str(FEATURESCAN_RAW).expect("featurescan.json is valid")) +} + +/// The static-scan features for one parser, by display name. +#[must_use] +pub fn parser_features(parser: &str) -> Option<&'static ParserFeatures> { + featurescan().parsers.iter().find(|p| p.parser == parser) +} + +/// The recursion-depth probe results (parsed once). +fn depth_scan() -> &'static DepthScan { + static CACHE: OnceLock = OnceLock::new(); + CACHE.get_or_init(|| serde_json::from_str(DEPTH_RAW).expect("depth.json is valid")) +} + +/// The recursion-depth result for one parser, by display name. +#[must_use] +pub fn parser_depth(parser: &str) -> Option<&'static DepthReport> { + depth_scan().parsers.iter().find(|p| p.parser == parser) +} + +/// Aggregate empirical panic totals for one parser across every dialect it runs: +/// `(panicked, attempted)`. The per-parser panic rate is `panicked / attempted`. +/// Returns `None` if nothing was attempted (e.g. an older snapshot without the +/// panic fields), so the caller can show the badge as unmeasured rather than 0. +#[must_use] +pub fn panic_totals(parser: &str) -> Option<(usize, usize)> { + let (mut panicked, mut attempted) = (0usize, 0usize); + for dialect in &bundle().dialects { + if let Some(m) = dialect.correctness.iter().find(|m| m.parser == parser) { + panicked += m.panicked; + attempted += m.attempted; + } + } + (attempted > 0).then_some((panicked, attempted)) +} + #[cfg(test)] mod tests { /// The committed snapshot must decompress and deserialize into the shared @@ -55,4 +102,15 @@ mod tests { assert!(!b.dialects.is_empty()); assert!(!b.parsers.is_empty()); } + + /// The committed feature-scan and depth snapshots must parse into the shared + /// schema, failing the build if `featurescan` output and `viz` drift. + #[test] + fn committed_featurescan_and_depth_parse() { + assert!(!super::featurescan().parsers.is_empty()); + assert!(!super::depth_scan().parsers.is_empty()); + // sqlparser-rs is covered by both scans. + assert!(super::parser_features("sqlparser-rs").is_some()); + assert!(super::parser_depth("sqlparser-rs").is_some()); + } } diff --git a/web/src/main.rs b/web/src/main.rs index d90c637..901cdc3 100644 --- a/web/src/main.rs +++ b/web/src/main.rs @@ -48,7 +48,7 @@ fn main() { #[component] fn App() -> Element { - // MAIN_CSS is force-referenced so the linker keeps the asset; the actual + // MAIN_CSS is force-referenced so the linker keeps the asset. The actual // is emitted into the static via with_static_head above. let _ = MAIN_CSS; rsx! { diff --git a/web/src/metadata.rs b/web/src/metadata.rs index c25397f..b248cfd 100644 --- a/web/src/metadata.rs +++ b/web/src/metadata.rs @@ -245,7 +245,7 @@ pub struct ParserMeta { } /// Releases at least this recent count as actively maintained. Roughly twelve -/// months before [`SNAPSHOT`]; `"YYYY-MM"` strings order by date. +/// months before [`SNAPSHOT`]. `"YYYY-MM"` strings order by date. const MAINTAINED_SINCE: &str = "2025-05"; /// Whether the crate's latest release is recent enough to look maintained. @@ -264,16 +264,6 @@ pub const fn pure_rust_description(pure: bool) -> &'static str { } } -/// A full sentence describing the crate's use of `unsafe`. -#[must_use] -pub fn unsafe_description(note: &str) -> String { - if note.is_empty() { - "Memory safe: the crate contains no unsafe code, so the compiler vouches for its memory safety.".to_string() - } else { - format!("Uses unsafe code for {note}, stepping outside the compiler's memory-safety guarantees.") - } -} - /// A full sentence describing how recently the crate was released. #[must_use] pub fn maintenance_description(last_release: &str) -> String {