From cbaaa371f552472917668877ffcdf68ec68058e9 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 1 Sep 2026 12:57:20 +0200 Subject: [PATCH 1/8] New conformance tag '# E[tag!]' to require at least one success --- conformance/README.md | 1 + conformance/src/main.py | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/conformance/README.md b/conformance/README.md index 0b60a9490..59f94ea10 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -63,6 +63,7 @@ Test cases support the following special comments for declaring where errors sho * `# E[tag]`, where `tag` is an arbitrary string: must appear multiple times in a file with the same tag. Exactly one line with this tag must raise an error. * `# E[tag+]`: like `# E[tag]`, but errors may be raised on multiple lines. +* `# E[tag!]`: like `# E[tag]`, but at least one line must not raise an error. Each comment may be followed by a colon plus an explanation of the error; the explanation is ignored by the scoring system. diff --git a/conformance/src/main.py b/conformance/src/main.py index 63f82ea72..be589f1a1 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -6,9 +6,9 @@ import re import sys import tomllib +from collections.abc import Sequence from pathlib import Path from time import time -from typing import Sequence import tomlkit @@ -65,7 +65,7 @@ def run_tests( def get_expected_errors(test_case: Path) -> tuple[ dict[int, tuple[int, int]], - dict[str, tuple[list[int], bool]], + dict[str, tuple[list[int], bool, bool]], ]: """Return the line numbers where type checkers are expected to produce an error. @@ -92,7 +92,7 @@ def f(): pass # E[final] with open(test_case, "r", encoding="utf-8") as f: lines = f.readlines() output: dict[int, tuple[int, int]] = {} - groups: dict[str, tuple[list[int], bool]] = {} + groups: dict[str, tuple[list[int], bool, bool]] = {} for i, line in enumerate(lines, start=1): line_without_comment, *_ = line.split("#") # Ignore lines with no non-comment content. This allows commenting out test cases. @@ -111,14 +111,22 @@ def f(): pass # E[final] tag = match.group(1) if tag.endswith("+"): allow_multiple = True + require_success = False + tag = tag[:-1] + elif tag.endswith("!"): + allow_multiple = True + require_success = True tag = tag[:-1] else: allow_multiple = False + require_success = False if tag not in groups: - groups[tag] = ([i], allow_multiple) + groups[tag] = ([i], allow_multiple, require_success) else: if groups[tag][1] != allow_multiple: raise ValueError(f"Error group {tag} has inconsistent allow_multiple value in {test_case}") + if groups[tag][2] != require_success: + raise ValueError(f"Error group {tag} has inconsistent require_success value in {test_case}") groups[tag][0].append(i) for group, linenos in groups.items(): if len(linenos) == 1: @@ -152,11 +160,13 @@ def diff_expected_errors( # We don't report an issue if the count differs, because type checkers may produce # multiple error messages for a single line. linenos_used_by_groups: set[int] = set() - for group, (linenos, allow_multiple) in error_groups.items(): + for group, (linenos, allow_multiple, require_success) in error_groups.items(): num_errors = sum(1 for lineno in linenos if lineno in errors) - if num_errors == 0: + if require_success and num_errors == len(linenos): + differences.append(f"Lines {', '.join(map(str, linenos))}: Expected at least one success (tag {group!r})") + elif num_errors == 0 and not require_success: differences.append(f"Lines {', '.join(map(str, linenos))}: Expected error (tag {group!r})") - elif num_errors == 1 or allow_multiple: + elif num_errors == 1 or allow_multiple or require_success: linenos_used_by_groups.update(linenos) else: differences.append(f"Lines {', '.join(map(str, linenos))}: Expected exactly one error (tag {group!r})") From ca6185f3eab57fe7003696a46740dfe85bcddae8 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 2 Sep 2026 13:26:27 +0200 Subject: [PATCH 2/8] Extend description per review --- conformance/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conformance/README.md b/conformance/README.md index 59f94ea10..7f28150dd 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -63,7 +63,8 @@ Test cases support the following special comments for declaring where errors sho * `# E[tag]`, where `tag` is an arbitrary string: must appear multiple times in a file with the same tag. Exactly one line with this tag must raise an error. * `# E[tag+]`: like `# E[tag]`, but errors may be raised on multiple lines. -* `# E[tag!]`: like `# E[tag]`, but at least one line must not raise an error. +* `# E[tag!]`: like `# E[tag]`, but zero or more lines with this tag may raise + an error while at least line must not raise an error. Each comment may be followed by a colon plus an explanation of the error; the explanation is ignored by the scoring system. From 8e71a2e09a04aea19dff167a64efaed1f7470e82 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 2 Sep 2026 13:29:22 +0200 Subject: [PATCH 3/8] Update docstring --- conformance/src/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conformance/src/main.py b/conformance/src/main.py index be589f1a1..d660c7543 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -71,9 +71,9 @@ def get_expected_errors(test_case: Path) -> tuple[ The return value is a tuple of two dictionaries: - The format of the first is {line number: (number of required errors, number of optional errors)}. - - The format of the second is {error tag: ([lines where the error may appear], allow multiple}. - If allow multiple is True, the error may appear on multiple lines; otherwise, it must - appear exactly once. + - The format of the second is {error tag: ([lines where the error may appear], allow multiple, require success)}. + If require success is True, at least one line can't raise an error; otherwise, if allow multiple is True, the + error may appear on multiple lines; otherwise, it must appear exactly once. For example, the following test case: From 64624cbd8cefcffc8628f55d3c812ffb6a89441d Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 2 Sep 2026 19:38:56 +0200 Subject: [PATCH 4/8] Refactor grouped error detection - Introduce an `ErrorMultiplicity` enum and return it from `get_expected_errors()`. - Extract `determine_group_error()` from `diff_expected_errors()` and simplify logic. - Rerun black on `main.py`. --- conformance/src/main.py | 106 +++++++++++++++++++++++++++------------- 1 file changed, 73 insertions(+), 33 deletions(-) diff --git a/conformance/src/main.py b/conformance/src/main.py index d660c7543..5564e3293 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -6,7 +6,8 @@ import re import sys import tomllib -from collections.abc import Sequence +from collections.abc import Container, Sequence +from enum import Enum from pathlib import Path from time import time @@ -63,17 +64,24 @@ def run_tests( update_type_checker_info(type_checker, root_dir) +class ErrorMultiplicity(Enum): + """How many lines with the same tag can have an error.""" + + SINGLE = 1 # exactly one line must have an error + MULTI = 2 # at least one line must have an error + REQUIRE_SUCCESS = 3 # at least one line must not have an error + + def get_expected_errors(test_case: Path) -> tuple[ dict[int, tuple[int, int]], - dict[str, tuple[list[int], bool, bool]], + dict[str, tuple[list[int], ErrorMultiplicity]], ]: """Return the line numbers where type checkers are expected to produce an error. The return value is a tuple of two dictionaries: - The format of the first is {line number: (number of required errors, number of optional errors)}. - - The format of the second is {error tag: ([lines where the error may appear], allow multiple, require success)}. - If require success is True, at least one line can't raise an error; otherwise, if allow multiple is True, the - error may appear on multiple lines; otherwise, it must appear exactly once. + - The format of the second is {error tag: ([lines where the error may appear], error multiplicity)}. + See the ErrorMultiplicy enum for the second argument. For example, the following test case: @@ -92,7 +100,7 @@ def f(): pass # E[final] with open(test_case, "r", encoding="utf-8") as f: lines = f.readlines() output: dict[int, tuple[int, int]] = {} - groups: dict[str, tuple[list[int], bool, bool]] = {} + groups: dict[str, tuple[list[int], ErrorMultiplicity]] = {} for i, line in enumerate(lines, start=1): line_without_comment, *_ = line.split("#") # Ignore lines with no non-comment content. This allows commenting out test cases. @@ -110,27 +118,26 @@ def f(): pass # E[final] for match in re.finditer(r"# E\[([^\]]+)\]", line): tag = match.group(1) if tag.endswith("+"): - allow_multiple = True - require_success = False + multiplicity: ErrorMultiplicity = ErrorMultiplicity.MULTI tag = tag[:-1] elif tag.endswith("!"): - allow_multiple = True - require_success = True + multiplicity = ErrorMultiplicity.REQUIRE_SUCCESS tag = tag[:-1] else: - allow_multiple = False - require_success = False + multiplicity = ErrorMultiplicity.SINGLE if tag not in groups: - groups[tag] = ([i], allow_multiple, require_success) + groups[tag] = ([i], multiplicity) else: - if groups[tag][1] != allow_multiple: - raise ValueError(f"Error group {tag} has inconsistent allow_multiple value in {test_case}") - if groups[tag][2] != require_success: - raise ValueError(f"Error group {tag} has inconsistent require_success value in {test_case}") + if groups[tag][1] != multiplicity: + raise ValueError( + f"Error group {tag} has inconsistent multiplicity value in {test_case}" + ) groups[tag][0].append(i) for group, linenos in groups.items(): if len(linenos) == 1: - raise ValueError(f"Error group {group} only appears on a single line in {test_case}") + raise ValueError( + f"Error group {group} only appears on a single line in {test_case}" + ) return output, groups @@ -148,34 +155,63 @@ def diff_expected_errors( lineno: [ error for error in errors_list - if not any(ignored in error for ignored in ignored_errors)] + if not any(ignored in error for ignored in ignored_errors) + ] for lineno, errors_list in errors.items() } - errors = {lineno: errors_list for lineno, errors_list in errors.items() if errors_list} + errors = { + lineno: errors_list for lineno, errors_list in errors.items() if errors_list + } differences: list[str] = [] for expected_lineno, (expected_count, _) in expected_errors.items(): if expected_lineno not in errors and expected_count > 0: - differences.append(f"Line {expected_lineno}: Expected {expected_count} errors") + differences.append( + f"Line {expected_lineno}: Expected {expected_count} errors" + ) # We don't report an issue if the count differs, because type checkers may produce # multiple error messages for a single line. linenos_used_by_groups: set[int] = set() - for group, (linenos, allow_multiple, require_success) in error_groups.items(): - num_errors = sum(1 for lineno in linenos if lineno in errors) - if require_success and num_errors == len(linenos): - differences.append(f"Lines {', '.join(map(str, linenos))}: Expected at least one success (tag {group!r})") - elif num_errors == 0 and not require_success: - differences.append(f"Lines {', '.join(map(str, linenos))}: Expected error (tag {group!r})") - elif num_errors == 1 or allow_multiple or require_success: + for group, (linenos, multiplicity) in error_groups.items(): + error = determine_group_error(group, linenos, multiplicity, errors) + if error is None: linenos_used_by_groups.update(linenos) else: - differences.append(f"Lines {', '.join(map(str, linenos))}: Expected exactly one error (tag {group!r})") + differences.append(error) for actual_lineno, actual_errors in errors.items(): - if actual_lineno not in expected_errors and actual_lineno not in linenos_used_by_groups: - differences.append(f"Line {actual_lineno}: Unexpected errors {actual_errors}") + if ( + actual_lineno not in expected_errors + and actual_lineno not in linenos_used_by_groups + ): + differences.append( + f"Line {actual_lineno}: Unexpected errors {actual_errors}" + ) return "".join(f"{diff}\n" for diff in differences) +def determine_group_error( + group: str, + group_linenos: Sequence[int], + multiplicity: ErrorMultiplicity, + error_linenos: Container[int], +) -> str | None: + """Return the error message for the given group or None.""" + num_errors = sum(1 for lineno in group_linenos if lineno in error_linenos) + match multiplicity: + case ErrorMultiplicity.SINGLE: + if num_errors == 0: + return f"Lines {', '.join(map(str, group_linenos))}: Expected error (tag {group!r})" + elif num_errors > 1: + return f"Lines {', '.join(map(str, group_linenos))}: Expected exactly one error (tag {group!r})" + case ErrorMultiplicity.MULTI: + if num_errors == 0: + return f"Lines {', '.join(map(str, group_linenos))}: Expected error (tag {group!r})" + case ErrorMultiplicity.REQUIRE_SUCCESS: + if num_errors == len(group_linenos): + return f"Lines {', '.join(map(str, group_linenos))}: Expected at least one success (tag {group!r})" + return None + + def update_output_for_test( type_checker: TypeChecker, results_dir: Path, @@ -201,7 +237,9 @@ def update_output_for_test( existing_results = {} ignored_errors = existing_results.get("ignore_errors", []) - errors_diff = "\n" + diff_expected_errors(type_checker, test_case, output, ignored_errors) + errors_diff = "\n" + diff_expected_errors( + type_checker, test_case, output, ignored_errors + ) old_errors_diff = "\n" + existing_results.get("errors_diff", "") if errors_diff != old_errors_diff: @@ -294,7 +332,9 @@ def main(): if not type_checker.install(): print(f"Skipping tests for {type_checker.name}") else: - run_tests(root_dir, type_checker, test_cases, verbose=options.verbose) + run_tests( + root_dir, type_checker, test_cases, verbose=options.verbose + ) # Generate a summary report. generate_summary(root_dir) From 7f7f565a55392b2225c1f4cba3c1d78225ed2204 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 2 Sep 2026 19:46:28 +0200 Subject: [PATCH 5/8] Revert unrelated formatting changes --- conformance/src/main.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/conformance/src/main.py b/conformance/src/main.py index 5564e3293..94f8c9958 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -129,15 +129,11 @@ def f(): pass # E[final] groups[tag] = ([i], multiplicity) else: if groups[tag][1] != multiplicity: - raise ValueError( - f"Error group {tag} has inconsistent multiplicity value in {test_case}" - ) + raise ValueError(f"Error group {tag} has inconsistent multiplicity value in {test_case}") groups[tag][0].append(i) for group, linenos in groups.items(): if len(linenos) == 1: - raise ValueError( - f"Error group {group} only appears on a single line in {test_case}" - ) + raise ValueError(f"Error group {group} only appears on a single line in {test_case}") return output, groups @@ -155,13 +151,10 @@ def diff_expected_errors( lineno: [ error for error in errors_list - if not any(ignored in error for ignored in ignored_errors) - ] + if not any(ignored in error for ignored in ignored_errors)] for lineno, errors_list in errors.items() } - errors = { - lineno: errors_list for lineno, errors_list in errors.items() if errors_list - } + errors = {lineno: errors_list for lineno, errors_list in errors.items() if errors_list} differences: list[str] = [] for expected_lineno, (expected_count, _) in expected_errors.items(): @@ -237,9 +230,7 @@ def update_output_for_test( existing_results = {} ignored_errors = existing_results.get("ignore_errors", []) - errors_diff = "\n" + diff_expected_errors( - type_checker, test_case, output, ignored_errors - ) + errors_diff = "\n" + diff_expected_errors(type_checker, test_case, output, ignored_errors) old_errors_diff = "\n" + existing_results.get("errors_diff", "") if errors_diff != old_errors_diff: @@ -332,9 +323,7 @@ def main(): if not type_checker.install(): print(f"Skipping tests for {type_checker.name}") else: - run_tests( - root_dir, type_checker, test_cases, verbose=options.verbose - ) + run_tests(root_dir, type_checker, test_cases, verbose=options.verbose) # Generate a summary report. generate_summary(root_dir) From 35a8b1011d0933c39fae6327feca0ede6342ee9f Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 2 Sep 2026 19:47:33 +0200 Subject: [PATCH 6/8] Remove unnecessary type annotation --- conformance/src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/src/main.py b/conformance/src/main.py index 94f8c9958..9d932811d 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -118,7 +118,7 @@ def f(): pass # E[final] for match in re.finditer(r"# E\[([^\]]+)\]", line): tag = match.group(1) if tag.endswith("+"): - multiplicity: ErrorMultiplicity = ErrorMultiplicity.MULTI + multiplicity = ErrorMultiplicity.MULTI tag = tag[:-1] elif tag.endswith("!"): multiplicity = ErrorMultiplicity.REQUIRE_SUCCESS From a34db42f359cef09d5c03e6161017924dd8f164f Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 2 Sep 2026 19:48:39 +0200 Subject: [PATCH 7/8] Revert one more formatting change --- conformance/src/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conformance/src/main.py b/conformance/src/main.py index 9d932811d..8c3460f0d 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -159,9 +159,7 @@ def diff_expected_errors( differences: list[str] = [] for expected_lineno, (expected_count, _) in expected_errors.items(): if expected_lineno not in errors and expected_count > 0: - differences.append( - f"Line {expected_lineno}: Expected {expected_count} errors" - ) + differences.append(f"Line {expected_lineno}: Expected {expected_count} errors") # We don't report an issue if the count differs, because type checkers may produce # multiple error messages for a single line. linenos_used_by_groups: set[int] = set() From bede490ab1b31d5908f28149c31dd00877f2a950 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 2 Sep 2026 20:52:09 +0200 Subject: [PATCH 8/8] Harden `linenos_used_by_groups` --- conformance/src/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conformance/src/main.py b/conformance/src/main.py index 8c3460f0d..0073dcb07 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -162,13 +162,13 @@ def diff_expected_errors( differences.append(f"Line {expected_lineno}: Expected {expected_count} errors") # We don't report an issue if the count differs, because type checkers may produce # multiple error messages for a single line. - linenos_used_by_groups: set[int] = set() + for group, (linenos, multiplicity) in error_groups.items(): error = determine_group_error(group, linenos, multiplicity, errors) - if error is None: - linenos_used_by_groups.update(linenos) - else: + if error is not None: differences.append(error) + + linenos_used_by_groups = {ln for linenos, _ in error_groups.values() for ln in linenos} for actual_lineno, actual_errors in errors.items(): if ( actual_lineno not in expected_errors