From 8582325b9051c57541e997c63d26dfaf3c51985b Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 23 Jun 2026 11:24:00 +0200 Subject: [PATCH 1/5] Regression tests: Handle no packages gracefully Before, when running regression tests for a certain version/platform combination that doesn't have any cases, mypy is run with invalid arguments (no files), and the tests fail. Now, a message is printed instead (unless the verbosity is `QUIET`) and the tests succeed. --- lib/ts_utils/utils.py | 4 ++++ tests/regr_test.py | 53 +++++++++++++++++++++++++++++++------------ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/lib/ts_utils/utils.py b/lib/ts_utils/utils.py index deb9e222843d..6c8cd022aa70 100644 --- a/lib/ts_utils/utils.py +++ b/lib/ts_utils/utils.py @@ -83,6 +83,10 @@ def print_warning(message: str) -> None: print(colored(message, "yellow")) +def print_skipped(message: str) -> None: + print(colored(message, "yellow")) + + def print_error(error: str, end: str = "\n", fix_path: tuple[str, str] = ("", "")) -> None: error_split = error.split("\n") old, new = fix_path diff --git a/tests/regr_test.py b/tests/regr_test.py index 3d310415061b..8e6e1d7f8c3d 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -33,6 +33,7 @@ get_all_testcase_directories, get_mypy_req, print_error, + print_skipped, venv_python, ) @@ -167,7 +168,7 @@ def setup_testcase_dir(package: DistributionTests, tempdir: Path, verbosity: Ver def run_testcases( package: DistributionTests, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity -) -> subprocess.CompletedProcess[str]: +) -> subprocess.CompletedProcess[str] | None: env_vars = dict(os.environ) new_test_case_dir = tempdir / TEST_CASES_DIR @@ -177,7 +178,6 @@ def run_testcases( configurations = mypy_configuration_from_distribution(package.name) with temporary_mypy_config_file(configurations) as temp_config: - # "--enable-error-code ignore-without-code" is purposefully omitted. # See https://github.com/python/typeshed/pull/8083 flags = [ @@ -216,18 +216,22 @@ def run_testcases( flags.extend(["--custom-typeshed-dir", str(custom_typeshed)]) - # If the test-case filename ends with -py39, - # only run the test if --python-version was set to 3.9 or higher (for example) + # If the test-case filename ends with e.g. -py314, + # only run the test if --python-version was set to 3.14 or higher (for example) + files: list[str] = [] for path in new_test_case_dir.rglob("*.py"): - if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem): + if match := re.fullmatch(r".*-py3(\d\d)", path.stem): minor_version_required = int(match[1]) assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS python_minor_version = int(version.split(".")[1]) if minor_version_required > python_minor_version: continue - flags.append(str(path)) + files.append(str(path)) + + if len(files) == 0: + return None - mypy_command = [python_exe, "-m", "mypy", *flags] + mypy_command = [python_exe, "-m", "mypy", *flags, *files] if verbosity is Verbosity.VERBOSE: description = f"{package.name}/{version}/{platform}" msg = f"{description}: {mypy_command=}\n" @@ -243,13 +247,20 @@ def run_testcases( @dataclass(frozen=True) class Result: code: int + + def print_description(self, verbosity: Verbosity) -> None: + raise NotImplementedError + + +@dataclass(frozen=True) +class RunResult(Result): command_run: str stderr: str stdout: str test_case_dir: Path tempdir: Path - def print_description(self) -> None: + def print_description(self, _: Verbosity) -> None: if self.code: print(f"{self.command_run}:", end=" ") print_error("FAILURE\n") @@ -260,6 +271,17 @@ def print_description(self) -> None: print_error(self.stdout, fix_path=replacements) +@dataclass(frozen=True) +class NoTestsResult(Result): + package: str + version: str + platform: str + + def print_description(self, verbosity: Verbosity) -> None: + if verbosity != Verbosity.QUIET: + print_skipped(f"No test cases found for {self.package!r} on Python {self.version} for platform {self.platform!r}.") + + def test_testcase_directory( package: DistributionTests, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path ) -> Result: @@ -268,12 +290,15 @@ def test_testcase_directory( if verbosity > Verbosity.QUIET: _PRINT_QUEUE.put(f"Running {msg}...") - proc_info = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity) - return Result( - code=proc_info.returncode, + result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity) + if result is None: + return NoTestsResult(0, package.name, version, platform) + + return RunResult( + code=result.returncode, command_run=msg, - stderr=proc_info.stderr, - stdout=proc_info.stdout, + stderr=result.stderr, + stdout=result.stdout, test_case_dir=package.test_cases_path, tempdir=tempdir, ) @@ -399,7 +424,7 @@ def main() -> ReturnCode: print() for result in results: - result.print_description() + result.print_description(verbosity) code = max(result.code for result in results) From 3b11691ac2c9924f12d3114fb7635eb9e05f016d Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 23 Jun 2026 11:26:09 +0200 Subject: [PATCH 2/5] Fix type error --- tests/regr_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 8e6e1d7f8c3d..fcd11b5299ed 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -260,7 +260,7 @@ class RunResult(Result): test_case_dir: Path tempdir: Path - def print_description(self, _: Verbosity) -> None: + def print_description(self, verbosity: Verbosity) -> None: # noqa: ARG002 if self.code: print(f"{self.command_run}:", end=" ") print_error("FAILURE\n") From db48bc917f362e4d38cc980f0f443ad3c9ce3d19 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 23 Jun 2026 11:27:41 +0200 Subject: [PATCH 3/5] Revert unnecessary change --- tests/regr_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index fcd11b5299ed..376a29a2b6c7 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -290,15 +290,15 @@ def test_testcase_directory( if verbosity > Verbosity.QUIET: _PRINT_QUEUE.put(f"Running {msg}...") - result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity) - if result is None: + proc_info = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity) + if proc_info is None: return NoTestsResult(0, package.name, version, platform) return RunResult( - code=result.returncode, + code=proc_info.returncode, command_run=msg, - stderr=result.stderr, - stdout=result.stdout, + stderr=proc_info.stderr, + stdout=proc_info.stdout, test_case_dir=package.test_cases_path, tempdir=tempdir, ) From d02bf155fcd6e91d0d5f72a5e07b31f724dac048 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 23 Jun 2026 13:38:13 +0200 Subject: [PATCH 4/5] Make `Result` abstract --- tests/regr_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 376a29a2b6c7..44a12351c015 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -13,6 +13,7 @@ import sys import tempfile import threading +from abc import ABCMeta, abstractmethod from collections.abc import Callable, Generator from contextlib import ExitStack, contextmanager, suppress from dataclasses import dataclass @@ -245,9 +246,10 @@ def run_testcases( @dataclass(frozen=True) -class Result: +class Result(metaclass=ABCMeta): code: int + @abstractmethod def print_description(self, verbosity: Verbosity) -> None: raise NotImplementedError From 442d77947737e5d26ffa120f096d06d09ea87c15 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 23 Jun 2026 13:40:08 +0200 Subject: [PATCH 5/5] Add `@override` --- tests/regr_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 44a12351c015..77b5d0cef0f9 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -21,6 +21,7 @@ from functools import partial from pathlib import Path from typing import TypeAlias +from typing_extensions import override from ts_utils.metadata import get_recursive_requirements, read_metadata from ts_utils.mypy import mypy_configuration_from_distribution, temporary_mypy_config_file @@ -262,7 +263,8 @@ class RunResult(Result): test_case_dir: Path tempdir: Path - def print_description(self, verbosity: Verbosity) -> None: # noqa: ARG002 + @override + def print_description(self, verbosity: Verbosity) -> None: if self.code: print(f"{self.command_run}:", end=" ") print_error("FAILURE\n") @@ -279,6 +281,7 @@ class NoTestsResult(Result): version: str platform: str + @override def print_description(self, verbosity: Verbosity) -> None: if verbosity != Verbosity.QUIET: print_skipped(f"No test cases found for {self.package!r} on Python {self.version} for platform {self.platform!r}.")