From f0bf6ccaf67d7bb9b1c45deeb655ad4dd69ff66b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 23 Jun 2026 12:13:41 +0300 Subject: [PATCH 1/4] Ignore failing buildbots that last ran over 14 days ago --- buildbotapi.py | 10 ++++++++++ tests/test_buildbotapi.py | 24 ++++++++++++++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/buildbotapi.py b/buildbotapi.py index c251f537..8529650c 100644 --- a/buildbotapi.py +++ b/buildbotapi.py @@ -1,4 +1,5 @@ import json +import time from dataclasses import dataclass from typing import Any, cast @@ -6,6 +7,11 @@ JSON = dict[str, Any] +# Builders whose most recent build was more than this many days ago +# are considered inactive and ignored when checking for failures +STALE_BUILDER_DAYS = 14 +SECONDS_PER_DAY = 24 * 60 * 60 + @dataclass class Builder: @@ -66,6 +72,10 @@ async def is_builder_failing_currently(self, builder: Builder) -> bool: if not builds: return False (build,) = builds + + age_days = (time.time() - build["complete_at"]) / SECONDS_PER_DAY + if age_days > STALE_BUILDER_DAYS: + return False if build["results"] == 2: return True return False diff --git a/tests/test_buildbotapi.py b/tests/test_buildbotapi.py index 0092505e..f45e2b56 100644 --- a/tests/test_buildbotapi.py +++ b/tests/test_buildbotapi.py @@ -108,17 +108,28 @@ async def test_buildbotapi_stable_builders() -> None: assert "stable" in all_builders[3].tags +# The most recent builds in success.json and failure.json +SUCCESS_COMPLETE_AT = 1728312495 +FAILURE_COMPLETE_AT = 1734198808 +DAY = buildbotapi.SECONDS_PER_DAY + + @pytest.mark.asyncio @pytest.mark.parametrize( - ["json_data", "expected"], + ["json_data", "now", "expected"], [ - ("tests/buildbotapi/success.json", False), - ("tests/buildbotapi/failure.json", True), - ("tests/buildbotapi/no-builds.json", False), + # Recent builds: judged on their result + ("tests/buildbotapi/success.json", SUCCESS_COMPLETE_AT + DAY, False), + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + DAY, True), + ("tests/buildbotapi/no-builds.json", FAILURE_COMPLETE_AT + DAY, False), + # Just inside the staleness cutoff: failure still counts + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 13 * DAY, True), + # Stale build (last run > 14 days ago): builder ignored + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 15 * DAY, False), ], ) -async def test_buildbotapi_is_builder_failing_currently_yes( - json_data: str, expected: bool +async def test_buildbotapi_is_builder_failing_currently( + monkeypatch: pytest.MonkeyPatch, json_data: str, now: int, expected: bool ) -> None: # Arrange mock_session = AsyncMock(aiohttp.ClientSession) @@ -126,6 +137,7 @@ async def test_buildbotapi_is_builder_failing_currently_yes( mock_session.get.return_value.__aenter__.return_value.text.return_value = load( json_data ) + monkeypatch.setattr("buildbotapi.time.time", lambda: now) api = buildbotapi.BuildBotAPI(mock_session) builder = buildbotapi.Builder(builderid=3) From 013e79d74797b6349c2a9bb5b343a4384f85d878 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:23:49 +0300 Subject: [PATCH 2/4] Remove whitespace Co-authored-by: Savannah Ostrowski --- buildbotapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbotapi.py b/buildbotapi.py index 8529650c..d05fce2c 100644 --- a/buildbotapi.py +++ b/buildbotapi.py @@ -8,7 +8,7 @@ JSON = dict[str, Any] # Builders whose most recent build was more than this many days ago -# are considered inactive and ignored when checking for failures +# are considered inactive and ignored when checking for failures STALE_BUILDER_DAYS = 14 SECONDS_PER_DAY = 24 * 60 * 60 From 48dcbc21c399095d23564e21b476844d723154bf Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:39:59 +0300 Subject: [PATCH 3/4] Test at the staleness cutoff: failure still counts --- tests/test_buildbotapi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_buildbotapi.py b/tests/test_buildbotapi.py index f45e2b56..a1b01123 100644 --- a/tests/test_buildbotapi.py +++ b/tests/test_buildbotapi.py @@ -124,6 +124,8 @@ async def test_buildbotapi_stable_builders() -> None: ("tests/buildbotapi/no-builds.json", FAILURE_COMPLETE_AT + DAY, False), # Just inside the staleness cutoff: failure still counts ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 13 * DAY, True), + # At the staleness cutoff: failure still counts + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 14 * DAY, True), # Stale build (last run > 14 days ago): builder ignored ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 15 * DAY, False), ], From e63c48dec7858e3b59897710a6e6ddcf26115cc2 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:23:37 +0300 Subject: [PATCH 4/4] Print count of ignored stale builders --- buildbotapi.py | 7 +++++-- run_release.py | 20 +++++++++++++++----- tests/test_buildbotapi.py | 7 +++++-- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/buildbotapi.py b/buildbotapi.py index d05fce2c..cf9f5d74 100644 --- a/buildbotapi.py +++ b/buildbotapi.py @@ -62,7 +62,10 @@ async def all_builders(self, branch: str | None = None) -> dict[int, Builder]: } return all_builders - async def is_builder_failing_currently(self, builder: Builder) -> bool: + async def is_builder_failing_currently(self, builder: Builder) -> bool | None: + """Return whether the most recent build failed, or None if the + builder is stale (ignored because its most recent build is older + than STALE_BUILDER_DAYS).""" builds_: dict[str, Any] = await self._fetch_json( f"https://buildbot.python.org/all/api/v2/builds?complete__eq=true" f"&&builderid__eq={builder.builderid}&&order=-complete_at" @@ -75,7 +78,7 @@ async def is_builder_failing_currently(self, builder: Builder) -> bool: age_days = (time.time() - build["complete_at"]) / SECONDS_PER_DAY if age_days > STALE_BUILDER_DAYS: - return False + return None if build["results"] == 2: return True return False diff --git a/run_release.py b/run_release.py index bae35d0e..5e06a385 100755 --- a/run_release.py +++ b/run_release.py @@ -35,7 +35,7 @@ import release as release_mod import sbom import update_version_next -from buildbotapi import BuildBotAPI, Builder +from buildbotapi import STALE_BUILDER_DAYS, BuildBotAPI, Builder from release import ReleaseShelf, Tag, Task, ask_question API_KEY_REGEXP = re.compile(r"(?P\w+):(?P\w+)") @@ -352,10 +352,10 @@ def check_sigstore_version(version: str) -> None: def check_buildbots(db: ReleaseShelf) -> None: - async def _check() -> set[Builder]: + async def _check() -> tuple[set[Builder], int]: async def _get_builder_status( buildbot_api: BuildBotAPI, the_builder: Builder - ) -> tuple[Builder, bool]: + ) -> tuple[Builder, bool | None]: return the_builder, await buildbot_api.is_builder_failing_currently( the_builder ) @@ -378,11 +378,21 @@ async def _get_builder_status( for the_builder in stable_builders.values() ] ) - return {the_builder for (the_builder, is_failing) in builders if is_failing} + stale_count = sum(is_failing is None for _, is_failing in builders) + return { + the_builder for (the_builder, is_failing) in builders if is_failing + }, stale_count + + failing_builders, stale_count = asyncio.run(_check()) + if stale_count: + print( + f"Ignoring {stale_count} stale builder{'' if stale_count == 1 else 's'}" + f" (no builds in the last {STALE_BUILDER_DAYS} days)" + ) - failing_builders = asyncio.run(_check()) if not failing_builders: return + print() print("The following buildbots are failing:") for builder in failing_builders: diff --git a/tests/test_buildbotapi.py b/tests/test_buildbotapi.py index a1b01123..d8cb5892 100644 --- a/tests/test_buildbotapi.py +++ b/tests/test_buildbotapi.py @@ -127,11 +127,14 @@ async def test_buildbotapi_stable_builders() -> None: # At the staleness cutoff: failure still counts ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 14 * DAY, True), # Stale build (last run > 14 days ago): builder ignored - ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 15 * DAY, False), + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 15 * DAY, None), ], ) async def test_buildbotapi_is_builder_failing_currently( - monkeypatch: pytest.MonkeyPatch, json_data: str, now: int, expected: bool + monkeypatch: pytest.MonkeyPatch, + json_data: str, + now: int, + expected: bool | None, ) -> None: # Arrange mock_session = AsyncMock(aiohttp.ClientSession)