Ignore failing buildbots that last ran over 14 days ago

buildbotapi.py

@@ -1,11 +1,17 @@
 import json
+import time
 from dataclasses import dataclass
 from typing import Any, cast
 
 from aiohttp.client import ClientSession
 
 JSON = dict[str, Any]
 
+# Builders whose most recent build was more than this many days ago
+# are considered inactive and ignored when checking for failures
+STALE_BUILDER_DAYS = 14
+SECONDS_PER_DAY = 24 * 60 * 60
+
 
 @dataclass
 class Builder:
@@ -56,7 +62,10 @@ async def all_builders(self, branch: str | None = None) -> dict[int, Builder]:
         }
         return all_builders
 
-    async def is_builder_failing_currently(self, builder: Builder) -> bool:
+    async def is_builder_failing_currently(self, builder: Builder) -> bool | None:
+        """Return whether the most recent build failed, or None if the
+        builder is stale (ignored because its most recent build is older
+        than STALE_BUILDER_DAYS)."""
         builds_: dict[str, Any] = await self._fetch_json(
             f"https://buildbot.python.org/all/api/v2/builds?complete__eq=true"
             f"&&builderid__eq={builder.builderid}&&order=-complete_at"
@@ -66,6 +75,10 @@ async def is_builder_failing_currently(self, builder: Builder) -> bool:
         if not builds:
             return False
         (build,) = builds
+
+        age_days = (time.time() - build["complete_at"]) / SECONDS_PER_DAY
+        if age_days > STALE_BUILDER_DAYS:
+            return None
         if build["results"] == 2:
             return True
         return False

run_release.py

@@ -35,7 +35,7 @@
 import release as release_mod
 import sbom
 import update_version_next
-from buildbotapi import BuildBotAPI, Builder
+from buildbotapi import STALE_BUILDER_DAYS, BuildBotAPI, Builder
 from release import ReleaseShelf, Tag, Task, ask_question
 
 API_KEY_REGEXP = re.compile(r"(?P<user>\w+):(?P<key>\w+)")
@@ -352,10 +352,10 @@ def check_sigstore_version(version: str) -> None:
 
 
 def check_buildbots(db: ReleaseShelf) -> None:
-    async def _check() -> set[Builder]:
+    async def _check() -> tuple[set[Builder], int]:
         async def _get_builder_status(
             buildbot_api: BuildBotAPI, the_builder: Builder
-        ) -> tuple[Builder, bool]:
+        ) -> tuple[Builder, bool | None]:
             return the_builder, await buildbot_api.is_builder_failing_currently(
                 the_builder
             )
@@ -378,11 +378,21 @@ async def _get_builder_status(
                     for the_builder in stable_builders.values()
                 ]
             )
-            return {the_builder for (the_builder, is_failing) in builders if is_failing}
+            stale_count = sum(is_failing is None for _, is_failing in builders)
+            return {
+                the_builder for (the_builder, is_failing) in builders if is_failing
+            }, stale_count
+
+    failing_builders, stale_count = asyncio.run(_check())
+    if stale_count:
+        print(
+            f"Ignoring {stale_count} stale builder{'' if stale_count == 1 else 's'}"
+            f" (no builds in the last {STALE_BUILDER_DAYS} days)"
+        )
 
-    failing_builders = asyncio.run(_check())
     if not failing_builders:
         return
+
     print()
     print("The following buildbots are failing:")
     for builder in failing_builders:

tests/test_buildbotapi.py

@@ -108,24 +108,41 @@ async def test_buildbotapi_stable_builders() -> None:
     assert "stable" in all_builders[3].tags
 
 
+# The most recent builds in success.json and failure.json
+SUCCESS_COMPLETE_AT = 1728312495
+FAILURE_COMPLETE_AT = 1734198808
+DAY = buildbotapi.SECONDS_PER_DAY
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    ["json_data", "expected"],
+    ["json_data", "now", "expected"],
     [
-        ("tests/buildbotapi/success.json", False),
-        ("tests/buildbotapi/failure.json", True),
-        ("tests/buildbotapi/no-builds.json", False),
+        # Recent builds: judged on their result
+        ("tests/buildbotapi/success.json", SUCCESS_COMPLETE_AT + DAY, False),
+        ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + DAY, True),
+        ("tests/buildbotapi/no-builds.json", FAILURE_COMPLETE_AT + DAY, False),
+        # Just inside the staleness cutoff: failure still counts
+        ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 13 * DAY, True),
+        # At the staleness cutoff: failure still counts
+        ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 14 * DAY, True),
+        # Stale build (last run > 14 days ago): builder ignored
+        ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 15 * DAY, None),
     ],
 )
-async def test_buildbotapi_is_builder_failing_currently_yes(
-    json_data: str, expected: bool
+async def test_buildbotapi_is_builder_failing_currently(
+    monkeypatch: pytest.MonkeyPatch,
+    json_data: str,
+    now: int,
+    expected: bool | None,
 ) -> None:
     # Arrange
     mock_session = AsyncMock(aiohttp.ClientSession)
     mock_session.get.return_value.__aenter__.return_value.status = 200
     mock_session.get.return_value.__aenter__.return_value.text.return_value = load(
         json_data
     )
+    monkeypatch.setattr("buildbotapi.time.time", lambda: now)
     api = buildbotapi.BuildBotAPI(mock_session)
     builder = buildbotapi.Builder(builderid=3)