diff --git a/buildbotapi.py b/buildbotapi.py index c251f537..cf9f5d74 100644 --- a/buildbotapi.py +++ b/buildbotapi.py @@ -1,4 +1,5 @@ import json +import time from dataclasses import dataclass from typing import Any, cast @@ -6,6 +7,11 @@ JSON = dict[str, Any] +# Builders whose most recent build was more than this many days ago +# are considered inactive and ignored when checking for failures +STALE_BUILDER_DAYS = 14 +SECONDS_PER_DAY = 24 * 60 * 60 + @dataclass class Builder: @@ -56,7 +62,10 @@ async def all_builders(self, branch: str | None = None) -> dict[int, Builder]: } return all_builders - async def is_builder_failing_currently(self, builder: Builder) -> bool: + async def is_builder_failing_currently(self, builder: Builder) -> bool | None: + """Return whether the most recent build failed, or None if the + builder is stale (ignored because its most recent build is older + than STALE_BUILDER_DAYS).""" builds_: dict[str, Any] = await self._fetch_json( f"https://buildbot.python.org/all/api/v2/builds?complete__eq=true" f"&&builderid__eq={builder.builderid}&&order=-complete_at" @@ -66,6 +75,10 @@ async def is_builder_failing_currently(self, builder: Builder) -> bool: if not builds: return False (build,) = builds + + age_days = (time.time() - build["complete_at"]) / SECONDS_PER_DAY + if age_days > STALE_BUILDER_DAYS: + return None if build["results"] == 2: return True return False diff --git a/run_release.py b/run_release.py index bae35d0e..5e06a385 100755 --- a/run_release.py +++ b/run_release.py @@ -35,7 +35,7 @@ import release as release_mod import sbom import update_version_next -from buildbotapi import BuildBotAPI, Builder +from buildbotapi import STALE_BUILDER_DAYS, BuildBotAPI, Builder from release import ReleaseShelf, Tag, Task, ask_question API_KEY_REGEXP = re.compile(r"(?P\w+):(?P\w+)") @@ -352,10 +352,10 @@ def check_sigstore_version(version: str) -> None: def check_buildbots(db: ReleaseShelf) -> None: - async def _check() -> set[Builder]: + async def _check() -> tuple[set[Builder], int]: async def _get_builder_status( buildbot_api: BuildBotAPI, the_builder: Builder - ) -> tuple[Builder, bool]: + ) -> tuple[Builder, bool | None]: return the_builder, await buildbot_api.is_builder_failing_currently( the_builder ) @@ -378,11 +378,21 @@ async def _get_builder_status( for the_builder in stable_builders.values() ] ) - return {the_builder for (the_builder, is_failing) in builders if is_failing} + stale_count = sum(is_failing is None for _, is_failing in builders) + return { + the_builder for (the_builder, is_failing) in builders if is_failing + }, stale_count + + failing_builders, stale_count = asyncio.run(_check()) + if stale_count: + print( + f"Ignoring {stale_count} stale builder{'' if stale_count == 1 else 's'}" + f" (no builds in the last {STALE_BUILDER_DAYS} days)" + ) - failing_builders = asyncio.run(_check()) if not failing_builders: return + print() print("The following buildbots are failing:") for builder in failing_builders: diff --git a/tests/test_buildbotapi.py b/tests/test_buildbotapi.py index 0092505e..d8cb5892 100644 --- a/tests/test_buildbotapi.py +++ b/tests/test_buildbotapi.py @@ -108,17 +108,33 @@ async def test_buildbotapi_stable_builders() -> None: assert "stable" in all_builders[3].tags +# The most recent builds in success.json and failure.json +SUCCESS_COMPLETE_AT = 1728312495 +FAILURE_COMPLETE_AT = 1734198808 +DAY = buildbotapi.SECONDS_PER_DAY + + @pytest.mark.asyncio @pytest.mark.parametrize( - ["json_data", "expected"], + ["json_data", "now", "expected"], [ - ("tests/buildbotapi/success.json", False), - ("tests/buildbotapi/failure.json", True), - ("tests/buildbotapi/no-builds.json", False), + # Recent builds: judged on their result + ("tests/buildbotapi/success.json", SUCCESS_COMPLETE_AT + DAY, False), + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + DAY, True), + ("tests/buildbotapi/no-builds.json", FAILURE_COMPLETE_AT + DAY, False), + # Just inside the staleness cutoff: failure still counts + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 13 * DAY, True), + # At the staleness cutoff: failure still counts + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 14 * DAY, True), + # Stale build (last run > 14 days ago): builder ignored + ("tests/buildbotapi/failure.json", FAILURE_COMPLETE_AT + 15 * DAY, None), ], ) -async def test_buildbotapi_is_builder_failing_currently_yes( - json_data: str, expected: bool +async def test_buildbotapi_is_builder_failing_currently( + monkeypatch: pytest.MonkeyPatch, + json_data: str, + now: int, + expected: bool | None, ) -> None: # Arrange mock_session = AsyncMock(aiohttp.ClientSession) @@ -126,6 +142,7 @@ async def test_buildbotapi_is_builder_failing_currently_yes( mock_session.get.return_value.__aenter__.return_value.text.return_value = load( json_data ) + monkeypatch.setattr("buildbotapi.time.time", lambda: now) api = buildbotapi.BuildBotAPI(mock_session) builder = buildbotapi.Builder(builderid=3)