Skip to content

Observe ephemeral runner completion via GitHub API #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
intel352 merged 2 commits into from
Jun 30, 2026
Merged

Observe ephemeral runner completion via GitHub API #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
07e9655
fix: observe runner jobs via github api
intel352 Jun 30, 2026
96d2e1f
fix: validate workflow run filters
intel352 Jun 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 152 additions & 1 deletion cmd/github-actions-runner-job/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ const (
defaultRunnerJobTimeout = 30 * time.Minute
)

var githubJobPollInterval = 5 * time.Second

func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "github-actions-runner-job failed: %v\n", err)
Expand Down Expand Up @@ -190,6 +192,43 @@ func (c *providerSidecarClient) dispatchWorkflow(ctx context.Context, repository
return c.do(ctx, http.MethodPost, path, body, http.StatusNoContent, nil)
}

func (c *providerSidecarClient) workflowRuns(ctx context.Context, repository, workflow string, createdAfter time.Time) ([]internal.GitHubWorkflowRun, error) {
owner, repo, err := splitRepository(repository)
if err != nil {
return nil, err
}
path := "/v1/actions/repos/" + url.PathEscape(owner) + "/" + url.PathEscape(repo) + "/workflows/" + url.PathEscape(workflow) + "/runs"
query := url.Values{}
if !createdAfter.IsZero() {
query.Set("created_after", createdAfter.UTC().Format(time.RFC3339))
}
if encoded := query.Encode(); encoded != "" {
path += "?" + encoded
}
var out struct {
WorkflowRuns []internal.GitHubWorkflowRun `json:"workflow_runs"`
}
if err := c.do(ctx, http.MethodGet, path, nil, http.StatusOK, &out); err != nil {
return nil, err
}
return out.WorkflowRuns, nil
}

func (c *providerSidecarClient) workflowRunJobs(ctx context.Context, repository string, runID int64) ([]internal.GitHubWorkflowJob, error) {
owner, repo, err := splitRepository(repository)
if err != nil {
return nil, err
}
path := fmt.Sprintf("/v1/actions/repos/%s/%s/actions/runs/%d/jobs", url.PathEscape(owner), url.PathEscape(repo), runID)
var out struct {
Jobs []internal.GitHubWorkflowJob `json:"jobs"`
}
if err := c.do(ctx, http.MethodGet, path, nil, http.StatusOK, &out); err != nil {
return nil, err
}
return out.Jobs, nil
}

func (c *providerSidecarClient) removeOrgRunner(ctx context.Context, organization string, runnerID int64) error {
path := fmt.Sprintf("/v1/actions/orgs/%s/runners/%d", url.PathEscape(organization), runnerID)
return c.do(ctx, http.MethodDelete, path, nil, http.StatusNoContent, nil)
Expand Down Expand Up @@ -280,12 +319,20 @@ func (d *runnerDriver) RunGitHubJob(ctx context.Context, mode internal.Ephemeral
if err != nil {
return result, err
}
dispatchedAfter := time.Now().UTC().Add(-10 * time.Second)
if err := d.sidecar.dispatchWorkflow(ctx, d.req.Repository, d.req.Workflow, dispatchRef, dispatchInputs); err != nil {
runner.cancel()
_ = runner.wait()
return result, err
}
if err := runner.waitForGitHubJob(ctx); err != nil {
completion, err := d.waitForGitHubCompletion(ctx, runner, spec.RunnerName, dispatchedAfter)
if completion.WorkflowRunID != 0 {
result.WorkflowRunID = completion.WorkflowRunID
}
if completion.WorkflowJobID != 0 {
result.WorkflowJobID = completion.WorkflowJobID
}
if err != nil {
return result, err
}
return result, nil
Expand Down Expand Up @@ -317,6 +364,110 @@ func (d *runnerDriver) workflowDispatchInputs(spec internal.EphemeralRunnerJobSp
return inputs, nil
}

type githubJobCompletion struct {
WorkflowRunID int64
WorkflowJobID int64
Success bool
Terminal bool
Message string
}

func (d *runnerDriver) waitForGitHubCompletion(ctx context.Context, runner *runningCommand, runnerName string, dispatchedAfter time.Time) (githubJobCompletion, error) {
ticker := time.NewTicker(githubJobPollInterval)
defer ticker.Stop()
var last githubJobCompletion
for {
select {
case err := <-runner.done:
runner.cancel()
if err != nil {
return last, fmt.Errorf("%s failed: %w: %s", filepath.Base(runner.path), err, runner.output())
}
if completion, err := d.observeGitHubJob(ctx, runnerName, dispatchedAfter); err == nil && completion.WorkflowRunID != 0 {
if completion.Terminal && !completion.Success {
return completion, fmt.Errorf("github workflow job failed: %s", completion.Message)
}
return completion, nil
}
return last, nil
case result := <-runner.result:
runner.cancel()
err := <-runner.done
if result.success {
if completion, observeErr := d.observeGitHubJob(ctx, runnerName, dispatchedAfter); observeErr == nil && completion.WorkflowRunID != 0 {
if completion.Terminal && !completion.Success {
return completion, fmt.Errorf("github workflow job failed after success marker: %s", completion.Message)
}
return completion, nil
}
return last, nil
}
if err != nil {
return last, fmt.Errorf("%s reported failed job and exited with %w: %s", filepath.Base(runner.path), err, result.line)
}
return last, fmt.Errorf("%s reported failed job: %s", filepath.Base(runner.path), result.line)
case <-ticker.C:
completion, err := d.observeGitHubJob(ctx, runnerName, dispatchedAfter)
if err != nil {
return last, err
}
if completion.WorkflowRunID != 0 {
last = completion
}
if !completion.Terminal {
continue
}
runner.cancel()
_ = <-runner.done
if completion.Success {
return completion, nil
}
return completion, fmt.Errorf("github workflow job failed: %s", completion.Message)
case <-ctx.Done():
runner.cancel()
err := <-runner.done
if err != nil {
return last, fmt.Errorf("%s timed out waiting for GitHub job completion: %w: %s", filepath.Base(runner.path), ctx.Err(), runner.output())
}
return last, ctx.Err()
}
}
}

func (d *runnerDriver) observeGitHubJob(ctx context.Context, runnerName string, dispatchedAfter time.Time) (githubJobCompletion, error) {
runs, err := d.sidecar.workflowRuns(ctx, d.req.Repository, d.req.Workflow, dispatchedAfter)
if err != nil {
return githubJobCompletion{}, err
}
for _, run := range runs {
jobs, err := d.sidecar.workflowRunJobs(ctx, d.req.Repository, run.ID)
if err != nil {
return githubJobCompletion{}, err
}
for _, job := range jobs {
if strings.TrimSpace(job.RunnerName) != runnerName {
continue
}
completion := githubJobCompletion{
WorkflowRunID: run.ID,
WorkflowJobID: job.ID,
Message: fmt.Sprintf("run=%d job=%d status=%s conclusion=%s runner=%s", run.ID, job.ID, job.Status, job.Conclusion, job.RunnerName),
}
if !strings.EqualFold(job.Status, "completed") && !strings.EqualFold(run.Status, "completed") {
return completion, nil
}
completion.Terminal = true
conclusion := strings.ToLower(strings.TrimSpace(job.Conclusion))
if conclusion == "" {
conclusion = strings.ToLower(strings.TrimSpace(run.Conclusion))
}
completion.Success = conclusion == "success" || conclusion == "succeeded"
return completion, nil
}
}
return githubJobCompletion{}, nil
}

func (d *runnerDriver) RemoveOrgRunner(ctx context.Context, organization string, runnerID int64) error {
return d.sidecar.removeOrgRunner(ctx, organization, runnerID)
}
Expand Down
166 changes: 165 additions & 1 deletion cmd/github-actions-runner-job/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ func TestT915CommandRunsDynamicProviderEnvelopeThroughSidecarAndRunner(t *testin
case r.Method == http.MethodDelete && r.URL.Path == "/v1/actions/orgs/GoCodeAlone/runners/42":
deleteCalls++
w.WriteHeader(http.StatusNoContent)
case r.Method == http.MethodGet && r.URL.Path == "/v1/actions/repos/GoCodeAlone/workflow-compute/workflows/dogfood.yml/runs":
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"workflow_runs":[]}`))
default:
t.Fatalf("unexpected sidecar request: %s %s", r.Method, r.URL.Path)
}
Expand Down Expand Up @@ -181,6 +184,9 @@ func TestT915CommandTreatsRunnerSuccessMarkerAsCompletion(t *testing.T) {
case r.Method == http.MethodDelete && r.URL.Path == "/v1/actions/orgs/GoCodeAlone/runners/42":
deleteCalls++
w.WriteHeader(http.StatusNoContent)
case r.Method == http.MethodGet && r.URL.Path == "/v1/actions/repos/GoCodeAlone/workflow-compute/workflows/dogfood.yml/runs":
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"workflow_runs":[]}`))
default:
t.Fatalf("unexpected sidecar request: %s %s", r.Method, r.URL.Path)
}
Expand All @@ -189,7 +195,7 @@ func TestT915CommandTreatsRunnerSuccessMarkerAsCompletion(t *testing.T) {

runnerDir := t.TempDir()
writeExecutable(t, filepath.Join(runnerDir, "config.sh"), "#!/bin/sh\nprintf '{\"agentId\":42}\\n' > .runner\n")
writeExecutable(t, filepath.Join(runnerDir, "run.sh"), "#!/bin/sh\ntouch \"$GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR/run.started\"\nwhile [ ! -f \"$GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR/dispatch.seen\" ]; do sleep 0.1; done\nprintf '2026-06-30T12:58:48Z: Job provider-target completed with result: Succeeded\\n'\ntouch \"$GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR/success.marker\"\nwhile true; do sleep 1; done\n")
writeExecutable(t, filepath.Join(runnerDir, "run.sh"), "#!/bin/sh\ntouch \"$GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR/run.started\"\nwhile [ ! -f \"$GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR/dispatch.seen\" ]; do sleep 0.1; done\nprintf '2026-06-30T12:58:48Z: Job provider-target completed with result: Succeeded\\n'\ntouch \"$GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR/success.marker\"\nexec sleep 1000\n")
t.Chdir(workspace)
t.Setenv("COMPUTE_GITHUB_RUNNER_PROVIDER_URL", sidecar.URL)
t.Setenv("COMPUTE_GITHUB_RUNNER_PROVIDER_TOKEN", "provider-token")
Expand Down Expand Up @@ -240,6 +246,164 @@ func TestT915CommandTreatsRunnerSuccessMarkerAsCompletion(t *testing.T) {
}
}

func TestT915CommandTreatsGitHubJobAPIAsCompletion(t *testing.T) {
oldPoll := githubJobPollInterval
githubJobPollInterval = 10 * time.Millisecond
t.Cleanup(func() { githubJobPollInterval = oldPoll })

var tokenCalls, dispatchCalls, runsCalls, jobsCalls, deleteCalls int
workspace := t.TempDir()
sidecar := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer provider-token" {
t.Fatalf("authorization = %q", got)
}
switch {
case r.Method == http.MethodPost && r.URL.Path == "/v1/actions/orgs/GoCodeAlone/runners/registration-token":
tokenCalls++
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"token":"runner-registration-token","expires_at":"2026-06-26T22:00:00Z"}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/actions/repos/GoCodeAlone/workflow-compute/workflows/dogfood.yml/dispatches":
dispatchCalls++
if err := os.WriteFile(filepath.Join(workspace, "dispatch.seen"), []byte("1\n"), 0o600); err != nil {
t.Fatalf("write dispatch marker: %v", err)
}
w.WriteHeader(http.StatusNoContent)
case r.Method == http.MethodGet && r.URL.Path == "/v1/actions/repos/GoCodeAlone/workflow-compute/workflows/dogfood.yml/runs":
runsCalls++
if r.URL.Query().Get("created_after") == "" {
t.Fatalf("created_after query is required")
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"workflow_runs":[{"id":28449657934,"status":"completed","conclusion":"success","html_url":"https://github.com/GoCodeAlone/workflow-compute/actions/runs/28449657934"}]}`))
case r.Method == http.MethodGet && r.URL.Path == "/v1/actions/repos/GoCodeAlone/workflow-compute/actions/runs/28449657934/jobs":
jobsCalls++
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"jobs":[{"id":84308154551,"run_id":28449657934,"status":"completed","conclusion":"success","runner_name":"wfc-stg-ghp-linux-abcdef987249-543210f71ee4","labels":["self-hosted","linux","wfc-ghp-stg"]}]}`))
case r.Method == http.MethodDelete && r.URL.Path == "/v1/actions/orgs/GoCodeAlone/runners/42":
deleteCalls++
w.WriteHeader(http.StatusNoContent)
default:
t.Fatalf("unexpected sidecar request: %s %s", r.Method, r.URL.String())
}
}))
t.Cleanup(sidecar.Close)

runnerDir := t.TempDir()
writeExecutable(t, filepath.Join(runnerDir, "config.sh"), "#!/bin/sh\nprintf '{\"agentId\":42}\\n' > .runner\n")
writeExecutable(t, filepath.Join(runnerDir, "run.sh"), "#!/bin/sh\ntouch \"$GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR/run.started\"\nexec sleep 1000\n")
t.Chdir(workspace)
t.Setenv("COMPUTE_GITHUB_RUNNER_PROVIDER_URL", sidecar.URL)
t.Setenv("COMPUTE_GITHUB_RUNNER_PROVIDER_TOKEN", "provider-token")
t.Setenv("GITHUB_ACTIONS_RUNNER_DIR", runnerDir)
t.Setenv("GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR", workspace)

input := strings.NewReader(`{
"protocol_version":"compute.v1alpha1",
"task_id":"task-abcdef9876543210",
"lease_id":"lease-1",
"provider_config":{"plugin_id":"workflow-plugin-github","provider_id":"github-actions-runner"},
"operation":"ephemeral_runner_job",
"input":{
"mode":"dispatch_then_wait",
"environment":"stg",
"os":"linux",
"worker_id":"worker-0123456789abcdef",
"task_id":"task-abcdef9876543210",
"organization":"GoCodeAlone",
"repository":"GoCodeAlone/workflow-compute",
"workflow":"dogfood.yml",
"ref":"main",
"runner_group":"ephemeral",
"timeout_seconds":2
}
}`)
var stdout, stderr bytes.Buffer
if err := runWithIO([]string{}, input, &stdout, &stderr); err != nil {
t.Fatalf("run dynamic provider: %v\nstderr:\n%s", err, stderr.String())
}
if tokenCalls != 1 || dispatchCalls != 1 || runsCalls == 0 || jobsCalls == 0 || deleteCalls != 1 {
t.Fatalf("sidecar calls: token=%d dispatch=%d runs=%d jobs=%d delete=%d", tokenCalls, dispatchCalls, runsCalls, jobsCalls, deleteCalls)
}
proof := readFile(t, filepath.Join(workspace, "github-runner-proof.json"))
for _, want := range []string{"28449657934", "84308154551", "removed", "wfc-stg-ghp-linux-abcdef987249-543210f71ee4"} {
if !strings.Contains(proof, want) {
t.Fatalf("proof missing %q:\n%s", want, proof)
}
}
}

func TestT915CommandRejectsFailedGitHubJobAPICompletion(t *testing.T) {
oldPoll := githubJobPollInterval
githubJobPollInterval = 10 * time.Millisecond
t.Cleanup(func() { githubJobPollInterval = oldPoll })

workspace := t.TempDir()
sidecar := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer provider-token" {
t.Fatalf("authorization = %q", got)
}
switch {
case r.Method == http.MethodPost && r.URL.Path == "/v1/actions/orgs/GoCodeAlone/runners/registration-token":
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"token":"runner-registration-token","expires_at":"2026-06-26T22:00:00Z"}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/actions/repos/GoCodeAlone/workflow-compute/workflows/dogfood.yml/dispatches":
w.WriteHeader(http.StatusNoContent)
case r.Method == http.MethodGet && r.URL.Path == "/v1/actions/repos/GoCodeAlone/workflow-compute/workflows/dogfood.yml/runs":
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"workflow_runs":[{"id":28449657935,"status":"completed","conclusion":"failure"}]}`))
case r.Method == http.MethodGet && r.URL.Path == "/v1/actions/repos/GoCodeAlone/workflow-compute/actions/runs/28449657935/jobs":
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"jobs":[{"id":84308154552,"run_id":28449657935,"status":"completed","conclusion":"failure","runner_name":"wfc-stg-ghp-linux-abcdef987249-543210f71ee4"}]}`))
case r.Method == http.MethodDelete && r.URL.Path == "/v1/actions/orgs/GoCodeAlone/runners/42":
w.WriteHeader(http.StatusNoContent)
default:
t.Fatalf("unexpected sidecar request: %s %s", r.Method, r.URL.String())
}
}))
t.Cleanup(sidecar.Close)

runnerDir := t.TempDir()
writeExecutable(t, filepath.Join(runnerDir, "config.sh"), "#!/bin/sh\nprintf '{\"agentId\":42}\\n' > .runner\n")
writeExecutable(t, filepath.Join(runnerDir, "run.sh"), "#!/bin/sh\nexit 0\n")
t.Chdir(workspace)
t.Setenv("COMPUTE_GITHUB_RUNNER_PROVIDER_URL", sidecar.URL)
t.Setenv("COMPUTE_GITHUB_RUNNER_PROVIDER_TOKEN", "provider-token")
t.Setenv("GITHUB_ACTIONS_RUNNER_DIR", runnerDir)
t.Setenv("GITHUB_ACTIONS_RUNNER_JOB_TEST_DIR", workspace)

input := strings.NewReader(`{
"protocol_version":"compute.v1alpha1",
"task_id":"task-abcdef9876543210",
"lease_id":"lease-1",
"provider_config":{"plugin_id":"workflow-plugin-github","provider_id":"github-actions-runner"},
"operation":"ephemeral_runner_job",
"input":{
"mode":"dispatch_then_wait",
"environment":"stg",
"os":"linux",
"worker_id":"worker-0123456789abcdef",
"task_id":"task-abcdef9876543210",
"organization":"GoCodeAlone",
"repository":"GoCodeAlone/workflow-compute",
"workflow":"dogfood.yml",
"ref":"main",
"runner_group":"ephemeral",
"timeout_seconds":2
}
}`)
var stdout, stderr bytes.Buffer
err := runWithIO([]string{}, input, &stdout, &stderr)
if err == nil || !strings.Contains(err.Error(), "github workflow job failed") {
t.Fatalf("expected failed GitHub job error, got %v\nstderr:\n%s", err, stderr.String())
}
proof := readFile(t, filepath.Join(workspace, "github-runner-proof.json"))
for _, want := range []string{"28449657935", "84308154552"} {
if !strings.Contains(proof, want) {
t.Fatalf("proof missing %q:\n%s", want, proof)
}
}
}

func TestT915RunnerCompletionMarkerRejectsFailedJob(t *testing.T) {
for _, line := range []string{
"2026-06-30T12:58:48Z: Job provider-target completed with result: Failed",
Expand Down
Loading