diff --git a/CHANGELOG.md b/CHANGELOG.md index 92fdc34be..68ec74841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Added a `GET /api/health/ready` endpoint that returns per-dependency health (Postgres, Redis, Zoekt) for use as a Kubernetes `readinessProbe` or load-balancer health check. The existing `GET /api/health` endpoint is unchanged and remains the liveness probe. [#1507](https://github.com/sourcebot-dev/sourcebot/pull/1507) +- Added a `?strict=true` query parameter to `GET /api/health/ready`. When set, the endpoint reports `503 / status: "degraded"` and `zoekt.status: "empty"` if the Zoekt shard set contains no indexed repositories, distinguishing "Zoekt is unreachable" from "Zoekt is up but the instance is not yet useful" for orchestrators. Default behavior is unchanged. [#1512](https://github.com/sourcebot-dev/sourcebot/pull/1512) + ### Changed - Vulnerability triage now keeps Linear issues synchronized with current security findings. diff --git a/docs/docs.json b/docs/docs.json index ad45e7fc8..419db0dbb 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -217,7 +217,7 @@ "icon": "server", "pages": [ "GET /api/version", - "GET /api/health" + "docs/api-reference/health" ] } ] diff --git a/docs/docs/api-reference/health.mdx b/docs/docs/api-reference/health.mdx new file mode 100644 index 000000000..bafc8cfa2 --- /dev/null +++ b/docs/docs/api-reference/health.mdx @@ -0,0 +1,130 @@ +--- +title: "Health Endpoints" +description: "Liveness and readiness probes for orchestrators and monitoring systems." +--- + +Sourcebot exposes two public health endpoints that follow the standard Kubernetes liveness / readiness split. Both are unauthenticated and return no user data. + +## Liveness: `GET /api/health` + +Returns `200 OK` with `{ "status": "ok" }` whenever the Next.js process is running and able to handle a request. Does not touch the database, Redis, or Zoekt. Use this for Kubernetes `livenessProbe` or Docker Compose `healthcheck.test`. A failing liveness probe means the process must be restarted. + +```bash +curl -fsS https://sourcebot.example.com/api/health +# {"status":"ok"} +``` + +## Readiness: `GET /api/health/ready` + +Returns `200 OK` with `{"status":"ok", "checks":{...}}` when Postgres, Redis, and Zoekt are all reachable. Returns `503 Service Unavailable` with `{"status":"degraded", "checks":{...}}` if any dependency is unreachable. Each check runs in parallel with a 2-second per-check timeout, so the worst-case request time is bounded even when a dependency hangs. + +Use this for Kubernetes `readinessProbe` or a load balancer health check. A failing readiness probe means the pod should be removed from the load-balancer rotation but not restarted. + +### Strict mode + +Add `?strict=true` to require the Zoekt shard set to be non-empty as well. The default mode only confirms the dependencies are reachable; strict mode confirms the instance can actually serve a search that returns results. A freshly-started instance, or one whose connection sync has silently failed, returns `503` until at least one repository is indexed. + +```bash +curl -fsS 'https://sourcebot.example.com/api/health/ready?strict=true' | jq .status +# "ok" Postgres, Redis, and Zoekt (with at least one indexed repo) are healthy +# "degraded" Something failed, see `checks` for which +``` + +The response always includes a top-level `strict` field so the operator can confirm the mode that was applied: + +```json +{ + "status": "degraded", + "strict": true, + "checks": { + "postgres": { "status": "ok", "latencyMs": 3 }, + "redis": { "status": "ok", "latencyMs": 1 }, + "zoekt": { "status": "empty", "latencyMs": 18, "error": "no repositories indexed (strict mode)" } + } +} +``` + +The Zoekt check distinguishes the two failure modes in its `status` field: +- `error`. The gRPC call itself failed (Zoekt unreachable, timeout, etc.). +- `empty`. Strict mode is on and the shard set is empty. The dependency is healthy; the instance is just not yet useful. + +### Response shape + +```json +{ + "status": "ok", + "strict": false, + "checks": { + "postgres": { "status": "ok", "latencyMs": 3 }, + "redis": { "status": "ok", "latencyMs": 1 }, + "zoekt": { "status": "ok", "latencyMs": 12 } + } +} +``` + +When degraded, each failed check carries an `error` field with the underlying message: + +```json +{ + "status": "degraded", + "strict": false, + "checks": { + "postgres": { "status": "ok", "latencyMs": 4 }, + "redis": { "status": "ok", "latencyMs": 1 }, + "zoekt": { "status": "error", "latencyMs": 2003, "error": "zoekt check timed out after 2000ms" } + } +} +``` + +| Check | What it probes | +|-------|---------------| +| `postgres` | `SELECT 1` via Prisma | +| `redis` | `PING` (rejects non-`PONG` responses) | +| `zoekt` | Empty `List` RPC (proves the gRPC channel is alive; bounded by the 2s per-check timeout) | + +### Example probes + + + + ```yaml + services: + sourcebot: + image: sourcebot/sourcebot:latest + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"] + interval: 30s + timeout: 5s + retries: 3 + # For dependency-aware probes, point the orchestrator at /api/health/ready + # instead. Sourcebot's example compose file does this via a sidecar. + ``` + + + ```yaml + livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/health/ready + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + # For deployments that should not receive traffic until the shard set + # is non-empty, switch to the strict readiness probe: + # path: /api/health/ready?strict=true + ``` + + + + +The readiness probe hits the database on every call. On large deployments with many pods, a high-frequency probe interval (sub-5s) can produce noticeable background load. A 10s interval with `failureThreshold: 3` is a good starting point. + diff --git a/packages/web/src/app/api/(server)/health/ready/route.test.ts b/packages/web/src/app/api/(server)/health/ready/route.test.ts new file mode 100644 index 000000000..b49269304 --- /dev/null +++ b/packages/web/src/app/api/(server)/health/ready/route.test.ts @@ -0,0 +1,378 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +const mocks = vi.hoisted(() => ({ + unsafePrisma: { + $queryRaw: vi.fn(), + }, + redisPing: vi.fn(), + zoektList: vi.fn(), +})); + +vi.mock('server-only', () => ({})); + +vi.mock('@/prisma', () => ({ + __unsafePrisma: mocks.unsafePrisma, +})); + +vi.mock('@/lib/redis', () => ({ + getRedisClient: () => ({ + ping: mocks.redisPing, + }), +})); + +vi.mock('@/lib/posthog', () => ({ + captureEvent: vi.fn(), +})); + +vi.mock('@/lib/zoektClient', () => ({ + loadZoektClient: () => ({ + List: mocks.zoektList, + }), +})); + +vi.mock('@sourcebot/shared', () => ({ + createLogger: () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: mockLoggerWarn, + error: vi.fn(), + }), +})); + +const mockLoggerWarn = vi.fn(); + +const { GET } = await import('./route'); + +// Minimal NextRequest stand-in. We only need `nextUrl.searchParams`; the +// runtime calls are not exercised. +const makeRequest = (search: Record = {}): NextRequest => { + const params = new URLSearchParams(search); + const url = `http://localhost/api/health/ready?${params.toString()}`; + return new NextRequest(url); +}; + +// Default Zoekt response: a single indexed repo. Tests that need a +// different shape override the mock implementation locally. +const defaultZoektResponse = { repos: [{}] }; + +// gRPC callback shape: (err, response) => void. Pulled out so the seven +// `mocks.zoektList.mockImplementation(...)` blocks below stay short. +type ZoektListCallback = ( + err: Error | null, + response?: { repos?: unknown[]; repos_map?: Record }, +) => void; + +describe('GET /api/health/ready', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.unsafePrisma.$queryRaw.mockResolvedValue([{ '?column?': 1 }]); + mocks.redisPing.mockResolvedValue('PONG'); + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, defaultZoektResponse); + }, + ); + }); + + test('returns 200 with status:ok and strict:false when all three dependencies are reachable', async () => { + const response = await GET(makeRequest()); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe('ok'); + expect(body.strict).toBe(false); + expect(body.checks.postgres.status).toBe('ok'); + expect(body.checks.redis.status).toBe('ok'); + expect(body.checks.zoekt.status).toBe('ok'); + expect(typeof body.checks.postgres.latencyMs).toBe('number'); + expect(typeof body.checks.redis.latencyMs).toBe('number'); + expect(typeof body.checks.zoekt.latencyMs).toBe('number'); + }); + + test('returns 503 with status:degraded and a generic postgres error when Postgres is unreachable', async () => { + const internalError = new Error('connection refused: host=db.internal.example.com:5432'); + mocks.unsafePrisma.$queryRaw.mockRejectedValue(internalError); + + const response = await GET(makeRequest()); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body.status).toBe('degraded'); + expect(body.checks.postgres.status).toBe('error'); + // The public error is generic. The internal message (which + // contains the host:port) must not leak. + expect(body.checks.postgres.error).toBe('postgres check failed: see server logs'); + expect(body.checks.postgres.error).not.toContain('db.internal.example.com'); + expect(body.checks.postgres.errorDetail).toBeUndefined(); + expect(body.checks.redis.status).toBe('ok'); + expect(body.checks.zoekt.status).toBe('ok'); + + // The full error detail is preserved on the server-side log call. + const warnCall = mockLoggerWarn.mock.calls[mockLoggerWarn.mock.calls.length - 1]; + expect(JSON.stringify(warnCall)).toContain('db.internal.example.com'); + }); + + test('returns 503 with status:degraded and a generic redis error when Redis ping fails', async () => { + const internalError = new Error('redis down: redis://10.0.0.42:6379'); + mocks.redisPing.mockRejectedValue(internalError); + + const response = await GET(makeRequest()); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body.status).toBe('degraded'); + expect(body.checks.postgres.status).toBe('ok'); + expect(body.checks.redis.status).toBe('error'); + expect(body.checks.redis.error).toBe('redis check failed: see server logs'); + expect(body.checks.redis.error).not.toContain('10.0.0.42'); + expect(body.checks.redis.errorDetail).toBeUndefined(); + expect(body.checks.zoekt.status).toBe('ok'); + }); + + test('returns 503 with status:degraded and a generic zoekt error when the gRPC call fails', async () => { + const internalError = new Error('UNAVAILABLE: zoekt-web-0.zoekt.svc.cluster.local:6070'); + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(internalError); + }, + ); + + const response = await GET(makeRequest()); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body.status).toBe('degraded'); + expect(body.checks.zoekt.status).toBe('error'); + expect(body.checks.zoekt.error).toBe('zoekt check failed: see server logs'); + expect(body.checks.zoekt.error).not.toContain('cluster.local'); + expect(body.checks.zoekt.errorDetail).toBeUndefined(); + expect(body.checks.postgres.status).toBe('ok'); + expect(body.checks.redis.status).toBe('ok'); + }); + + test('returns 503 with status:degraded when Redis returns a non-PONG response', async () => { + mocks.redisPing.mockResolvedValue('NOT-PONG'); + + const response = await GET(makeRequest()); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body.status).toBe('degraded'); + expect(body.checks.redis.status).toBe('error'); + // "unexpected ping response" is the internal message; the public + // path should not include it. + expect(body.checks.redis.error).toBe('redis check failed: see server logs'); + }); + + test('runs all three checks in parallel (Promise.all)', async () => { + const delay = 50; + mocks.unsafePrisma.$queryRaw.mockImplementation( + () => new Promise((resolve) => setTimeout(() => resolve([{}]), delay)), + ); + mocks.redisPing.mockImplementation( + () => new Promise((resolve) => setTimeout(() => resolve('PONG'), delay)), + ); + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + setTimeout(() => callback(null, defaultZoektResponse), delay); + }, + ); + + const start = Date.now(); + const response = await GET(makeRequest()); + const elapsed = Date.now() - start; + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe('ok'); + // Generous upper bound to avoid flakes; serial would be ~3x delay. + expect(elapsed).toBeLessThan(delay * 2.5); + }); + + test('does not surface check rejections as unhandled promise rejections', async () => { + // The check rejects synchronously (well within the 2s timeout). The + // no-op `.catch` attached in `withTimeout` must absorb that + // rejection so the Node process does not log an + // unhandled-promise-rejection warning while the readiness request + // has already moved on. + const checkRejection = new Error('check rejected'); + const unhandled: unknown[] = []; + const onUnhandled = (err: unknown) => { unhandled.push(err); }; + process.on('unhandledRejection', onUnhandled); + + try { + mocks.unsafePrisma.$queryRaw.mockRejectedValue(checkRejection); + mocks.redisPing.mockResolvedValue('PONG'); + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, defaultZoektResponse); + }, + ); + + const response = await GET(makeRequest()); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body.checks.postgres.status).toBe('error'); + // Give the rejection microtask a chance to fire and propagate. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(unhandled).not.toContain(checkRejection); + } finally { + process.off('unhandledRejection', onUnhandled); + } + }); + + test('issues the Zoekt List RPC with empty options (max_wall_time is a SearchOptions field, not ListOptions)', async () => { + // Regression guard: the earlier draft of the Zoekt probe passed + // `{ opts: { max_wall_time: ... } }` to the `List` RPC. That field + // belongs to `SearchOptions` and is silently ignored by `List` + // (whose `ListOptions` only carries `field`). The 2s client-side + // timeout is the only thing that actually bounds the call. The + // probe must therefore issue the smallest valid request, which is + // an empty options object. + const response = await GET(makeRequest()); + expect(response.status).toBe(200); + expect(mocks.zoektList).toHaveBeenCalledTimes(1); + expect(mocks.zoektList).toHaveBeenCalledWith({}, expect.any(Function)); + }); + + test('reports zoekt.status:"error" (not a thrown exception) when the gRPC callback fires with no response', async () => { + // Regression guard: a malformed gRPC response can fire the callback + // with `err === null` and `result === undefined`. Without the + // `!result` check the route would throw on `response.repos` and + // surface the failure as a generic Node unhandled-rejection path + // instead of the controlled `error` status. `zoektSearch` applies + // the same `error || !response` guard. + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, undefined); + }, + ); + + const response = await GET(makeRequest()); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body.status).toBe('degraded'); + expect(body.checks.zoekt.status).toBe('error'); + expect(body.checks.zoekt.error).toBe('zoekt check failed: see server logs'); + expect(body.checks.postgres.status).toBe('ok'); + expect(body.checks.redis.status).toBe('ok'); + }); + + describe('?strict=true', () => { + test('returns 200 with status:ok and strict:true when Zoekt has at least one indexed repo', async () => { + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, { repos: [{ repository: { name: 'foo' } }] }); + }, + ); + + const response = await GET(makeRequest({ strict: 'true' })); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe('ok'); + expect(body.strict).toBe(true); + expect(body.checks.zoekt.status).toBe('ok'); + }); + + test('returns 503 with status:degraded and zoekt.status:"empty" when Zoekt has no indexed repos', async () => { + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, { repos: [] }); + }, + ); + + const response = await GET(makeRequest({ strict: 'true' })); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body.status).toBe('degraded'); + expect(body.strict).toBe(true); + expect(body.checks.zoekt.status).toBe('empty'); + expect(body.checks.zoekt.error).toContain('no repositories indexed'); + expect(body.checks.postgres.status).toBe('ok'); + expect(body.checks.redis.status).toBe('ok'); + }); + + test('returns 503 in strict mode when the Zoekt response uses repos_map (RepoListFieldReposMap)', async () => { + // The gRPC server may return `repos_map` (a numeric-keyed object) + // when `ListOptions.Field = RepoListFieldReposMap`. Empty + // `repos_map` should also be treated as empty in strict mode. + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, { repos_map: {} }); + }, + ); + + const response = await GET(makeRequest({ strict: 'true' })); + const body = await response.json(); + + expect(response.status).toBe(503); + expect(body.checks.zoekt.status).toBe('empty'); + }); + + test('returns 200 in strict mode when repos_map is non-empty', async () => { + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, { repos_map: { 1: { repository: { name: 'bar' } } } }); + }, + ); + + const response = await GET(makeRequest({ strict: 'true' })); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.checks.zoekt.status).toBe('ok'); + }); + }); + + describe('?strict=false and absent', () => { + test('returns 200 with strict:false and zoekt.status:ok when Zoekt has zero repos', async () => { + // Backward-compat: the default behavior must NOT consider an + // empty Zoekt shard set as a failure. Operators who do not opt + // in to strict mode should see the same response as before. + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, { repos: [] }); + }, + ); + + const response = await GET(makeRequest({ strict: 'false' })); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe('ok'); + expect(body.strict).toBe(false); + expect(body.checks.zoekt.status).toBe('ok'); + }); + + test('treats an absent strict parameter as strict:false', async () => { + mocks.zoektList.mockImplementation( + (_request: unknown, callback: ZoektListCallback) => { + callback(null, { repos: [] }); + }, + ); + + const response = await GET(makeRequest()); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.strict).toBe(false); + expect(body.checks.zoekt.status).toBe('ok'); + }); + }); + + describe('invalid ?strict value', () => { + test('returns 400 with a clear error message when strict is not parseable', async () => { + const response = await GET(makeRequest({ strict: 'yes' })); + const body = await response.json(); + + expect(response.status).toBe(400); + expect(body.message).toBeDefined(); + expect(body.message).toMatch(/strict/); + }); + }); +}); diff --git a/packages/web/src/app/api/(server)/health/ready/route.ts b/packages/web/src/app/api/(server)/health/ready/route.ts new file mode 100644 index 000000000..e3d4948aa --- /dev/null +++ b/packages/web/src/app/api/(server)/health/ready/route.ts @@ -0,0 +1,233 @@ +import { createLogger } from '@sourcebot/shared'; +import { NextRequest } from 'next/server'; +import { z } from 'zod'; + +import { apiHandler } from '@/lib/apiHandler'; +import { __unsafePrisma } from '@/prisma'; +import { getRedisClient } from '@/lib/redis'; +import { queryParamsSchemaValidationError, serviceErrorResponse } from '@/lib/serviceError'; +import { loadZoektClient, ZoektListResponse } from '@/lib/zoektClient'; + +// Per-check timeout. The three checks run in parallel, so the worst-case +// request time is bounded by this value even when one dependency hangs. +const READINESS_TIMEOUT_MS = 2000; + +const logger = createLogger('health-ready'); + +type CheckStatus = 'ok' | 'error' | 'empty'; +type CheckResult = { + status: CheckStatus; + latencyMs: number; + /** Client-safe error description. Optional; present when status !== 'ok'. */ + error?: string; + /** + * Server-side error detail. Never sent over the wire: the public + * `error` field is the only one that ends up in the JSON response. + * Logged by the GET handler when the overall probe is degraded. + */ + errorDetail?: string; +}; +type ReadinessResponse = { + status: 'ok' | 'degraded'; + strict: boolean; + checks: { + postgres: CheckResult; + redis: CheckResult; + zoekt: CheckResult; + }; +}; + +const queryParamsSchema = z.object({ + // `z.coerce.boolean()` is a footgun: it just calls `Boolean(value)` and + // would treat the string `"false"` as truthy. Accept only the two + // literal strings we mean to support and transform to a boolean. + strict: z + .string() + .optional() + .refine( + (v) => v === undefined || v === 'true' || v === 'false', + { message: 'strict must be "true" or "false"' }, + ) + .transform((v) => v === 'true'), +}); + +// Runs `check()` and rejects if it has not settled after `timeoutMs`. When the +// timeout fires, the underlying check promise may still resolve or reject +// later; the no-op `.catch` below attaches to that promise so a late +// rejection does not surface as an unhandled-promise-rejection in the Node +// process while the readiness request has already moved on. +const withTimeout = async ( + label: string, + check: () => Promise, + timeoutMs: number, +): Promise => { + const checkPromise = check(); + checkPromise.catch(() => { /* swallowed: see comment above */ }); + + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + reject(new Error(`${label} check timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }); + try { + return await Promise.race([checkPromise, timeout]); + } finally { + if (timer) { + clearTimeout(timer); + } + } +}; + +// The readiness route is unauthenticated and may be reachable at a public +// URL, so per-check error strings returned to the caller are intentionally +// generic. The full `err` is preserved on the `CheckResult` (under +// `errorDetail`) and logged server-side; the public `error` field carries +// only the check label and a hint about the failure mode. +const PUBLIC_ERROR = (label: string, detail: string) => `${label} check failed: ${detail}`; + +const checkPostgres = async (): Promise => { + const start = Date.now(); + try { + await withTimeout('postgres', async () => { + await __unsafePrisma.$queryRaw`SELECT 1`; + }, READINESS_TIMEOUT_MS); + return { status: 'ok', latencyMs: Date.now() - start }; + } catch (err) { + return { + status: 'error', + latencyMs: Date.now() - start, + error: PUBLIC_ERROR('postgres', 'see server logs'), + errorDetail: err instanceof Error ? err.message : String(err), + }; + } +}; + +const checkRedis = async (): Promise => { + const start = Date.now(); + try { + const redis = getRedisClient(); + await withTimeout('redis', async () => { + const pong = await redis.ping(); + if (pong !== 'PONG') { + throw new Error(`unexpected ping response: ${pong}`); + } + }, READINESS_TIMEOUT_MS); + return { status: 'ok', latencyMs: Date.now() - start }; + } catch (err) { + return { + status: 'error', + latencyMs: Date.now() - start, + error: PUBLIC_ERROR('redis', 'see server logs'), + errorDetail: err instanceof Error ? err.message : String(err), + }; + } +}; + +const checkZoekt = async (strict: boolean): Promise => { + const start = Date.now(); + try { + // The List response is captured so the strict path can decide + // whether the empty-shard case is acceptable. + const response = await withTimeout('zoekt', async () => { + // Build the client inside the timeout so a first-call init stall + // (e.g., vendored proto load) is also bounded. + const client = await loadZoektClient(); + return await new Promise((resolve, reject) => { + // Empty `List` is the smallest request that exercises the + // gRPC channel end-to-end. It returns an empty result, not + // an error, even when no repos are indexed. The 2s + // client-side `READINESS_TIMEOUT_MS` is the only timeout + // that actually bounds the call: `max_wall_time` is a + // `SearchOptions` field and does not apply to `List` + // (`ListOptions` only carries `field`). + client.List({}, (err, result) => { + // `zoektSearch` rejects when the callback fires with + // no error but no response either (see + // `features/search/zoektSearcher.ts`). Do the same + // here: a missing result would otherwise throw on + // `response.repos` and surface as a generic `error` + // instead of the intended `empty` strict-mode status. + if (err) { + reject(err); + } else if (!result) { + reject(new Error('zoekt List RPC returned no response')); + } else { + resolve(result); + } + }); + }); + }, READINESS_TIMEOUT_MS); + + if (strict) { + // Check both `repos` (Field = RepoListFieldRepos) and + // `repos_map` (Field = RepoListFieldReposMap) so the result is + // correct regardless of which field the server populates. + const repos = response.repos ?? []; + const reposMap = response.repos_map ?? {}; + if (repos.length === 0 && Object.keys(reposMap).length === 0) { + return { + status: 'empty', + latencyMs: Date.now() - start, + error: 'no repositories indexed (strict mode)', + }; + } + } + + return { status: 'ok', latencyMs: Date.now() - start }; + } catch (err) { + return { + status: 'error', + latencyMs: Date.now() - start, + error: PUBLIC_ERROR('zoekt', 'see server logs'), + errorDetail: err instanceof Error ? err.message : String(err), + }; + } +}; + +// eslint-disable-next-line authz/require-auth-wrapper -- public readiness probe, no user data returned +export const GET = apiHandler(async (request: NextRequest) => { + const rawParams = { + strict: request.nextUrl.searchParams.get('strict') ?? undefined, + }; + const parsed = queryParamsSchema.safeParse(rawParams); + + if (!parsed.success) { + return serviceErrorResponse( + queryParamsSchemaValidationError(parsed.error) + ); + } + + const { strict } = parsed.data; + + const [postgres, redis, zoekt] = await Promise.all([ + checkPostgres(), + checkRedis(), + checkZoekt(strict), + ]); + + // `errorDetail` is server-side only and must never reach the response + // body. The full per-check result (including `errorDetail`) is what + // the logger emits; the public response uses the stripped view. + const stripDetail = ({ errorDetail: _errorDetail, ...publicCheck }: CheckResult) => publicCheck; + const publicChecks = { + postgres: stripDetail(postgres), + redis: stripDetail(redis), + zoekt: stripDetail(zoekt), + }; + const healthy = postgres.status === 'ok' && redis.status === 'ok' && zoekt.status === 'ok'; + + if (!healthy) { + logger.warn('readiness check failed', { + checks: { postgres, redis, zoekt }, + strict, + }); + } + + const body: ReadinessResponse = { + status: healthy ? 'ok' : 'degraded', + strict, + checks: publicChecks, + }; + return Response.json(body, { status: healthy ? 200 : 503 }); +}, { track: false }); diff --git a/packages/web/src/lib/zoektClient.ts b/packages/web/src/lib/zoektClient.ts new file mode 100644 index 000000000..8e161ed66 --- /dev/null +++ b/packages/web/src/lib/zoektClient.ts @@ -0,0 +1,74 @@ +// Thin wrapper around the vendored Zoekt webserver gRPC client. Kept in +// a separate module so the readiness route can mock it in tests without +// pulling the gRPC + @opentelemetry CJS chain at test-load time. + +export type ZoektListRequest = { + opts?: { max_wall_time?: { seconds: number; nanos: number } }; +}; + +// Loose summary of the `List` response. The full gRPC type is much wider +// (see `proto/zoekt/webserver/v1/ListResponse.ts`); we only need the bits +// the readiness probe inspects, so callers can mock it with a small object. +export type ZoektListResponse = { + repos?: unknown[]; + repos_map?: Record; +}; + +export type ZoektClient = { + List: (request: ZoektListRequest, callback: (err: Error | null, result: ZoektListResponse) => void) => void; +}; + +let cachedClient: ZoektClient | undefined; + +const buildClient = async (): Promise => { + const [grpc, protoLoader, nodePath, shared] = await Promise.all([ + import('@grpc/grpc-js'), + import('@grpc/proto-loader'), + import('node:path'), + import('@sourcebot/shared'), + ]); + + const protoBasePath = nodePath.join(process.cwd(), '../../vendor/zoekt/grpc/protos'); + const protoPath = nodePath.join(protoBasePath, 'zoekt/webserver/v1/webserver.proto'); + + const packageDefinition = protoLoader.loadSync(protoPath, { + keepCase: true, + longs: Number, + enums: String, + defaults: true, + oneofs: true, + includeDirs: [protoBasePath], + }); + + const proto = grpc.loadPackageDefinition(packageDefinition) as unknown as { + zoekt: { webserver: { v1: { WebserverService: new (address: string, credentials: unknown, options?: Record) => ZoektClient } } }; + }; + + const zoektUrl = new URL(shared.env.ZOEKT_WEBSERVER_URL); + const grpcAddress = `${zoektUrl.hostname}:${zoektUrl.port}`; + // Match the channel options used by `zoektSearcher` so a healthy Zoekt + // instance with a large repo catalog does not fail the probe just + // because the default 4MB gRPC receive cap is smaller than the + // response. + return new proto.zoekt.webserver.v1.WebserverService( + grpcAddress, + grpc.credentials.createInsecure(), + { + 'grpc.max_receive_message_length': 500 * 1024 * 1024, // 500MB + 'grpc.max_send_message_length': 500 * 1024 * 1024, // 500MB + }, + ); +}; + +// Returns a connected client, building it on first use. Only successful +// builds are cached: a transient init failure does not poison subsequent +// readiness probes, so the next call can retry. +export const loadZoektClient = async (): Promise => { + if (cachedClient) { + return cachedClient; + } + const client = await buildClient(); + cachedClient = client; + return client; +}; +