diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d4227da4..457de3370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [EE] Added DPoP sender-constrained OAuth tokens for MCP clients. [#1395](https://github.com/sourcebot-dev/sourcebot/pull/1395) - [EE] Added text file attachments to Ask Sourcebot, letting users attach text/code/config files to a chat message via the paperclip button, drag-and-drop, or paste, with large pastes auto-converted to attachments. [#1374](https://github.com/sourcebot-dev/sourcebot/pull/1374) - [EE] Added image attachments to Ask Sourcebot, letting users attach images to a chat message when the selected model supports image input. [#1375](https://github.com/sourcebot-dev/sourcebot/pull/1375) +- Added deployment system resource stats (CPU cores + cgroup quota, host + container memory, disk, load average) to the service ping, so resource issues can be diagnosed more quickly. [#1424](https://github.com/sourcebot-dev/sourcebot/pull/1424) ### Fixed - Send anonymous server-side PostHog events as personless so unauthenticated requests don't inflate person counts. [#1367](https://github.com/sourcebot-dev/sourcebot/pull/1367) diff --git a/docs/docs/misc/service-ping.mdx b/docs/docs/misc/service-ping.mdx index 73bcdadea..3b6142e5d 100644 --- a/docs/docs/misc/service-ping.mdx +++ b/docs/docs/misc/service-ping.mdx @@ -25,6 +25,19 @@ The data contained within the Service Ping is limited to: | `isTelemetryEnabled` | `boolean` | Whether anonymous product telemetry is enabled. | | `isLanguageModelConfigured` | `boolean` | Whether at least one language model is configured. | | `activationCode` | `string` | Activation code, if your instance has one bound. | +| `systemInfo` | `object` | Resource facts about the deployment (CPU, memory, disk). See below. | + +The `systemInfo` object contains a best-effort snapshot of the resources the deployment is running with. It is used to diagnose resource issues (e.g. insufficient RAM). Any field that cannot be read is reported as `null`. + +| Field | Type | Description | +| --- | --- | --- | +| `platform` | `string` | Operating system platform (e.g. `linux`). | +| `arch` | `string` | CPU architecture (e.g. `x64`, `arm64`). | +| `cpuQuota` | `number \| null` | Effective cgroup CPU limit in cores, or `null` if unlimited. | +| `memoryLimitMiB` | `number \| null` | cgroup memory limit in MiB, or `null` if unlimited. | +| `memoryUsedMiB` | `number \| null` | Current cgroup memory usage, in MiB. | +| `diskTotalMiB` | `number \| null` | Total disk space for the data volume, in MiB. | +| `diskFreeMiB` | `number \| null` | Free disk space for the data volume, in MiB. | You can fetch the schema for the Service Ping by submitting a GET request to `https://deployments.sourcebot.dev/schema` @@ -59,6 +72,15 @@ body: { "deploymentType": "other", "isTelemetryEnabled": true, "isLanguageModelConfigured": true, + "systemInfo": { + "platform": "linux", + "arch": "x64", + "cpuQuota": 4, + "memoryLimitMiB": 8192, + "memoryUsedMiB": 3072, + "diskTotalMiB": 102400, + "diskFreeMiB": 54968 + }, "activationCode": "sb_act_3a925f51...." } ``` \ No newline at end of file diff --git a/packages/web/src/features/billing/servicePing.ts b/packages/web/src/features/billing/servicePing.ts index e20129a8a..6a0290c90 100644 --- a/packages/web/src/features/billing/servicePing.ts +++ b/packages/web/src/features/billing/servicePing.ts @@ -14,6 +14,7 @@ import { ServicePingRequest } from "./types"; import { ServiceErrorException } from "@/lib/serviceError"; import { getConfiguredLanguageModels } from "@/features/chat/utils.server"; import { activeMembershipWhere } from "@/features/membership/utils"; +import { getSystemInfo } from "./systemInfo"; const logger = createLogger('service-ping'); @@ -79,6 +80,12 @@ export const syncWithLighthouse = async (orgId: number) => { const isLanguageModelConfigured = (await getConfiguredLanguageModels()).length > 0; + // Best-effort — a failure to collect system info must never prevent the ping. + const systemInfo = await getSystemInfo().catch((error) => { + logger.warn(`Failed to collect system info for service ping: ${error}`); + return undefined; + }); + const payload: ServicePingRequest = { installId: env.SOURCEBOT_INSTALL_ID, version: SOURCEBOT_VERSION, @@ -91,6 +98,7 @@ export const syncWithLighthouse = async (orgId: number) => { deploymentType: inferDeploymentType(), isTelemetryEnabled: env.SOURCEBOT_TELEMETRY_DISABLED === 'false', isLanguageModelConfigured, + ...(systemInfo && { systemInfo }), ...(activationCode && { activationCode }), }; diff --git a/packages/web/src/features/billing/systemInfo.ts b/packages/web/src/features/billing/systemInfo.ts new file mode 100644 index 000000000..ca9051401 --- /dev/null +++ b/packages/web/src/features/billing/systemInfo.ts @@ -0,0 +1,143 @@ +import os from "node:os"; +import fs from "node:fs/promises"; +import { env, createLogger } from "@sourcebot/shared"; +import { SystemInfo } from "./types"; + +const logger = createLogger('system-info'); + +// cgroup v2 (unified hierarchy). +const CGROUP_V2_MEMORY_MAX = '/sys/fs/cgroup/memory.max'; +const CGROUP_V2_MEMORY_CURRENT = '/sys/fs/cgroup/memory.current'; +const CGROUP_V2_CPU_MAX = '/sys/fs/cgroup/cpu.max'; + +// cgroup v1 (legacy hierarchy). +const CGROUP_V1_MEMORY_LIMIT = '/sys/fs/cgroup/memory/memory.limit_in_bytes'; +const CGROUP_V1_MEMORY_USAGE = '/sys/fs/cgroup/memory/memory.usage_in_bytes'; +const CGROUP_V1_CPU_QUOTA = '/sys/fs/cgroup/cpu/cpu.cfs_quota_us'; +const CGROUP_V1_CPU_PERIOD = '/sys/fs/cgroup/cpu/cpu.cfs_period_us'; + +/** + * Collects a snapshot of the resources this deployment is running with, so we + * can quickly diagnose resource issues (e.g. "the customer doesn't have enough + * RAM") from the service ping. + * + * Everything is best-effort: any value we can't read is reported as `null` so a + * partial snapshot still goes out. Resource figures come from the container's + * cgroup (and `statfs` for disk) rather than host-level readings, which in a + * Dockerized or Kubernetes deployment report the underlying machine, not the + * container's actual limits. + */ +export const getSystemInfo = async (): Promise => { + const [memoryLimitBytes, memoryUsedBytes, cpuQuota, disk] = await Promise.all([ + readCgroupMemoryLimit(), + readCgroupMemoryUsage(), + readCgroupCpuQuota(), + readDiskUsage(env.DATA_CACHE_DIR), + ]); + + return { + platform: os.platform(), + arch: os.arch(), + cpuQuota, + memoryLimitMiB: memoryLimitBytes === null ? null : bytesToMiB(memoryLimitBytes), + memoryUsedMiB: memoryUsedBytes === null ? null : bytesToMiB(memoryUsedBytes), + diskTotalMiB: disk.totalBytes === null ? null : bytesToMiB(disk.totalBytes), + diskFreeMiB: disk.freeBytes === null ? null : bytesToMiB(disk.freeBytes), + }; +}; + +const BYTES_PER_MIB = 1024 * 1024; + +// Memory and disk are both reported in whole MiB (binary, 1024-based units), +// matching how RAM and container/k8s volumes are sized. +const bytesToMiB = (bytes: number): number => Math.round(bytes / BYTES_PER_MIB); + +const readFileTrimmed = async (path: string): Promise => { + try { + const contents = await fs.readFile(path, 'utf8'); + return contents.trim(); + } catch { + // The file won't exist outside of a cgroup-managed environment (e.g. + // local dev on macOS), which is expected — treat as "unknown". + return null; + } +}; + +/** + * Parses a byte count from a cgroup file, returning `null` for "unlimited": + * the literal `max` in cgroup v2, or a sentinel larger than JS can safely + * represent in cgroup v1 (e.g. 9223372036854771712). + */ +const parseCgroupBytes = (raw: string | null): number | null => { + if (raw === null || raw === 'max') { + return null; + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < 0) { + return null; + } + return value; +}; + +const readCgroupMemoryLimit = async (): Promise => { + const v2 = parseCgroupBytes(await readFileTrimmed(CGROUP_V2_MEMORY_MAX)); + if (v2 !== null) { + return v2; + } + return parseCgroupBytes(await readFileTrimmed(CGROUP_V1_MEMORY_LIMIT)); +}; + +const readCgroupMemoryUsage = async (): Promise => { + const v2 = parseCgroupBytes(await readFileTrimmed(CGROUP_V2_MEMORY_CURRENT)); + if (v2 !== null) { + return v2; + } + return parseCgroupBytes(await readFileTrimmed(CGROUP_V1_MEMORY_USAGE)); +}; + +/** + * Returns the effective CPU limit in cores (e.g. 0.5, 2), or `null` when there + * is no quota (unlimited) or it can't be read. + */ +const readCgroupCpuQuota = async (): Promise => { + // cgroup v2: `cpu.max` is " " (in microseconds), or + // "max " when unlimited. + const v2 = await readFileTrimmed(CGROUP_V2_CPU_MAX); + if (v2 !== null) { + const [quotaRaw, periodRaw] = v2.split(/\s+/); + if (quotaRaw === 'max') { + return null; + } + const quota = Number(quotaRaw); + const period = Number(periodRaw); + if (Number.isFinite(quota) && quota > 0 && Number.isFinite(period) && period > 0) { + return roundToTwo(quota / period); + } + return null; + } + + // cgroup v1: cpu.cfs_quota_us / cpu.cfs_period_us; a quota of -1 is unlimited. + const quota = Number(await readFileTrimmed(CGROUP_V1_CPU_QUOTA)); + const period = Number(await readFileTrimmed(CGROUP_V1_CPU_PERIOD)); + if (!Number.isFinite(quota) || quota <= 0 || !Number.isFinite(period) || period <= 0) { + return null; + } + return roundToTwo(quota / period); +}; + +const readDiskUsage = async (path: string): Promise<{ totalBytes: number | null; freeBytes: number | null }> => { + try { + const stats = await fs.statfs(path); + return { + totalBytes: stats.blocks * stats.bsize, + // bavail is blocks available to unprivileged users, which best + // reflects what the deployment can actually write. + freeBytes: stats.bavail * stats.bsize, + }; + } catch (error) { + logger.debug(`Failed to read disk usage for ${path}: ${error}`); + return { totalBytes: null, freeBytes: null }; + } +}; + +const roundToTwo = (n: number): number => Math.round(n * 100) / 100; diff --git a/packages/web/src/features/billing/types.ts b/packages/web/src/features/billing/types.ts index 0be11074b..f97523fc3 100644 --- a/packages/web/src/features/billing/types.ts +++ b/packages/web/src/features/billing/types.ts @@ -1,5 +1,30 @@ import { z } from "zod"; +/** + * A best-effort snapshot of the host / container resources the deployment is + * running with. Used to quickly diagnose resource issues (e.g. insufficient + * RAM) from the service ping. Every field is best-effort; anything we can't + * read is reported as `null`. + */ +export const systemInfoSchema = z.object({ + platform: z.string(), + arch: z.string(), + + // CPU. `cpuQuota` is the effective cgroup CPU limit in cores (null when + // unset or unreadable), which is what a container is actually allowed to use. + cpuQuota: z.number().nonnegative().nullable(), + + // Memory, in MiB, from the cgroup: the container's actual RAM limit and + // current usage (null when unset or unreadable). + memoryLimitMiB: z.number().nonnegative().nullable(), + memoryUsedMiB: z.number().nonnegative().nullable(), + + // Disk, in MiB, for the DATA_CACHE_DIR volume (where repos are indexed). + diskTotalMiB: z.number().nonnegative().nullable(), + diskFreeMiB: z.number().nonnegative().nullable(), +}); +export type SystemInfo = z.infer; + export const servicePingRequestSchema = z.object({ installId: z.string(), version: z.string(), @@ -18,6 +43,8 @@ export const servicePingRequestSchema = z.object({ isTelemetryEnabled: z.boolean(), isLanguageModelConfigured: z.boolean(), activationCode: z.string().optional(), + // optional for back-compat with Lighthouse deployments that predate it. + systemInfo: systemInfoSchema.optional(), }); export type ServicePingRequest = z.infer;