feat(cloud): Sentry, Axiom telemetry, and observability fixes

apps/cloud/package.json

@@ -11,7 +11,7 @@
     "db:generate": "drizzle-kit generate",
     "build": "vite build",
     "preview": "vite preview",
-    "deploy": "vite build && wrangler deploy",
+    "deploy": "op run --env-file=.env.production -- sh -c 'vite build && wrangler deploy'",
     "cf-typegen": "wrangler types",
     "typecheck": "tsgo --noEmit",
     "test": "vitest run",
@@ -22,6 +22,7 @@
     "@cloudflare/vite-plugin": "^1.31.1",
     "@effect-atom/atom": "^0.5.0",
     "@effect-atom/atom-react": "^0.5.0",
+    "@effect/opentelemetry": "^0.63.0",
     "@effect/platform": "catalog:",
     "@executor/api": "workspace:*",
     "@executor/env": "workspace:*",
@@ -36,6 +37,14 @@
     "@executor/sdk": "workspace:*",
     "@executor/storage-postgres": "workspace:*",
     "@modelcontextprotocol/sdk": "^1.29.0",
+    "@opentelemetry/exporter-logs-otlp-http": "^0.214.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.214.0",
+    "@opentelemetry/resources": "^2.6.1",
+    "@opentelemetry/sdk-logs": "^0.214.0",
+    "@opentelemetry/sdk-trace-base": "^2.6.1",
+    "@opentelemetry/semantic-conventions": "^1.40.0",
+    "@sentry/cloudflare": "^10.48.0",
+    "@sentry/react": "^10.48.0",
     "@tanstack/react-router": "catalog:",
     "@tanstack/react-start": "catalog:",
     "@workos-inc/node": "^8.11.1",

apps/cloud/src/api.ts

@@ -3,6 +3,7 @@
 // ---------------------------------------------------------------------------
 
 import { env } from "cloudflare:workers";
+import * as Sentry from "@sentry/cloudflare";
 import {
   HttpApiBuilder,
   HttpApiSwagger,
@@ -48,6 +49,7 @@ import { DbService } from "./services/db";
 import { createOrgExecutor } from "./services/executor";
 import { TeamOrgApi } from "./team/compose";
 import { TeamHandlers } from "./team/handlers";
+import { TelemetryLive } from "./services/telemetry";
 import { server } from "./env";
 
 // ---------------------------------------------------------------------------
@@ -73,7 +75,7 @@ const SharedServices = Layer.mergeAll(
   UserStoreLive,
   WorkOSAuth.Default,
   HttpServer.layerContext,
-);
+).pipe(Layer.provideMerge(TelemetryLive));
 const ProtectedCloudApiLive = HttpApiBuilder.api(ProtectedCloudApi).pipe(
   Layer.provide(
     Layer.mergeAll(
@@ -249,6 +251,7 @@ const handleAutumnRequest = async (request: Request): Promise<Response> => {
 
   return Effect.runPromise(program.pipe(Effect.provide(SharedServices), Effect.scoped)).catch(
     (err) => {
+      Sentry.captureException(err);
       console.error("[autumn] request failed:", err instanceof Error ? err.stack : err);
       return Response.json({ error: "Internal server error" }, { status: 500 });
     },
@@ -259,9 +262,41 @@ const handleAutumnRequest = async (request: Request): Promise<Response> => {
 // Widget token endpoint — returns a WorkOS widget token for the session user
 // ---------------------------------------------------------------------------
 
+// ---------------------------------------------------------------------------
+// Sentry tunnel — proxies client events to Sentry to bypass ad blockers
+// ---------------------------------------------------------------------------
+
+const SENTRY_HOST = "o4511196985556992.ingest.us.sentry.io";
+const SENTRY_PROJECT_IDS = new Set(["4511197001416704", "4511197010067456"]);
+
+const handleSentryTunnel = async (request: Request): Promise<Response> => {
+  try {
+    const body = await request.text();
+    const header = body.split("\n")[0];
+    const { dsn } = JSON.parse(header) as { dsn: string };
+    const url = new URL(dsn);
+    const projectId = url.pathname.replace("/", "");
+
+    if (url.host !== SENTRY_HOST || !SENTRY_PROJECT_IDS.has(projectId)) {
+      return new Response("Invalid Sentry DSN", { status: 400 });
+    }
+
+    return fetch(`https://${SENTRY_HOST}/api/${projectId}/envelope/`, {
+      method: "POST",
+      body,
+    });
+  } catch {
+    return new Response("Invalid request", { status: 400 });
+  }
+};
+
 export const handleApiRequest = async (request: Request): Promise<Response> => {
   const pathname = new URL(request.url).pathname;
 
+  if (pathname === "/sentry-tunnel" && request.method === "POST") {
+    return handleSentryTunnel(request);
+  }
+
   if (isTeamPath(pathname)) {
     const handler = createTeamHandler();
     try {
@@ -293,6 +328,30 @@ export const handleApiRequest = async (request: Request): Promise<Response> => {
       const codeExecutor = makeDynamicWorkerExecutor({ loader: env.LOADER });
       const handler = yield* buildProtectedHandler(org.id, org.name, codeExecutor);
       const response = yield* Effect.promise(() => handler.handler(request));
+
+      // Sanitize error responses — only let declared API errors through,
+      // replace anything else with a generic message to prevent leaking internals
+      if (!response.ok) {
+        const sanitized = yield* Effect.promise(async () => {
+          try {
+            const body = await response.clone().json();
+            const hasTag = body && typeof body === "object" && "_tag" in body;
+            if (!hasTag) {
+              Sentry.captureMessage(`API ${response.status}: ${pathname}`, { level: "error" });
+              return Response.json(
+                { error: "Internal server error" },
+                { status: response.status >= 500 ? 500 : response.status },
+              );
+            }
+          } catch {
+            Sentry.captureMessage(`API ${response.status}: ${pathname}`, { level: "error" });
+            return Response.json({ error: "Internal server error" }, { status: 500 });
+          }
+          return null;
+        });
+        if (sanitized) return { response: sanitized, orgId: org.id };
+      }
+
       return { response, orgId: org.id };
     });
 
@@ -325,10 +384,8 @@ export const handleApiRequest = async (request: Request): Promise<Response> => {
 
     return result.response;
   } catch (err) {
+    Sentry.captureException(err);
     console.error("[api] request failed:", err instanceof Error ? err.stack : err);
-    return Response.json(
-      { error: err instanceof Error ? err.message : "Internal server error" },
-      { status: 500 },
-    );
+    return Response.json({ error: "Internal server error" }, { status: 500 });
   }
 };

apps/cloud/src/env.ts

@@ -12,6 +12,9 @@ const serverShape = {
   WORKOS_COOKIE_PASSWORD: Env.string("WORKOS_COOKIE_PASSWORD"),
   VITE_PUBLIC_SITE_URL: Env.stringOr("VITE_PUBLIC_SITE_URL", ""),
   AUTUMN_SECRET_KEY: Env.stringOr("AUTUMN_SECRET_KEY", ""),
+  SENTRY_DSN: Env.stringOr("SENTRY_DSN", ""),
+  AXIOM_TOKEN: Env.stringOr("AXIOM_TOKEN", ""),
+  AXIOM_DATASET: Env.stringOr("AXIOM_DATASET", "executor-cloud"),
 };
 
 type SharedEnv = Readonly<{
@@ -27,6 +30,9 @@ type ServerEnv = SharedEnv &
     WORKOS_COOKIE_PASSWORD: string;
     VITE_PUBLIC_SITE_URL: string;
     AUTUMN_SECRET_KEY: string;
+    SENTRY_DSN: string;
+    AXIOM_TOKEN: string;
+    AXIOM_DATASET: string;
   }>;
 
 type WebEnv = Readonly<Record<string, never>>;

apps/cloud/src/mcp-session.ts

@@ -6,14 +6,19 @@ import { DurableObject, env } from "cloudflare:workers";
 import { Data, Effect, Layer } from "effect";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { WorkerTransport, type TransportState } from "agents/mcp";
+import { drizzle } from "drizzle-orm/postgres-js";
+import postgres from "postgres";
 
 import { createExecutorMcpServer } from "@executor/host-mcp";
 import { makeDynamicWorkerExecutor } from "@executor/runtime-dynamic-worker";
+import type { DrizzleDb } from "@executor/storage-postgres";
+import * as sharedSchema from "@executor/storage-postgres/schema";
 
 import { UserStoreService } from "./auth/context";
 import { server } from "./env";
 import { createOrgExecutor } from "./services/executor";
 import { DbService } from "./services/db";
+import * as cloudSchema from "./services/schema";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -23,47 +28,19 @@ export type McpSessionInit = {
   organizationId: string;
 };
 
-// Heartbeat interval — keeps the DO alive by re-scheduling an alarm before
-// Cloudflare's ~60s idle eviction kicks in.
 const HEARTBEAT_MS = 30 * 1000;
-
-// Session timeout — clean up after no requests for this long.
-// TODO: Make tier-based — free users get 60s, paid users get 5 minutes.
 const SESSION_TIMEOUT_MS = 5 * 60 * 1000;
 
 // ---------------------------------------------------------------------------
-// Session initialization effect
+// Errors
 // ---------------------------------------------------------------------------
 
-const DbLive = DbService.Live;
-const UserStoreLive = UserStoreService.Live.pipe(Layer.provide(DbLive));
-const Services = Layer.mergeAll(DbLive, UserStoreLive);
-
 class OrganizationNotFoundError extends Data.TaggedError("OrganizationNotFoundError")<{
   readonly organizationId: string;
 }> {}
 
-const initSession = (organizationId: string) =>
-  Effect.gen(function* () {
-    const users = yield* UserStoreService;
-    const org = yield* users.use((store) => store.getOrganization(organizationId));
-
-    if (!org) {
-      return yield* new OrganizationNotFoundError({ organizationId });
-    }
-
-    const executor = yield* createOrgExecutor(org.id, org.name, server.ENCRYPTION_KEY);
-
-    const codeExecutor = makeDynamicWorkerExecutor({ loader: env.LOADER });
-    const mcpServer = yield* Effect.promise(() =>
-      createExecutorMcpServer({ executor, codeExecutor }),
-    );
-
-    return mcpServer;
-  }).pipe(Effect.provide(Services));
-
 // ---------------------------------------------------------------------------
-// JSON-RPC error response helper
+// Helpers
 // ---------------------------------------------------------------------------
 
 const jsonRpcError = (status: number, code: number, message: string) =>
@@ -72,6 +49,32 @@ const jsonRpcError = (status: number, code: number, message: string) =>
     headers: { "content-type": "application/json" },
   });
 
+const combinedSchema = { ...sharedSchema, ...cloudSchema };
+
+/**
+ * Create a long-lived DB connection for the DO lifetime.
+ *
+ * Unlike the per-request `DbService.Live` used in `api.ts`, this connection
+ * stays open across requests within the DO. DOs have a single-threaded
+ * execution model — there's no cross-request socket reuse issue because
+ * only one request runs at a time. The connection is closed when the DO
+ * cleans up (timeout or eviction).
+ */
+const makeLongLivedDb = (): { db: DrizzleDb; end: () => Promise<void> } => {
+  const connectionString = env.HYPERDRIVE?.connectionString ?? server.DATABASE_URL;
+  const sql = postgres(connectionString, {
+    max: 1,
+    idle_timeout: 20,
+    max_lifetime: 300,
+    connect_timeout: 10,
+    onnotice: () => undefined,
+  });
+  return {
+    db: drizzle(sql, { schema: combinedSchema }) as DrizzleDb,
+    end: () => sql.end({ timeout: 0 }).catch(() => undefined),
+  };
+};
+
 // ---------------------------------------------------------------------------
 // Durable Object
 // ---------------------------------------------------------------------------
@@ -81,6 +84,7 @@ export class McpSessionDO extends DurableObject {
   private transport: WorkerTransport | null = null;
   private initialized = false;
   private lastActivityMs = 0;
+  private dbHandle: { db: DrizzleDb; end: () => Promise<void> } | null = null;
 
   private makeStorage() {
     return {
@@ -96,7 +100,25 @@ export class McpSessionDO extends DurableObject {
   async init(token: McpSessionInit): Promise<void> {
     if (this.initialized) return;
 
-    this.mcpServer = await Effect.runPromise(initSession(token.organizationId));
+    // Create a long-lived DB connection for the DO's lifetime
+    this.dbHandle = makeLongLivedDb();
+
+    const DbLive = Layer.succeed(DbService, this.dbHandle.db);
+    const UserStoreLive = UserStoreService.Live.pipe(Layer.provide(DbLive));
+    const Services = Layer.mergeAll(DbLive, UserStoreLive);
+
+    const program = Effect.gen(function* () {
+      const users = yield* UserStoreService;
+      const org = yield* users.use((store) => store.getOrganization(token.organizationId));
+      if (!org)
+        return yield* new OrganizationNotFoundError({ organizationId: token.organizationId });
+
+      const executor = yield* createOrgExecutor(org.id, org.name, server.ENCRYPTION_KEY);
+      const codeExecutor = makeDynamicWorkerExecutor({ loader: env.LOADER });
+      return yield* Effect.promise(() => createExecutorMcpServer({ executor, codeExecutor }));
+    }).pipe(Effect.provide(Services));
+
+    this.mcpServer = await Effect.runPromise(program);
 
     this.transport = new WorkerTransport({
       sessionIdGenerator: () => this.ctx.id.toString(),
@@ -143,6 +165,10 @@ export class McpSessionDO extends DurableObject {
       await this.mcpServer.close().catch(() => undefined);
       this.mcpServer = null;
     }
+    if (this.dbHandle) {
+      await this.dbHandle.end();
+      this.dbHandle = null;
+    }
     this.initialized = false;
   }
 }

apps/cloud/src/routes/__root.tsx

@@ -1,4 +1,5 @@
 import React from "react";
+import * as Sentry from "@sentry/react";
 import { HeadContent, Scripts, createRootRoute } from "@tanstack/react-router";
 import { AutumnProvider } from "autumn-js/react";
 import { ExecutorProvider } from "@executor/react/api/provider";
@@ -8,6 +9,16 @@ import { LoginPage } from "../web/pages/login";
 import { Shell } from "../web/shell";
 import appCss from "@executor/react/globals.css?url";
 
+if (typeof window !== "undefined" && import.meta.env.VITE_PUBLIC_SENTRY_DSN) {
+  Sentry.init({
+    dsn: import.meta.env.VITE_PUBLIC_SENTRY_DSN,
+    tunnel: "/api/sentry-tunnel",
+    tracesSampleRate: 0,
+    replaysSessionSampleRate: 0.1,
+    replaysOnErrorSampleRate: 1.0,
+  });
+}
+
 export const Route = createRootRoute({
   head: () => ({
     meta: [

apps/cloud/src/server.ts

@@ -1,8 +1,17 @@
+import * as Sentry from "@sentry/cloudflare";
 import handler from "@tanstack/react-start/server-entry";
 
 // Export Durable Objects as named exports
 export { McpSessionDO } from "./mcp-session";
 
-export default {
-  fetch: handler.fetch,
-};
+export default Sentry.withSentry(
+  (env: Record<string, string>) => ({
+    dsn: env.SENTRY_DSN,
+    tracesSampleRate: 0,
+    enableLogs: true,
+    sendDefaultPii: true,
+  }),
+  {
+    fetch: handler.fetch,
+  },
+);

apps/cloud/src/services/telemetry.ts

@@ -0,0 +1,35 @@
+// ---------------------------------------------------------------------------
+// OpenTelemetry setup — pipes Effect spans + logs to Axiom via OTLP
+// ---------------------------------------------------------------------------
+
+import { Layer } from "effect";
+import { WebSdk, Tracer as OtelTracer } from "@effect/opentelemetry";
+import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { server } from "../env";
+
+const makeResourceLayer = () =>
+  WebSdk.layer(() => ({
+    resource: {
+      serviceName: "executor-cloud",
+      serviceVersion: "1.0.0",
+    },
+    spanProcessor: new BatchSpanProcessor(
+      new OTLPTraceExporter({
+        url: "https://api.axiom.co/v1/traces",
+        headers: {
+          Authorization: `Bearer ${server.AXIOM_TOKEN}`,
+          "X-Axiom-Dataset": server.AXIOM_DATASET,
+        },
+      }),
+    ),
+  }));
+
+/**
+ * Full telemetry layer — provides Effect Tracer backed by OTEL → Axiom.
+ * All existing `Effect.withSpan` calls automatically become distributed traces.
+ * No-op when AXIOM_TOKEN is not set.
+ */
+export const TelemetryLive: Layer.Layer<never> = server.AXIOM_TOKEN
+  ? OtelTracer.layerGlobal.pipe(Layer.provide(makeResourceLayer()))
+  : Layer.empty;