From fbf3d0389e1a8ddf35747858a7ef2c1c1e0ecdca Mon Sep 17 00:00:00 2001 From: Nathan Walker Date: Wed, 9 Sep 2026 14:12:22 -0700 Subject: [PATCH 1/8] fix: report live video as macOS-only on Windows and Linux builds Non-macOS builds link native_stubs.c instead of the macOS native bridge, so neither iOS simulator nor Android emulator H.264 streaming can work there. The browser previously showed the raw JSON stub error in the stream area and kept retrying WebRTC offers that could never succeed. Server: - Add platform.rs as the single source of truth for the live-video capability, its user-facing reason, and the CLI note. - Report `hostOs` and a `liveVideo` block from /api/health and /api/stream-quality. - Reject WebRTC offers early with 501 and the platform explanation via a new AppError::Unsupported variant. - Print a "Live video:" note after the service URLs on non-macOS hosts. - Share one clear message across the H.264 encoder stubs; drop the unused WIP helper and the duplicated #[test] attribute. Client: - Parse `{"error": ...}` bodies from failed offer and stream-config posts. - Read `liveVideo` from the stream-quality response, pause the live stream, and show the reason in the viewport instead of the reconnect loop. - Disable the stream transport, encoder, frame rate, and resolution controls with a "Requires macOS" note when the server cannot stream. Docs: platform support table, health/REST field docs, video guide, and a troubleshooting entry for the message. --- AGENTS.md | 7 ++ README.md | 2 +- docs/api/health.md | 11 ++ docs/api/rest.md | 5 +- docs/guide/installation.md | 20 +++- docs/guide/troubleshooting.md | 14 +++ docs/guide/video.md | 12 ++ packages/client/src/api/types.ts | 13 +++ packages/client/src/app/AppShell.tsx | 41 ++++--- .../src/features/simulators/SimulatorMenu.tsx | 28 ++++- .../stream/liveVideoCapability.test.ts | 38 ++++++ .../features/stream/liveVideoCapability.ts | 21 ++++ .../stream/streamWorkerClient.test.ts | 55 +++++++++ .../src/features/stream/streamWorkerClient.ts | 39 ++++++- .../client/src/features/toolbar/Toolbar.tsx | 3 + packages/client/src/styles/components.css | 7 ++ packages/server/native_stubs.c | 20 ++-- packages/server/src/api/routes.rs | 21 ++++ packages/server/src/error.rs | 9 ++ packages/server/src/main.rs | 8 ++ packages/server/src/platform.rs | 109 ++++++++++++++++++ packages/server/src/transport/webrtc.rs | 9 ++ 22 files changed, 459 insertions(+), 33 deletions(-) create mode 100644 packages/client/src/features/stream/liveVideoCapability.test.ts create mode 100644 packages/client/src/features/stream/liveVideoCapability.ts create mode 100644 packages/server/src/platform.rs diff --git a/AGENTS.md b/AGENTS.md index a8facf46..f5150a54 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,13 @@ The native side should own anything that depends on macOS frameworks, `xcrun sim Defines REST routes for simulator control, health, metrics, and chrome assets. - `packages/server/src/transport/webrtc.rs` Exposes the H.264 WebRTC offer/answer endpoint for browser live video. +- `packages/server/src/platform.rs` + Single source of truth for host platform capabilities. Live H.264 video + (iOS simulator and Android emulator) exists only in the macOS build; the + Windows and Linux builds compile `packages/server/native_stubs.c` in place + of the native bridge. Health, stream-quality, the WebRTC offer error, the + CLI banner, and the browser client all read this module's `liveVideo` + capability instead of hard-coding platform checks. - `packages/server/src/webkit.rs` Discovers simulator WebKit Remote Inspector targets and bridges WebInspectorUI WebSocket traffic to the simulator `webinspectord` binary-plist socket. diff --git a/README.md b/README.md index ec6cc2f4..8d96a25a 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ view inside the editor. ## Features -- Supports native H.264 streaming for both iOS simulators and Android emulators +- Supports native H.264 streaming for both iOS simulators and Android emulators (live video requires the macOS build; the Windows and Linux CLIs manage Android emulators without a browser stream) - Full simulator control & inspection using private iOS accessibility APIs and Android UIAutomator - available using `simdeck` CLI - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - Profiling built-in: CPU, memory, disk writes, network throughput, hang signals, and stack sampling diff --git a/docs/api/health.md b/docs/api/health.md index 6bfdd6c3..745dadd8 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -21,6 +21,8 @@ Example: "serverKind": "launchAgent", "timestamp": 1714094761.234, "videoCodec": "auto", + "hostOs": "macos", + "liveVideo": { "supported": true, "requires": "macos" }, "androidGpu": "host", "lowLatency": false, "realtimeStream": true, @@ -47,10 +49,19 @@ Important fields: | `httpPort` | Port serving UI and API | | `serverKind` | `launchAgent` or `standalone` | | `videoCodec` | Requested codec mode: `auto`, `hardware`, or `software` | +| `hostOs` | Server build target: `macos`, `windows`, or `linux` | +| `liveVideo` | Whether this build can encode the live H.264 stream | | `androidGpu` | Android emulator renderer mode for SimDeck-owned boots | | `streamQuality` | Active stream profile and limits | | `webRtc` | ICE settings the browser should use | +`liveVideo` is `{ "supported": true, "requires": "macos" }` on macOS. Windows +and Linux builds return `supported: false` plus a `reason` string, because live +H.264 encoding for both iOS simulators and Android emulators lives in the macOS +native bridge. Clients should show that reason instead of opening a WebRTC +offer; the offer endpoint answers `501 Not Implemented` with the same message. +`GET /api/stream-quality` includes the same `liveVideo` block. + When auth is required, the `401` JSON body still includes `serverId`, `advertiseHost`, `hostId`, `hostName`, `httpPort`, and `serverKind` so native clients can group endpoints before pairing. ## Metrics diff --git a/docs/api/rest.md b/docs/api/rest.md index 16f76f22..d3319b33 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -52,7 +52,10 @@ curl -X POST \ | `GET` | `/api/stream-quality` | Current stream quality settings | | `POST` | `/api/stream-quality` | Update stream quality settings | -See [Health and metrics](/api/health) for details. +See [Health and metrics](/api/health) for details. Both `/api/health` and +`/api/stream-quality` include a `liveVideo` block; check `liveVideo.supported` +before opening a WebRTC offer, because Windows and Linux builds cannot encode +the live stream and answer offers with `501` and the `liveVideo.reason` text. ## Devices diff --git a/docs/guide/installation.md b/docs/guide/installation.md index e8d973ab..33d84401 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -2,10 +2,28 @@ ## Requirements -- macOS on Apple Silicon. +- macOS on Apple Silicon for the full experience. - Xcode with the simulator runtimes you want to use. - Node.js 18 or newer. +## Platform support + +The npm package installs a native CLI for macOS, Windows, and Linux, but only +the macOS build includes the native simulator bridge and H.264 encoder. + +| Capability | macOS | Windows / Linux | +| -------------------------------------------------- | ----- | ----------------------- | +| iOS simulator control, inspection, and streaming | Yes | No | +| Android emulator control and inspection | Yes | Yes | +| Live H.264 browser stream (iOS and Android) | Yes | No | +| `--video-codec`, stream quality, and encoder menus | Yes | Reported as unavailable | + +On Windows and Linux the CLI prints a `Live video:` note after the service +URLs, `GET /api/health` and `GET /api/stream-quality` report +`liveVideo.supported: false` with the reason, the browser shows that reason in +place of the device screen, and the WebRTC offer endpoint answers `501` with +the same message. See [Video and streaming](./video.md) for the encoder details. + Check Xcode selection if you have multiple installs: ```sh diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 7ad04818..e34b2869 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -107,6 +107,20 @@ Android IDs in SimDeck use `android:`. ## Stream is black or stuck +### Live video requires macOS + +```text +Live H.264 video streaming requires macOS. This SimDeck build for Windows can manage devices but does not include the native H.264 encoder... +``` + +This is expected on the Windows and Linux builds, for both iOS simulators and +Android emulators. Those builds link a stub instead of the macOS native bridge +that owns VideoToolbox and x264 encoding, so the browser stream, the encoder +menu, and `--video-codec` cannot work there. Device control, `describe`, +screenshots, and the inspectors still run. Use a Mac to see the live screen, or +pair a Windows or Linux browser with a SimDeck service running on a Mac. Check +`liveVideo.supported` in `GET /api/health` when scripting against the service. + ### Timed out waiting for the first frame Try software encoding: diff --git a/docs/guide/video.md b/docs/guide/video.md index d7318675..83fa78d0 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -5,6 +5,18 @@ SimDeck streams live device video to the browser. Local iOS sessions default to iOS simulator H.264 uses VideoToolbox for hardware encoding and x264 for software encoding. Android emulator H.264 uses the emulator gRPC `streamScreenshot` API when SimDeck owns the boot. SimDeck receives raw RGBA frames, pads odd dimensions for H.264, and encodes them on the Mac. If the gRPC endpoint is unavailable, SimDeck falls back to the emulator `-share-vid` display surface and reads BGRA frames from the `videmulator` shared memory region. +## Platform requirement + +Live video needs the macOS build. Both encoders above live in the macOS native +bridge, so the Windows and Linux CLIs link a stub instead and cannot stream iOS +simulators or Android emulators to the browser. Those builds still manage +Android emulators, but `--video-codec`, stream quality, and the encoder menu +have no effect there. The service reports this through `liveVideo` in +`GET /api/health` and `GET /api/stream-quality`, prints a `Live video:` note +when it starts, and answers WebRTC offers with `501 Not Implemented` and the +same explanation. The browser client hides the retry loop and shows that +message in place of the device screen. + ## When encoding runs SimDeck starts encoding when a browser stream needs H.264 frames. For iOS, the diff --git a/packages/client/src/api/types.ts b/packages/client/src/api/types.ts index 0b03eaf4..a503812e 100644 --- a/packages/client/src/api/types.ts +++ b/packages/client/src/api/types.ts @@ -187,12 +187,25 @@ export interface ChromeDevToolsTargetDiscovery { warnings: string[]; } +/** + * Whether the server build can encode the live H.264 browser stream. Only the + * macOS build links the native encoder; Windows and Linux builds report + * `supported: false` with a user-facing `reason`. + */ +export interface LiveVideoCapability { + supported: boolean; + requires?: string; + reason?: string | null; +} + export interface HealthResponse { ok: boolean; serverId?: string; advertiseHost?: string; hostId?: string; hostName?: string; + hostOs?: string; + liveVideo?: LiveVideoCapability; httpPort?: number; serverKind?: | "launchAgent" diff --git a/packages/client/src/app/AppShell.tsx b/packages/client/src/app/AppShell.tsx index b8e97209..75c6d544 100644 --- a/packages/client/src/app/AppShell.tsx +++ b/packages/client/src/app/AppShell.tsx @@ -38,6 +38,7 @@ import type { AccessibilitySourcePreference, AccessibilityTreeResponse, ChromeProfile, + LiveVideoCapability, SimulatorMetadata, SimulatorStateResponse, TouchPhase, @@ -55,6 +56,7 @@ import { simulatorUsesInsetChromeButtons, } from "../features/simulators/simulatorDisplay"; import { useSimulatorList } from "../features/simulators/useSimulatorList"; +import { liveVideoUnavailableReason } from "../features/stream/liveVideoCapability"; import { sendWebRtcControlMessage } from "../features/stream/streamWorkerClient"; import type { StreamConfig, @@ -174,6 +176,7 @@ clearLegacyVolatileUiState(); interface StreamQualityResponse { ok?: boolean; + liveVideo?: LiveVideoCapability; quality?: { fps?: number; maxEdge?: number; @@ -413,9 +416,9 @@ function simulatorDisplayReady(simulator: SimulatorMetadata): boolean { const display = simulator.privateDisplay; return Boolean( simulator.isBooted && - display?.displayReady && - display.displayWidth > 0 && - display.displayHeight > 0, + display?.displayReady && + display.displayWidth > 0 && + display.displayHeight > 0, ); } @@ -625,6 +628,10 @@ export function AppShell({ ); const [streamConfigApplyKey, setStreamConfigApplyKey] = useState(0); const [streamConfigReady, setStreamConfigReady] = useState(false); + // Non-empty when the connected server build cannot encode live video (for + // example the Windows or Linux CLI). The stream stays paused and the reason + // is shown instead of retrying WebRTC offers that can never succeed. + const [liveVideoUnavailable, setLiveVideoUnavailable] = useState(""); const [touchIndicators, setTouchIndicators] = useState([]); const [selectedSimulatorState, setSelectedSimulatorState] = useState(null); @@ -847,6 +854,7 @@ export function AppShell({ if (requestId !== streamConfigRequestIdRef.current) { return; } + setLiveVideoUnavailable(liveVideoUnavailableReason(response.liveVideo)); if ( !options?.ignoreUserGrace && Date.now() - streamConfigUserChangeAtRef.current < @@ -930,7 +938,7 @@ export function AppShell({ streamCanvasKey, } = useLiveStream({ canvasElement: streamCanvasElement, - paused: !streamConfigReady, + paused: !streamConfigReady || Boolean(liveVideoUnavailable), remote: remoteStream, simulator: selectedSimulator, streamConfig: effectiveStreamConfig, @@ -1035,8 +1043,8 @@ export function AppShell({ selectedSimulator != null && shouldRenderNativeChrome(selectedSimulator); const deviceChromeToggleActive = Boolean( selectedSupportsChrome && - deviceChromeVisible && - !selectedChromeAssetsFailed, + deviceChromeVisible && + !selectedChromeAssetsFailed, ); const shouldRenderChrome = deviceChromeToggleActive; const viewportChromeProfile = shouldRenderChrome ? chromeProfile : null; @@ -1127,8 +1135,8 @@ export function AppShell({ : recordingOverlayLabel || captureStatus?.label || ""; const captureOverlayBusy = Boolean( isInstallingApp || - captureStatus?.busy || - screenRecording?.phase === "stopping", + captureStatus?.busy || + screenRecording?.phase === "stopping", ); const autoViewportOffsetY = viewMode === "manual" ? 0 : -zoomDockReservedHeight / 2; @@ -2153,13 +2161,15 @@ export function AppShell({ const viewportStatusOverlayLabel = (providerDisconnected ? NOT_CONNECTED_MESSAGE : "") || simulatorStatusOverlayLabel || + liveVideoUnavailable || streamStatusMessage || (selectedSimulator ? visibleListError : ""); const viewportHasStreamError = Boolean( providerDisconnected || - streamStatus.state === "error" || - visibleStreamError || - (selectedSimulator && visibleListError), + liveVideoUnavailable || + streamStatus.state === "error" || + visibleStreamError || + (selectedSimulator && visibleListError), ); const deviceTransform = `translate(${pan.x}px, ${pan.y + autoViewportOffsetY}px) scale(${effectiveZoom})`; const chromeScreenRect = computeChromeScreenRect( @@ -3812,6 +3822,7 @@ export function AppShell({ recordingActive={screenRecording?.phase === "recording"} recordingStopping={screenRecording?.phase === "stopping"} remoteStream={remoteStream} + liveVideoUnavailableReason={liveVideoUnavailable} search={search} selectedSimulator={selectedSimulator} selectedSimulatorIdentifier={selectedSimulatorDetail} @@ -3821,8 +3832,8 @@ export function AppShell({ }} showBootButton={Boolean( selectedSimulator && - !selectedSimulator.isBooted && - !selectedSimulatorTransitionKind, + !selectedSimulator.isBooted && + !selectedSimulatorTransitionKind, )} streamConfig={effectiveStreamConfig} streamTransport={streamTransport} @@ -4325,8 +4336,8 @@ function normalizeMaxEdge( function isAndroidSimulator(simulator: SimulatorMetadata | null): boolean { return Boolean( simulator?.platform === "android-emulator" || - simulator?.deviceTypeIdentifier === "android-emulator" || - simulator?.udid.startsWith("android:"), + simulator?.deviceTypeIdentifier === "android-emulator" || + simulator?.udid.startsWith("android:"), ); } diff --git a/packages/client/src/features/simulators/SimulatorMenu.tsx b/packages/client/src/features/simulators/SimulatorMenu.tsx index c9c1cc9f..94bc815e 100644 --- a/packages/client/src/features/simulators/SimulatorMenu.tsx +++ b/packages/client/src/features/simulators/SimulatorMenu.tsx @@ -49,6 +49,8 @@ interface SimulatorMenuProps { recordingActive: boolean; recordingStopping: boolean; remoteStream?: boolean; + /** Set when the server build cannot stream video; disables stream controls. */ + liveVideoUnavailableReason?: string; selectedSimulator: SimulatorMetadata | null; showBootButton: boolean; showStopButton: boolean; @@ -94,6 +96,7 @@ export function SimulatorMenu({ recordingActive, recordingStopping, remoteStream = false, + liveVideoUnavailableReason = "", selectedSimulator, showBootButton, showStopButton, @@ -123,6 +126,7 @@ export function SimulatorMenu({ const canRotateSelectedSimulator = selectedSimulator != null && !simulatorHasFixedOrientation(selectedSimulator); + const streamControlsDisabled = liveVideoUnavailableReason !== ""; return (
+ {streamControlsDisabled ? ( +

+ {LIVE_VIDEO_UNAVAILABLE_NOTE} +

+ ) : null}