diff --git a/examples/README.md b/examples/README.md index 7a78f3b5cc..ead2c20f25 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,7 +2,7 @@ Runnable, typechecked Node.js examples for the `agent-device` SDK surface exposed to Node consumers. Source of truth for the API itself is -[Typed Client](../website/docs/docs/client-api.md). Two guards keep these files in sync with that +[Node.js API](../website/docs/docs/client-api.md). Two guards keep these files in sync with that doc: `src/__tests__/client-api-examples-drift.test.ts` checks the doc's subpath API manifest against what these examples import, and `test/integration/client-api-doc-snippets.test.ts` compiles every fenced TypeScript code block in the doc itself against the real `agent-device/*` sources. diff --git a/src/__tests__/client-api-examples-drift.test.ts b/src/__tests__/client-api-examples-drift.test.ts index 61218967d6..183606dcc2 100644 --- a/src/__tests__/client-api-examples-drift.test.ts +++ b/src/__tests__/client-api-examples-drift.test.ts @@ -24,6 +24,7 @@ import { describe, test } from 'vitest'; const CLIENT_API_DOC_PATH = 'website/docs/docs/client-api.md'; const EXAMPLES_SDK_DIR = 'examples/sdk'; +const PACKAGE_JSON_PATH = 'package.json'; type SubpathManifest = ReadonlyMap>; @@ -37,7 +38,7 @@ const REQUIRED_EXAMPLE_SYMBOLS: readonly { subpath: string; symbol: string }[] = { subpath: 'agent-device/batch', symbol: 'runBatch' }, ]; -// Parses the "Public subpath API" bullet list: a top-level `- \`agent-device...\`` +// Parses the "API reference" bullet list: a top-level `- \`agent-device...\`` // bullet starts a subpath section; backtick-quoted identifiers on its nested // bullet lines (stripping a trailing `(...)` call signature) are that // subpath's documented symbols, until the next top-level bullet. @@ -100,6 +101,12 @@ function listExampleFiles(dir: string): string[] { const manifest = parseSubpathManifest(fs.readFileSync(CLIENT_API_DOC_PATH, 'utf8')); const exampleFiles = listExampleFiles(EXAMPLES_SDK_DIR); +const packageExports = Object.keys( + (JSON.parse(fs.readFileSync(PACKAGE_JSON_PATH, 'utf8')) as { exports?: Record }) + .exports ?? {}, +) + .map((subpath) => (subpath === '.' ? 'agent-device' : `agent-device${subpath.slice(1)}`)) + .sort(); const importsByFile = new Map( exampleFiles.map((file) => [file, extractImportedSymbols(fs.readFileSync(file, 'utf8'))]), ); @@ -109,7 +116,15 @@ describe('examples/sdk vs client-api.md drift guard', () => { assert.ok( manifest.size > 0, `${CLIENT_API_DOC_PATH} did not yield a parseable subpath API manifest; ` + - 'has the "Public subpath API exposed for Node consumers" list moved or changed format?', + 'has the "API reference" entry-point list moved or changed format?', + ); + }); + + test('client-api.md documents every published package entry point', () => { + assert.deepEqual( + [...manifest.keys()].sort(), + packageExports, + `${CLIENT_API_DOC_PATH}'s public subpath manifest must match ${PACKAGE_JSON_PATH}#exports`, ); }); diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index 1317ac49a6..b950d0cd5a 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -1,97 +1,130 @@ [ { - "name": "introduction", - "type": "file", - "label": "Introduction" - }, - { - "name": "installation", - "type": "file", - "label": "Installation" - }, - { - "name": "agent-setup", - "type": "file", - "label": "AI Agent Setup" - }, - { - "name": "quick-start", - "type": "file", - "label": "Quick Start" - }, - { - "name": "debugging-profiling", - "type": "file", - "label": "Debugging & Profiling" - }, - { - "name": "client-api", - "type": "file", - "label": "Typed Client" - }, - { - "name": "commands", - "type": "file", - "label": "Commands" - }, - { - "name": "configuration", - "type": "file", - "label": "Configuration" - }, - { - "name": "remote-proxy", - "type": "file", - "label": "Remote Proxy" - }, - { - "name": "device-clouds", - "type": "file", - "label": "Device Clouds" - }, - { - "name": "security-trust", - "type": "file", - "label": "Security & Trust" - }, - { - "name": "batching", - "type": "file", - "label": "Batching" - }, - { - "name": "selectors", - "type": "file", - "label": "Selectors" - }, - { - "name": "sessions", - "type": "file", - "label": "Sessions" - }, - { - "name": "replay-e2e", - "type": "file", - "label": "Replay & E2E" - }, - { - "name": "snapshots", - "type": "file", - "label": "Snapshots" - }, - { - "name": "ios-snapshot-elements", - "type": "file", - "label": "iOS Snapshot Elements" - }, - { - "name": "known-limitations", - "type": "file", - "label": "Known Limitations" - }, - { - "name": "migrating-gestures", - "type": "file", - "label": "Migrating Gestures" + "type": "custom-link", + "label": "Getting Started", + "collapsible": false, + "items": [ + { + "type": "custom-link", + "label": "Introduction", + "link": "/docs/introduction" + }, + { + "type": "custom-link", + "label": "Installation", + "link": "/docs/installation" + }, + { + "type": "custom-link", + "label": "AI Agent Setup", + "link": "/docs/agent-setup" + }, + { + "type": "custom-link", + "label": "Quick Start", + "link": "/docs/quick-start" + } + ] + }, + { + "type": "custom-link", + "label": "Using agent-device", + "collapsible": false, + "items": [ + { + "type": "custom-link", + "label": "Commands", + "link": "/docs/commands" + }, + { + "type": "custom-link", + "label": "Configuration", + "link": "/docs/configuration" + }, + { + "type": "custom-link", + "label": "Selectors", + "link": "/docs/selectors" + }, + { + "type": "custom-link", + "label": "Sessions", + "link": "/docs/sessions" + }, + { + "type": "custom-link", + "label": "Batching", + "link": "/docs/batching" + }, + { + "type": "custom-link", + "label": "Snapshots", + "link": "/docs/snapshots" + }, + { + "type": "custom-link", + "label": "Replay & E2E testing", + "link": "/docs/replay-e2e" + }, + { + "type": "custom-link", + "label": "Debugging & Profiling", + "link": "/docs/debugging-profiling" + } + ] + }, + { + "type": "custom-link", + "label": "Integrations", + "collapsible": false, + "items": [ + { + "type": "custom-link", + "label": "Node.js API", + "link": "/docs/client-api" + }, + { + "type": "custom-link", + "label": "AI SDK", + "link": "/docs/ai-sdk" + }, + { + "type": "custom-link", + "label": "Eve", + "link": "/docs/eve" + }, + { + "type": "custom-link", + "label": "Remote Proxy", + "link": "/docs/remote-proxy" + }, + { + "type": "custom-link", + "label": "Device Clouds", + "link": "/docs/device-clouds" + } + ] + }, + { + "type": "custom-link", + "label": "Reference", + "collapsible": false, + "items": [ + { + "type": "custom-link", + "label": "Security & Trust", + "link": "/docs/security-trust" + }, + { + "type": "custom-link", + "label": "Known Limitations", + "link": "/docs/known-limitations" + }, + { + "type": "custom-link", + "label": "Migrating Gestures", + "link": "/docs/migrating-gestures" + } + ] } ] diff --git a/website/docs/docs/ai-sdk.md b/website/docs/docs/ai-sdk.md new file mode 100644 index 0000000000..736c2c91f8 --- /dev/null +++ b/website/docs/docs/ai-sdk.md @@ -0,0 +1,78 @@ +--- +title: AI SDK +--- + +# AI SDK + +[Vercel's AI SDK](https://ai-sdk.dev/) can expose `agent-device` client methods as typed tools in a Node.js agent. `ToolLoopAgent` owns the model loop, while `createAgentDeviceClient()` owns the device session and keeps tool implementations aligned with the CLI command contracts. + +Install the integration dependencies: + +```bash +pnpm add agent-device ai zod +``` + +The example below gives the model two deliberately small tools: one for observing the current UI and one for pressing an element returned by that observation. + +```ts +import { ToolLoopAgent, tool } from 'ai'; +import { createAgentDeviceClient } from 'agent-device'; +import { z } from 'zod'; + +const client = createAgentDeviceClient({ + session: 'ai-sdk-agent', + lockPolicy: 'reject', +}); + +const agent = new ToolLoopAgent({ + model: process.env.AI_MODEL!, + instructions: [ + 'Inspect the current UI before acting.', + 'Only press an element ref returned by the latest snapshot.', + 'Stop and explain when the requested state cannot be verified.', + ].join('\n'), + tools: { + snapshot: tool({ + description: 'Return the interactive elements in the current device UI.', + inputSchema: z.object({}), + execute: async () => await client.capture.snapshot({ interactiveOnly: true }), + }), + press: tool({ + description: 'Press an element from the latest snapshot by its @e ref.', + inputSchema: z.object({ + ref: z.string().regex(/^@e\d+$/), + }), + execute: async ({ ref }) => await client.interactions.press({ ref }), + }), + }, +}); + +try { + await client.apps.open({ + app: 'com.example.app', + platform: 'ios', + }); + + const result = await agent.generate({ + prompt: 'Navigate to Notifications and verify that notifications are enabled.', + }); + + console.log(result.text); +} finally { + await client.sessions.close(); +} +``` + +Set `AI_MODEL` to a model available through your configured AI SDK provider. See the AI SDK references for [`ToolLoopAgent`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool-loop-agent) and [`tool()`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool). + +## Designing device tools + +Prefer focused tools over a single tool that accepts an arbitrary command name and arguments: + +- Validate tool input with a schema. In particular, constrain element refs to values such as `@e12` and make the model observe before acting. +- Keep the `agent-device` client outside tool execution so calls share one named session. +- Return typed client results directly unless the result needs an application-specific projection. +- Let the host application own session cleanup with `try`/`finally`; do not rely on the model to close the session. +- Use AI SDK's `needsApproval` option for actions that require human confirmation in your product. + +See [Node.js API](/docs/client-api) for the complete client surface and runnable, typechecked `agent-device` examples. diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index c543f858f3..dc383644cd 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -1,23 +1,44 @@ --- -title: Typed Client +title: Node.js API --- -# Typed Client +# Node.js API -Use `createAgentDeviceClient()` when you want to drive the daemon from application code instead of shelling out to the CLI. +Use `createAgentDeviceClient()` to give a Node.js agent typed access to device automation instead of shelling out to the CLI. Its methods can be exposed as model tools, called from deterministic orchestration code, or combined with another Node.js agent framework. -For remote Metro-backed flows, import the reusable Node APIs instead of spawning the `agent-device` binary. The CLI uses the same helpers internally. +Start with the [AI SDK](/docs/ai-sdk) or [Eve](/docs/eve) integration guide for complete tool-calling examples. The client is framework-neutral, so the same pattern works with other solutions that accept JavaScript or TypeScript functions as tools. -Public subpath API exposed for Node consumers: +## Runnable examples + +The repository includes [runnable, typechecked Node.js examples](https://github.com/callstack/agent-device/tree/main/examples/sdk) that import the same published `agent-device/*` entry points used by consumers: + +| Example | Demonstrates | +| --- | --- | +| [`client-session.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/client-session.ts) | Open, snapshot, interact, handle typed errors, and always close the session | +| [`contracts-result.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/contracts-result.ts) | Consume snapshot results with helpers from `agent-device/contracts` | +| [`batch-orchestration.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/batch-orchestration.ts) | Run a batch through a custom transport | +| [`metro-runtime.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/metro-runtime.ts) | Normalize a Metro URL and resolve runtime transport hints | + +The examples are checked against the source SDK using their dedicated [`tsconfig.json`](https://github.com/callstack/agent-device/blob/main/examples/sdk/tsconfig.json). After building the package with `pnpm build`, run an example directly with Node: + +```bash +node --experimental-strip-types examples/sdk/client-session.ts +``` + +## API reference + +Supported public entry points for Node consumers: - `agent-device` - `createAgentDeviceClient(options?)` - `createLocalArtifactAdapter(options?)` - `AppError`, `isAgentDeviceError(error)`, `normalizeAgentDeviceError(error)` - `centerOfRect(rect)` - - root types are limited to the typed client contracts used by hosted adapters, such as `AppListOptions`, `BackCommandOptions`, `ScrollOptions`, and command result types. - `agent-device/io` - - artifact adapter types, file input refs, and file output refs + - `createLocalArtifactAdapter(options?)` + - types: `ArtifactAdapter`, `ArtifactDescriptor`, `CreateTempFileOptions`, `FileInputRef`, + `FileOutputRef`, `LocalArtifactAdapterOptions`, `OutputVisibility`, `ReserveOutputOptions`, + `ReservedOutputFile`, `ResolveInputOptions`, `ResolvedInputFile`, `TemporaryFile` - `agent-device/metro` - `buildBundleUrl(baseUrl, platform)` - `normalizeBaseUrl(baseUrl)` @@ -48,6 +69,7 @@ Public subpath API exposed for Node consumers: - `parseFindArgs(args)` - types: `FindMatchOptions` - `agent-device/install-source` + - `ARCHIVE_EXTENSIONS` - `isTrustedInstallSourceUrl(sourceUrl)` - `validateDownloadSourceUrl(url)` - types: `MaterializeInstallSource` @@ -68,46 +90,16 @@ Public subpath API exposed for Node consumers: - `new LimrunRuntime(options)` - `runtime.getDeviceSession(device)` - types: `LimrunRuntimeOptions`, `LimrunDeviceSession`, `LimrunAndroidDeviceSession`, - `LimrunIosDeviceSession`, `AndroidAdbProvider`, `AndroidKeyboardState`, - `AndroidKeyboardDismissResult` - -The `contracts`, `selectors`, `finders`, `install-source`, `android-adb`, `limrun`, `artifacts`, `batch`, `metro`, `remote-config`, and `io` subpaths are the supported Node entry points. The former compatibility subpaths `agent-device/android-apps` and `agent-device/daemon`, plus hosted-runtime subpaths `agent-device/cloud-webdriver`, `agent-device/commands`, `agent-device/backend`, `agent-device/testing/conformance`, and `agent-device/observability`, are not published. + `LimrunIosDeviceSession`, `LimrunForegroundApp`, `LimrunInstalledApp`, + `LimrunIosCommandExecution`, `LimrunIosCommandResult`, `LimrunRecordingQuality`, + `LimrunIosRemoteInstallOptions`, `LimrunIosRemoteInstallResult`, `AndroidAdbProvider`, + `AndroidKeyboardState`, `AndroidKeyboardDismissResult` ## Basic usage -```ts -import { createAgentDeviceClient } from 'agent-device'; - -const client = createAgentDeviceClient({ - session: 'qa-ios', - lockPolicy: 'reject', - lockPlatform: 'ios', -}); +The canonical client example is embedded below. It is also runnable from [`examples/sdk/client-session.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/client-session.ts). -const devices = await client.devices.list({ platform: 'ios' }); -const capabilities = await client.devices.capabilities({ platform: 'ios' }); -const apps = await client.apps.list({ platform: 'ios' }); -const device = devices.find((candidate) => candidate.name === 'iPhone 16') ?? devices[0]; -if (!device) { - throw new Error('No iOS device available'); -} -if (!capabilities.availableCommands.includes('snapshot')) { - throw new Error('Selected target does not support snapshots'); -} - -await client.apps.open({ - app: 'com.apple.Preferences', - platform: 'ios', - udid: device.id, - runtime: { - metroHost: '127.0.0.1', - metroPort: 8081, - }, -}); - -const snapshot = await client.capture.snapshot({ interactiveOnly: true }); - -await client.sessions.close(); +```ts file="/../examples/sdk/client-session.ts" ``` `client.devices.capabilities()` returns `{ device, availableCommands }`, using the same capability matrix as the CLI. Use it when a dynamic integration needs to decide which command names are valid for the selected target. @@ -267,6 +259,7 @@ Vega OS client support is currently VVD-only and covers device discovery, app op Supported command methods: - `wait` +- `alert` - `appState` - `back` - `home` @@ -275,20 +268,27 @@ Supported command methods: - `keyboard` - `clipboard` - `tvRemote` -- `alert` - -Additional CLI-backed methods are exposed on their domain groups with typed option objects so Node consumers do not need to build raw daemon requests: - -- `client.devices.boot()` -- `client.devices.capabilities()` -- `client.devices.shutdown()` -- `client.apps.push()` -- `client.apps.triggerEvent()` -- `client.capture.diff()` -- `client.interactions.click()`, `press()`, `longPress()`, `swipe()`, `pan()`, `fling()`, `focus()`, `type()`, `fill()`, `scroll()`, `pinch()`, `rotateGesture()`, `transformGesture()`, `get()`, `is()`, `find()` +- `reactNative` +- `doctor` +- `prepare` +- `viewport` + +The deprecated `rotate()` alias remains available for compatibility; use `orientation()` in new integrations. + +The complete domain-client method map is: + +- `client.devices.list()`, `capabilities()`, `boot()`, `shutdown()` +- `client.sessions.list()`, `stateDir()`, `close()`, `saveScript()`, `artifacts()` +- `client.apps.install()`, `reinstall()`, `installFromSource()`, `list()`, `open()`, `close()`, `push()`, `triggerEvent()` +- `client.materializations.release()` +- `client.leases.allocate()`, `heartbeat()`, `release()` +- `client.metro.prepare()`, `reload()` +- `client.capture.snapshot()`, `screenshot()`, `diff()` +- `client.interactions.click()`, `press()`, `longPress()`, `swipe()`, `pan()`, `fling()`, `swipeGesture()`, `focus()`, `type()`, `fill()`, `scroll()`, `pinch()`, `rotateGesture()`, `transformGesture()`, `get()`, `is()`, `find()` - `client.replay.run()` and `client.replay.test()` - `client.batch.run()` - `client.observability.perf()`, `logs()`, `events()`, `network()`, and `audio()` +- `client.debug.symbols()` - `client.recording.record()` and `client.recording.trace()` - `client.settings.update()` @@ -331,21 +331,9 @@ executor. Use `agent-device/batch` when a bridge or in-process runner receives daemon-shaped requests but owns command dispatch itself. The helper keeps validation, inherited flags, serial execution, partial results, and error envelopes aligned with the daemon batch command. -```ts -import { runBatch } from 'agent-device/batch'; -import type { DaemonResponse } from 'agent-device/contracts'; - -type BatchRequest = Parameters[0]; - -async function handleBatch(req: BatchRequest): Promise { - return await runBatch(req, req.session ?? 'default', async (stepReq) => { - try { - return { ok: true, data: await dispatch(stepReq) }; - } catch (error) { - return bridgeErrorToDaemonResponse(error); - } - }); -} +The standalone custom-transport example is embedded below from [`examples/sdk/batch-orchestration.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/batch-orchestration.ts). + +```ts file="/../examples/sdk/batch-orchestration.ts" ``` ## Android `installFromSource()` diff --git a/website/docs/docs/debugging-profiling.md b/website/docs/docs/debugging-profiling.md index 0af55a43cb..053411c1e8 100644 --- a/website/docs/docs/debugging-profiling.md +++ b/website/docs/docs/debugging-profiling.md @@ -192,5 +192,5 @@ agent-device perf trace stop --kind perfetto --out app.perfetto-trace ## Where to go deeper - Full command reference: [Commands](/docs/commands) -- Typed client observability APIs: [Typed Client](/docs/client-api) +- Node.js observability APIs: [Node.js API](/docs/client-api) - Session behavior and lifecycle: [Sessions](/docs/sessions) diff --git a/website/docs/docs/eve.md b/website/docs/docs/eve.md new file mode 100644 index 0000000000..5e6138ba9f --- /dev/null +++ b/website/docs/docs/eve.md @@ -0,0 +1,83 @@ +--- +title: Eve +--- + +# Eve + +[Eve](https://eve.dev/) is Vercel's filesystem-first framework for durable agents. Files under `agent/tools/` become typed model tools, which makes them a natural place to adapt the `agent-device` Node.js client. + +Create an Eve project, then add `agent-device`: + +```bash +npx eve@latest init mobile-agent +cd mobile-agent +pnpm add agent-device +``` + +Add `agent/tools/agent_device.ts`: + +```ts +import { createAgentDeviceClient } from 'agent-device'; +import { defineTool } from 'eve/tools'; +import { z } from 'zod'; + +const client = createAgentDeviceClient({ + session: 'eve-agent', + lockPolicy: 'reject', +}); + +export default defineTool({ + description: 'Inspect or interact with the current device UI.', + inputSchema: z.discriminatedUnion('action', [ + z.object({ + action: z.literal('open'), + app: z.string().min(1), + platform: z.enum(['ios', 'android']), + }), + z.object({ + action: z.literal('snapshot'), + }), + z.object({ + action: z.literal('press'), + ref: z.string().regex(/^@e\d+$/), + }), + z.object({ + action: z.literal('close'), + }), + ]), + async execute(input) { + switch (input.action) { + case 'open': + return await client.apps.open({ app: input.app, platform: input.platform }); + case 'snapshot': + return await client.capture.snapshot({ interactiveOnly: true }); + case 'press': + return await client.interactions.press({ ref: input.ref }); + case 'close': + return await client.sessions.close(); + } + }, +}); +``` + +Eve discovers the file automatically; no tool registry is required. Tell the agent how to use it in `agent/instructions.md`: + +```md +Use the agent_device tool to inspect and operate the app. + +- Open the requested app before inspecting it. +- Call snapshot before every interaction. +- Only press an @e ref from the latest snapshot. +- Verify the requested outcome with another snapshot. +- Call close when the device task is complete. +``` + +For CI or another short-lived host, the outer runner should still close the named `agent-device` session in its own cleanup path. Model-directed `close` is useful during the normal tool loop, but it is not a replacement for deterministic cleanup after errors or cancellation. + +## Runtime placement + +Run the tool in Eve's app runtime when it needs local access to simulators, emulators, platform tooling, and daemon state. If Eve is hosted separately from the devices, connect through an [agent-device remote proxy](/docs/remote-proxy) instead. + +Eve is currently beta, so check its [current documentation](https://eve.dev/) when upgrading. For a production example, read [Building Mobile QA Agents With Vercel Eve](https://www.callstack.com/blog/building-reviewable-mobile-qa-agents-with-vercel-eve), which covers a PR QA agent using an `agent_device` tool and a deterministic CI runner. + +See [Node.js API](/docs/client-api) for the complete client surface. diff --git a/website/docs/docs/ios-snapshot-elements.md b/website/docs/docs/ios-snapshot-elements.md deleted file mode 100644 index e44780baa2..0000000000 --- a/website/docs/docs/ios-snapshot-elements.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: iOS Snapshot Element Support ---- - -# iOS Snapshot Element Support - -This page tracks how `snapshot -i` presents common iOS interaction surfaces. It -is scoped to the XCTest-backed iOS snapshot path and the default non-raw -presentation. Use `snapshot --raw` or `snapshot --json` when investigating the -provider tree itself. - -Sources used for this checklist: - -- Apple Human Interface Guidelines: [Components](https://developer.apple.com/design/human-interface-guidelines/components/) -- Apple Human Interface Guidelines: [Selection and input](https://developer.apple.com/design/human-interface-guidelines/selection-and-input) -- Apple Human Interface Guidelines: [Navigation and search](https://developer.apple.com/design/human-interface-guidelines/navigation-and-search) -- Apple Human Interface Guidelines: [Menus and actions](https://developer.apple.com/design/human-interface-guidelines/menus-and-actions) -- UIKit accessibility behavior: [UIAccessibility](https://developer.apple.com/documentation/UIKit/uiaccessibility-protocol) and [UIAccessibilityTraits](https://developer.apple.com/documentation/uikit/uiaccessibilitytraits) - -## Status Key - -| Status | Meaning | -| --- | --- | -| Supported | Expected to produce one actionable, unambiguous node in `snapshot -i`. | -| Pass-through | Preserved when XCTest exposes it, but no custom iOS presentation shaping yet. | -| Command-supported | Handled by a dedicated command rather than snapshot presentation. | -| Needs fixture | Known common element, but we still need raw tree and screenshot-overlay samples before claiming support. | -| Out of scope | Useful UI information, but not an interaction target for `snapshot -i`. | - -## Matrix - -| Apple/HIG area | Common iOS surface | Expected XCTest/accessibility shape | Current support | Tested with | Notes | -| --- | --- | --- | --- | --- | --- | -| Menus and actions | Button, toolbar button, pull-down or menu button | `Button`, sometimes wrapped in `Other` with the same label/rect | Supported | Settings rows, Expensify Home, Expensify action menu | Same-rect `Other -> Button` wrappers collapse to the button. Descendant duplicate text/image decoration is suppressed. | -| Menus and actions | Link | `Link`, sometimes wrapped in repeated static/link nodes | Supported | Unit fixture from Settings privacy link shape | Repeated link/static descendants collapse to one link. Needs more live Safari/Mail coverage. | -| Selection and input | Switch toggle | `Cell -> Button -> Switch`, or a standalone `Switch` | Supported | Settings Camera live screen, unit fixture | Row-backed switch collapses to the switch control, not the row/button wrapper. | -| Selection and input | Toggle button | Usually `Button` with selected/value/trait semantics | Pass-through | None yet | We keep the button. Need real samples to decide whether selected/value should be surfaced more explicitly. | -| Selection and input | Checkbox | `CheckBox`/`Checkbox` or button-like custom node | Pass-through | None yet | Common on macOS and web-like/custom iOS apps. Interactive filtering keeps role strings containing checkbox, but iOS shaping has no special checkbox rules. | -| Selection and input | Radio button/group | `RadioButton`, `RadioGroup`, or button-like custom nodes | Pass-through | None yet | Interactive filtering keeps role strings containing radio. Need grouped native/custom samples. | -| Selection and input | Text field | `TextField`, `SecureTextField`, or wrapper `Other -> TextField` | Supported | Expensify Search, unit fixture | Fillability already recognizes text/search/secure/text-view types. Wrapper collapses to the editable node. | -| Selection and input | Search field | `SearchField`, often inside toolbar/search wrappers | Supported | Settings search field, Expensify search field | Settings toolbar duplicates collapse to one search field plus Dictate button. | -| Selection and input | Text view / multiline editor | `TextView`/`TextArea` | Pass-through | Unit coverage for read/fill helpers | Fillability recognizes text views, but iOS presentation has no live complex editor sample yet. | -| Selection and input | Token field | `TextField`, token cells/buttons, or custom wrappers | Needs fixture | None yet | Need Mail/Calendar recipient-field samples. Likely mixed editable field plus token buttons. | -| Selection and input | Digit entry / OTP | Multiple `TextField`s, one hidden text field, or custom key grid | Needs fixture | None yet | We should avoid collapsing distinct digit boxes until tested. | -| Selection and input | Slider | `Slider`, usually adjustable and rect-backed | Pass-through | None yet | `snapshot -i` keeps rect-backed controls. Need Settings volume/brightness samples to verify value formatting and adjustability. | -| Selection and input | Stepper | `Stepper` or increment/decrement buttons | Needs fixture | None yet | Need native examples to decide whether to expose one adjustable control or separate buttons. | -| Selection and input | Segmented control | `SegmentedControl` with segment buttons, or collapsed adjustable-like control | Needs fixture | None yet | Current workflow docs still call out horizontal tab/filter bars that may collapse into one composite. Needs real-app coverage. | -| Selection and input | Picker / picker wheel | `Picker`, `PickerWheel`, option cells, or modal sheet | Needs fixture | None yet | Need Clock/Calendar/Settings samples. We should not over-collapse value wheels before testing. | -| Selection and input | Date picker | `DatePicker`, calendar buttons, picker wheels, compact button + popover | Needs fixture | None yet | Test compact, inline, and wheels styles separately. | -| Selection and input | Virtual keyboard and keys | `Keyboard`, `Key` | Command-supported | Expensify Search | Offscreen keyboard subtrees are suppressed when they are below the app viewport. Keyboard interaction belongs to `keyboard`/`fill`/`type`; visible app-owned Done buttons remain normal buttons. | -| Navigation and search | Navigation bar / toolbar | `NavigationBar`, `Toolbar`, title text, bar buttons | Supported | Settings, Expensify Search | Navigation/action buttons remain; duplicated title/search wrapper nodes are suppressed when they duplicate semantic children. | -| Navigation and search | Tab bar | `TabBar`, tab buttons, or custom `Other` nodes | Pass-through | Expensify Home | Expensify tabs are exposed as distinct `Other` items. Need native UITabBar samples to decide if `TabBar` container should stay visible or collapse. | -| Navigation and search | Sidebar/list navigation | `Table`, `CollectionView`, `Cell`, row button wrappers | Supported | Settings top/bottom, Expensify lists | Row-backed Settings cells collapse to a single `Cell`. Scroll-hidden above/below hints stay on the scroll container. | -| Navigation and search | Search results list | `ScrollView`/`Table`/`CollectionView` plus rows | Supported | Expensify Search | Repeated legitimate list rows do not trigger the duplicate-nav warning when they are stacked instead of overlapping. | -| Menus and actions | Context menu / edit menu | `Menu`, `MenuItem`, sheet/popover, or transient system surface | Needs fixture | None yet | Need long-press samples from Photos/Files/Text selection. | -| Menus and actions | Activity/share view | Sheet with buttons, collections, extension cells | Needs fixture | None yet | Treat as normal app/system UI until samples show duplication patterns. | -| Presentation | Alert | `Alert` plus buttons, or native alert command result | Command-supported | Provider tests | Use `alert wait/get/accept/dismiss` for native alerts. Snapshot should still be inspected when an app-owned sheet looks like an alert. | -| Presentation | Sheet, bottom sheet, popover | `Sheet`/`Popover`/`Other` wrappers plus buttons/fields | Supported for common wrappers | Expensify action menu | Backdrop dismiss and action-row wrappers collapse when semantic children are present. Need native sheet/popover examples. | -| Status | Activity indicator, progress indicator, gauge | `ActivityIndicator`, `ProgressIndicator`, `Other` with value | Out of scope | None yet | These are important state, but not primary interaction targets. They should stay in full snapshots; `snapshot -i` may omit them unless rect-backed. | -| Status | Page control / page indicator | `PageIndicator`, adjustable trait, dots/buttons | Needs fixture | None yet | Often interactive via swipe or adjustable actions. Need samples before deciding presentation. | -| Layout and organization | Static text, headers, images, decoration | `StaticText`, `Image`, `Other` | Supported as descendants/noise | Settings, Expensify | Repeated static/link/image descendants under an actionable parent are suppressed. Standalone useful text remains visible when retained by the interactive tree. | -| Layout and organization | Scroll bars / scroll indicators | `Other` with labels like vertical scroll bar | Supported | Settings top/bottom, Expensify lists | Scroll indicator nodes are removed from the interactive tree and converted into `[content above/below ... hidden]` hints on the nearest scroll container. | -| Layout and organization | Collection/table/scroll containers | `CollectionView`, `Table`, `ScrollView` | Supported | Settings, Expensify | Containers remain when they communicate scrollability or hidden content. | - -## Next Apps To Sample - -Use `snapshot -i`, `snapshot -i --raw --json`, and `screenshot --overlay-refs` for each sample before updating this page. - -| Element family | Suggested apps/screens | -| --- | --- | -| Slider / page indicator | Settings > Sounds & Haptics, Photos onboarding/carousel, Control Center if accessible | -| Segmented controls | Photos library filters, Calendar view switcher, App Store tabs/filters | -| Picker/date picker | Clock alarm editor, Calendar event editor, Reminders date picker | -| Stepper | Calendar/Reminders repeat or count fields, any system form that exposes increment/decrement | -| Token field | Mail compose recipients, Calendar invitees | -| Context/edit menu | Files long-press, Photos long-press, text selection in Notes | -| Native tab bar | App Store, Phone, Photos | -| Share/activity sheet | Photos share, Safari share | - -When adding coverage, record: - -- app and screen; -- shaped `snapshot -i` output; -- raw `snapshot -i --raw --json` node shape for the relevant subtree; -- overlay screenshot path; -- whether the result is supported, pass-through, or needs implementation work. diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 997140e28c..efa41c68b4 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -43,7 +43,6 @@ It does not automatically switch to AX. - Use `diff snapshot` between mutations to validate structural changes with lower output volume. - Use `snapshot --diff` when you discover the feature from snapshot help, but keep `diff snapshot` as the default exploration command. - Keep `--raw` for troubleshooting only when you need the full tree instead of visible-first output. -- For the current iOS interaction-element support matrix, see [iOS Snapshot Element Support](/docs/ios-snapshot-elements). `diff snapshot` and `snapshot --diff` behavior: - First run initializes baseline (`baselineInitialized: true` in JSON).