From e81fa5f0dcaad832ab66663287307fa47f681dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 18:02:27 +0200 Subject: [PATCH 1/7] docs: add collapsible sidebar sections --- website/docs/docs/_meta.json | 218 ++++++++++++++++++++--------------- 1 file changed, 125 insertions(+), 93 deletions(-) diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index 1317ac49a6..18b7aa9814 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -1,97 +1,129 @@ [ { - "name": "introduction", - "type": "file", - "label": "Introduction" - }, - { - "name": "installation", - "type": "file", - "label": "Installation" - }, - { - "name": "agent-setup", - "type": "file", - "label": "AI Agent Setup" - }, - { - "name": "quick-start", - "type": "file", - "label": "Quick Start" - }, - { - "name": "debugging-profiling", - "type": "file", - "label": "Debugging & Profiling" - }, - { - "name": "client-api", - "type": "file", - "label": "Typed Client" - }, - { - "name": "commands", - "type": "file", - "label": "Commands" - }, - { - "name": "configuration", - "type": "file", - "label": "Configuration" - }, - { - "name": "remote-proxy", - "type": "file", - "label": "Remote Proxy" - }, - { - "name": "device-clouds", - "type": "file", - "label": "Device Clouds" - }, - { - "name": "security-trust", - "type": "file", - "label": "Security & Trust" - }, - { - "name": "batching", - "type": "file", - "label": "Batching" - }, - { - "name": "selectors", - "type": "file", - "label": "Selectors" - }, - { - "name": "sessions", - "type": "file", - "label": "Sessions" - }, - { - "name": "replay-e2e", - "type": "file", - "label": "Replay & E2E" - }, - { - "name": "snapshots", - "type": "file", - "label": "Snapshots" - }, - { - "name": "ios-snapshot-elements", - "type": "file", - "label": "iOS Snapshot Elements" - }, - { - "name": "known-limitations", - "type": "file", - "label": "Known Limitations" - }, - { - "name": "migrating-gestures", - "type": "file", - "label": "Migrating Gestures" + "type": "custom-link", + "label": "Getting Started", + "collapsible": true, + "collapsed": false, + "items": [ + { + "type": "custom-link", + "label": "Introduction", + "link": "/docs/introduction" + }, + { + "type": "custom-link", + "label": "Installation", + "link": "/docs/installation" + }, + { + "type": "custom-link", + "label": "AI Agent Setup", + "link": "/docs/agent-setup" + }, + { + "type": "custom-link", + "label": "Quick Start", + "link": "/docs/quick-start" + } + ] + }, + { + "type": "custom-link", + "label": "Using agent-device", + "collapsible": true, + "collapsed": true, + "items": [ + { + "type": "custom-link", + "label": "Commands", + "link": "/docs/commands" + }, + { + "type": "custom-link", + "label": "Configuration", + "link": "/docs/configuration" + }, + { + "type": "custom-link", + "label": "Selectors", + "link": "/docs/selectors" + }, + { + "type": "custom-link", + "label": "Sessions", + "link": "/docs/sessions" + }, + { + "type": "custom-link", + "label": "Batching", + "link": "/docs/batching" + }, + { + "type": "custom-link", + "label": "Snapshots", + "link": "/docs/snapshots" + }, + { + "type": "custom-link", + "label": "Replay & E2E", + "link": "/docs/replay-e2e" + }, + { + "type": "custom-link", + "label": "Debugging & Profiling", + "link": "/docs/debugging-profiling" + } + ] + }, + { + "type": "custom-link", + "label": "Integrations", + "collapsible": true, + "collapsed": true, + "items": [ + { + "type": "custom-link", + "label": "Typed Client", + "link": "/docs/client-api" + }, + { + "type": "custom-link", + "label": "Remote Proxy", + "link": "/docs/remote-proxy" + }, + { + "type": "custom-link", + "label": "Device Clouds", + "link": "/docs/device-clouds" + } + ] + }, + { + "type": "custom-link", + "label": "Reference", + "collapsible": true, + "collapsed": true, + "items": [ + { + "type": "custom-link", + "label": "Security & Trust", + "link": "/docs/security-trust" + }, + { + "type": "custom-link", + "label": "iOS Snapshot Elements", + "link": "/docs/ios-snapshot-elements" + }, + { + "type": "custom-link", + "label": "Known Limitations", + "link": "/docs/known-limitations" + }, + { + "type": "custom-link", + "label": "Migrating Gestures", + "link": "/docs/migrating-gestures" + } + ] } ] From 4e3d3859053b4648c7764e47eb4b950db62233a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 18:11:15 +0200 Subject: [PATCH 2/7] docs: refine sidebar sections --- website/docs/docs/_meta.json | 19 ++--- website/docs/docs/client-api.md | 4 +- website/docs/docs/debugging-profiling.md | 2 +- website/docs/docs/ios-snapshot-elements.md | 86 ---------------------- website/docs/docs/snapshots.md | 1 - 5 files changed, 8 insertions(+), 104 deletions(-) delete mode 100644 website/docs/docs/ios-snapshot-elements.md diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index 18b7aa9814..cd2403f9ea 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -2,8 +2,7 @@ { "type": "custom-link", "label": "Getting Started", - "collapsible": true, - "collapsed": false, + "collapsible": false, "items": [ { "type": "custom-link", @@ -30,8 +29,7 @@ { "type": "custom-link", "label": "Using agent-device", - "collapsible": true, - "collapsed": true, + "collapsible": false, "items": [ { "type": "custom-link", @@ -78,12 +76,11 @@ { "type": "custom-link", "label": "Integrations", - "collapsible": true, - "collapsed": true, + "collapsible": false, "items": [ { "type": "custom-link", - "label": "Typed Client", + "label": "Node.js API", "link": "/docs/client-api" }, { @@ -101,19 +98,13 @@ { "type": "custom-link", "label": "Reference", - "collapsible": true, - "collapsed": true, + "collapsible": false, "items": [ { "type": "custom-link", "label": "Security & Trust", "link": "/docs/security-trust" }, - { - "type": "custom-link", - "label": "iOS Snapshot Elements", - "link": "/docs/ios-snapshot-elements" - }, { "type": "custom-link", "label": "Known Limitations", diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index c543f858f3..cead82b81f 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -1,8 +1,8 @@ --- -title: Typed Client +title: Node.js API --- -# Typed Client +# Node.js API Use `createAgentDeviceClient()` when you want to drive the daemon from application code instead of shelling out to the CLI. diff --git a/website/docs/docs/debugging-profiling.md b/website/docs/docs/debugging-profiling.md index 0af55a43cb..053411c1e8 100644 --- a/website/docs/docs/debugging-profiling.md +++ b/website/docs/docs/debugging-profiling.md @@ -192,5 +192,5 @@ agent-device perf trace stop --kind perfetto --out app.perfetto-trace ## Where to go deeper - Full command reference: [Commands](/docs/commands) -- Typed client observability APIs: [Typed Client](/docs/client-api) +- Node.js observability APIs: [Node.js API](/docs/client-api) - Session behavior and lifecycle: [Sessions](/docs/sessions) diff --git a/website/docs/docs/ios-snapshot-elements.md b/website/docs/docs/ios-snapshot-elements.md deleted file mode 100644 index e44780baa2..0000000000 --- a/website/docs/docs/ios-snapshot-elements.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: iOS Snapshot Element Support ---- - -# iOS Snapshot Element Support - -This page tracks how `snapshot -i` presents common iOS interaction surfaces. It -is scoped to the XCTest-backed iOS snapshot path and the default non-raw -presentation. Use `snapshot --raw` or `snapshot --json` when investigating the -provider tree itself. - -Sources used for this checklist: - -- Apple Human Interface Guidelines: [Components](https://developer.apple.com/design/human-interface-guidelines/components/) -- Apple Human Interface Guidelines: [Selection and input](https://developer.apple.com/design/human-interface-guidelines/selection-and-input) -- Apple Human Interface Guidelines: [Navigation and search](https://developer.apple.com/design/human-interface-guidelines/navigation-and-search) -- Apple Human Interface Guidelines: [Menus and actions](https://developer.apple.com/design/human-interface-guidelines/menus-and-actions) -- UIKit accessibility behavior: [UIAccessibility](https://developer.apple.com/documentation/UIKit/uiaccessibility-protocol) and [UIAccessibilityTraits](https://developer.apple.com/documentation/uikit/uiaccessibilitytraits) - -## Status Key - -| Status | Meaning | -| --- | --- | -| Supported | Expected to produce one actionable, unambiguous node in `snapshot -i`. | -| Pass-through | Preserved when XCTest exposes it, but no custom iOS presentation shaping yet. | -| Command-supported | Handled by a dedicated command rather than snapshot presentation. | -| Needs fixture | Known common element, but we still need raw tree and screenshot-overlay samples before claiming support. | -| Out of scope | Useful UI information, but not an interaction target for `snapshot -i`. | - -## Matrix - -| Apple/HIG area | Common iOS surface | Expected XCTest/accessibility shape | Current support | Tested with | Notes | -| --- | --- | --- | --- | --- | --- | -| Menus and actions | Button, toolbar button, pull-down or menu button | `Button`, sometimes wrapped in `Other` with the same label/rect | Supported | Settings rows, Expensify Home, Expensify action menu | Same-rect `Other -> Button` wrappers collapse to the button. Descendant duplicate text/image decoration is suppressed. | -| Menus and actions | Link | `Link`, sometimes wrapped in repeated static/link nodes | Supported | Unit fixture from Settings privacy link shape | Repeated link/static descendants collapse to one link. Needs more live Safari/Mail coverage. | -| Selection and input | Switch toggle | `Cell -> Button -> Switch`, or a standalone `Switch` | Supported | Settings Camera live screen, unit fixture | Row-backed switch collapses to the switch control, not the row/button wrapper. | -| Selection and input | Toggle button | Usually `Button` with selected/value/trait semantics | Pass-through | None yet | We keep the button. Need real samples to decide whether selected/value should be surfaced more explicitly. | -| Selection and input | Checkbox | `CheckBox`/`Checkbox` or button-like custom node | Pass-through | None yet | Common on macOS and web-like/custom iOS apps. Interactive filtering keeps role strings containing checkbox, but iOS shaping has no special checkbox rules. | -| Selection and input | Radio button/group | `RadioButton`, `RadioGroup`, or button-like custom nodes | Pass-through | None yet | Interactive filtering keeps role strings containing radio. Need grouped native/custom samples. | -| Selection and input | Text field | `TextField`, `SecureTextField`, or wrapper `Other -> TextField` | Supported | Expensify Search, unit fixture | Fillability already recognizes text/search/secure/text-view types. Wrapper collapses to the editable node. | -| Selection and input | Search field | `SearchField`, often inside toolbar/search wrappers | Supported | Settings search field, Expensify search field | Settings toolbar duplicates collapse to one search field plus Dictate button. | -| Selection and input | Text view / multiline editor | `TextView`/`TextArea` | Pass-through | Unit coverage for read/fill helpers | Fillability recognizes text views, but iOS presentation has no live complex editor sample yet. | -| Selection and input | Token field | `TextField`, token cells/buttons, or custom wrappers | Needs fixture | None yet | Need Mail/Calendar recipient-field samples. Likely mixed editable field plus token buttons. | -| Selection and input | Digit entry / OTP | Multiple `TextField`s, one hidden text field, or custom key grid | Needs fixture | None yet | We should avoid collapsing distinct digit boxes until tested. | -| Selection and input | Slider | `Slider`, usually adjustable and rect-backed | Pass-through | None yet | `snapshot -i` keeps rect-backed controls. Need Settings volume/brightness samples to verify value formatting and adjustability. | -| Selection and input | Stepper | `Stepper` or increment/decrement buttons | Needs fixture | None yet | Need native examples to decide whether to expose one adjustable control or separate buttons. | -| Selection and input | Segmented control | `SegmentedControl` with segment buttons, or collapsed adjustable-like control | Needs fixture | None yet | Current workflow docs still call out horizontal tab/filter bars that may collapse into one composite. Needs real-app coverage. | -| Selection and input | Picker / picker wheel | `Picker`, `PickerWheel`, option cells, or modal sheet | Needs fixture | None yet | Need Clock/Calendar/Settings samples. We should not over-collapse value wheels before testing. | -| Selection and input | Date picker | `DatePicker`, calendar buttons, picker wheels, compact button + popover | Needs fixture | None yet | Test compact, inline, and wheels styles separately. | -| Selection and input | Virtual keyboard and keys | `Keyboard`, `Key` | Command-supported | Expensify Search | Offscreen keyboard subtrees are suppressed when they are below the app viewport. Keyboard interaction belongs to `keyboard`/`fill`/`type`; visible app-owned Done buttons remain normal buttons. | -| Navigation and search | Navigation bar / toolbar | `NavigationBar`, `Toolbar`, title text, bar buttons | Supported | Settings, Expensify Search | Navigation/action buttons remain; duplicated title/search wrapper nodes are suppressed when they duplicate semantic children. | -| Navigation and search | Tab bar | `TabBar`, tab buttons, or custom `Other` nodes | Pass-through | Expensify Home | Expensify tabs are exposed as distinct `Other` items. Need native UITabBar samples to decide if `TabBar` container should stay visible or collapse. | -| Navigation and search | Sidebar/list navigation | `Table`, `CollectionView`, `Cell`, row button wrappers | Supported | Settings top/bottom, Expensify lists | Row-backed Settings cells collapse to a single `Cell`. Scroll-hidden above/below hints stay on the scroll container. | -| Navigation and search | Search results list | `ScrollView`/`Table`/`CollectionView` plus rows | Supported | Expensify Search | Repeated legitimate list rows do not trigger the duplicate-nav warning when they are stacked instead of overlapping. | -| Menus and actions | Context menu / edit menu | `Menu`, `MenuItem`, sheet/popover, or transient system surface | Needs fixture | None yet | Need long-press samples from Photos/Files/Text selection. | -| Menus and actions | Activity/share view | Sheet with buttons, collections, extension cells | Needs fixture | None yet | Treat as normal app/system UI until samples show duplication patterns. | -| Presentation | Alert | `Alert` plus buttons, or native alert command result | Command-supported | Provider tests | Use `alert wait/get/accept/dismiss` for native alerts. Snapshot should still be inspected when an app-owned sheet looks like an alert. | -| Presentation | Sheet, bottom sheet, popover | `Sheet`/`Popover`/`Other` wrappers plus buttons/fields | Supported for common wrappers | Expensify action menu | Backdrop dismiss and action-row wrappers collapse when semantic children are present. Need native sheet/popover examples. | -| Status | Activity indicator, progress indicator, gauge | `ActivityIndicator`, `ProgressIndicator`, `Other` with value | Out of scope | None yet | These are important state, but not primary interaction targets. They should stay in full snapshots; `snapshot -i` may omit them unless rect-backed. | -| Status | Page control / page indicator | `PageIndicator`, adjustable trait, dots/buttons | Needs fixture | None yet | Often interactive via swipe or adjustable actions. Need samples before deciding presentation. | -| Layout and organization | Static text, headers, images, decoration | `StaticText`, `Image`, `Other` | Supported as descendants/noise | Settings, Expensify | Repeated static/link/image descendants under an actionable parent are suppressed. Standalone useful text remains visible when retained by the interactive tree. | -| Layout and organization | Scroll bars / scroll indicators | `Other` with labels like vertical scroll bar | Supported | Settings top/bottom, Expensify lists | Scroll indicator nodes are removed from the interactive tree and converted into `[content above/below ... hidden]` hints on the nearest scroll container. | -| Layout and organization | Collection/table/scroll containers | `CollectionView`, `Table`, `ScrollView` | Supported | Settings, Expensify | Containers remain when they communicate scrollability or hidden content. | - -## Next Apps To Sample - -Use `snapshot -i`, `snapshot -i --raw --json`, and `screenshot --overlay-refs` for each sample before updating this page. - -| Element family | Suggested apps/screens | -| --- | --- | -| Slider / page indicator | Settings > Sounds & Haptics, Photos onboarding/carousel, Control Center if accessible | -| Segmented controls | Photos library filters, Calendar view switcher, App Store tabs/filters | -| Picker/date picker | Clock alarm editor, Calendar event editor, Reminders date picker | -| Stepper | Calendar/Reminders repeat or count fields, any system form that exposes increment/decrement | -| Token field | Mail compose recipients, Calendar invitees | -| Context/edit menu | Files long-press, Photos long-press, text selection in Notes | -| Native tab bar | App Store, Phone, Photos | -| Share/activity sheet | Photos share, Safari share | - -When adding coverage, record: - -- app and screen; -- shaped `snapshot -i` output; -- raw `snapshot -i --raw --json` node shape for the relevant subtree; -- overlay screenshot path; -- whether the result is supported, pass-through, or needs implementation work. diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 997140e28c..efa41c68b4 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -43,7 +43,6 @@ It does not automatically switch to AX. - Use `diff snapshot` between mutations to validate structural changes with lower output volume. - Use `snapshot --diff` when you discover the feature from snapshot help, but keep `diff snapshot` as the default exploration command. - Keep `--raw` for troubleshooting only when you need the full tree instead of visible-first output. -- For the current iOS interaction-element support matrix, see [iOS Snapshot Element Support](/docs/ios-snapshot-elements). `diff snapshot` and `snapshot --diff` behavior: - First run initializes baseline (`baselineInitialized: true` in JSON). From a8f4a7359daebc03f04d95be3f36e5ea38bf5c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 18:17:44 +0200 Subject: [PATCH 3/7] docs: clarify replay sidebar label --- website/docs/docs/_meta.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index cd2403f9ea..602468d34b 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -63,7 +63,7 @@ }, { "type": "custom-link", - "label": "Replay & E2E", + "label": "Replay & E2E testing", "link": "/docs/replay-e2e" }, { From 77f491c9f6c54502b24935380df4693d73359f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 18:20:52 +0200 Subject: [PATCH 4/7] docs: link runnable Node.js API examples --- examples/README.md | 2 +- website/docs/docs/client-api.md | 69 +++++++++++---------------------- 2 files changed, 23 insertions(+), 48 deletions(-) diff --git a/examples/README.md b/examples/README.md index 7a78f3b5cc..ead2c20f25 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,7 +2,7 @@ Runnable, typechecked Node.js examples for the `agent-device` SDK surface exposed to Node consumers. Source of truth for the API itself is -[Typed Client](../website/docs/docs/client-api.md). Two guards keep these files in sync with that +[Node.js API](../website/docs/docs/client-api.md). Two guards keep these files in sync with that doc: `src/__tests__/client-api-examples-drift.test.ts` checks the doc's subpath API manifest against what these examples import, and `test/integration/client-api-doc-snippets.test.ts` compiles every fenced TypeScript code block in the doc itself against the real `agent-device/*` sources. diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index cead82b81f..4f47c37e7a 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -8,6 +8,23 @@ Use `createAgentDeviceClient()` when you want to drive the daemon from applicati For remote Metro-backed flows, import the reusable Node APIs instead of spawning the `agent-device` binary. The CLI uses the same helpers internally. +## Runnable examples + +The repository includes [runnable, typechecked Node.js examples](https://github.com/callstack/agent-device/tree/main/examples/sdk) that import the same published `agent-device/*` entry points used by consumers: + +| Example | Demonstrates | +| --- | --- | +| [`client-session.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/client-session.ts) | Open, snapshot, interact, handle typed errors, and always close the session | +| [`contracts-result.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/contracts-result.ts) | Consume snapshot results with helpers from `agent-device/contracts` | +| [`batch-orchestration.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/batch-orchestration.ts) | Run a batch through a custom transport | +| [`metro-runtime.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/metro-runtime.ts) | Normalize a Metro URL and resolve runtime transport hints | + +The examples are checked against the source SDK using their dedicated [`tsconfig.json`](https://github.com/callstack/agent-device/blob/main/examples/sdk/tsconfig.json). After building the package with `pnpm build`, run an example directly with Node: + +```bash +node --experimental-strip-types examples/sdk/client-session.ts +``` + Public subpath API exposed for Node consumers: - `agent-device` @@ -75,39 +92,9 @@ The `contracts`, `selectors`, `finders`, `install-source`, `android-adb`, `limru ## Basic usage -```ts -import { createAgentDeviceClient } from 'agent-device'; - -const client = createAgentDeviceClient({ - session: 'qa-ios', - lockPolicy: 'reject', - lockPlatform: 'ios', -}); - -const devices = await client.devices.list({ platform: 'ios' }); -const capabilities = await client.devices.capabilities({ platform: 'ios' }); -const apps = await client.apps.list({ platform: 'ios' }); -const device = devices.find((candidate) => candidate.name === 'iPhone 16') ?? devices[0]; -if (!device) { - throw new Error('No iOS device available'); -} -if (!capabilities.availableCommands.includes('snapshot')) { - throw new Error('Selected target does not support snapshots'); -} - -await client.apps.open({ - app: 'com.apple.Preferences', - platform: 'ios', - udid: device.id, - runtime: { - metroHost: '127.0.0.1', - metroPort: 8081, - }, -}); +The canonical client example is embedded below. It is also runnable from [`examples/sdk/client-session.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/client-session.ts). -const snapshot = await client.capture.snapshot({ interactiveOnly: true }); - -await client.sessions.close(); +```ts file="/../examples/sdk/client-session.ts" ``` `client.devices.capabilities()` returns `{ device, availableCommands }`, using the same capability matrix as the CLI. Use it when a dynamic integration needs to decide which command names are valid for the selected target. @@ -331,21 +318,9 @@ executor. Use `agent-device/batch` when a bridge or in-process runner receives daemon-shaped requests but owns command dispatch itself. The helper keeps validation, inherited flags, serial execution, partial results, and error envelopes aligned with the daemon batch command. -```ts -import { runBatch } from 'agent-device/batch'; -import type { DaemonResponse } from 'agent-device/contracts'; - -type BatchRequest = Parameters[0]; - -async function handleBatch(req: BatchRequest): Promise { - return await runBatch(req, req.session ?? 'default', async (stepReq) => { - try { - return { ok: true, data: await dispatch(stepReq) }; - } catch (error) { - return bridgeErrorToDaemonResponse(error); - } - }); -} +The standalone custom-transport example is embedded below from [`examples/sdk/batch-orchestration.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/batch-orchestration.ts). + +```ts file="/../examples/sdk/batch-orchestration.ts" ``` ## Android `installFromSource()` From bacc5d6a41956ab52c823b825959e8c9d1206b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 18:25:57 +0200 Subject: [PATCH 5/7] docs: align Node.js API reference --- .../client-api-examples-drift.test.ts | 15 +++++++ website/docs/docs/client-api.md | 43 ++++++++++++------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/__tests__/client-api-examples-drift.test.ts b/src/__tests__/client-api-examples-drift.test.ts index 61218967d6..7f9bbea4a0 100644 --- a/src/__tests__/client-api-examples-drift.test.ts +++ b/src/__tests__/client-api-examples-drift.test.ts @@ -24,6 +24,7 @@ import { describe, test } from 'vitest'; const CLIENT_API_DOC_PATH = 'website/docs/docs/client-api.md'; const EXAMPLES_SDK_DIR = 'examples/sdk'; +const PACKAGE_JSON_PATH = 'package.json'; type SubpathManifest = ReadonlyMap>; @@ -100,6 +101,12 @@ function listExampleFiles(dir: string): string[] { const manifest = parseSubpathManifest(fs.readFileSync(CLIENT_API_DOC_PATH, 'utf8')); const exampleFiles = listExampleFiles(EXAMPLES_SDK_DIR); +const packageExports = Object.keys( + (JSON.parse(fs.readFileSync(PACKAGE_JSON_PATH, 'utf8')) as { exports?: Record }) + .exports ?? {}, +) + .map((subpath) => (subpath === '.' ? 'agent-device' : `agent-device${subpath.slice(1)}`)) + .sort(); const importsByFile = new Map( exampleFiles.map((file) => [file, extractImportedSymbols(fs.readFileSync(file, 'utf8'))]), ); @@ -113,6 +120,14 @@ describe('examples/sdk vs client-api.md drift guard', () => { ); }); + test('client-api.md documents every published package entry point', () => { + assert.deepEqual( + [...manifest.keys()].sort(), + packageExports, + `${CLIENT_API_DOC_PATH}'s public subpath manifest must match ${PACKAGE_JSON_PATH}#exports`, + ); + }); + test('every symbol an example imports from agent-device is documented in client-api.md', () => { const undocumented: string[] = []; for (const [file, importsBySubpath] of importsByFile) { diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 4f47c37e7a..c386ad060d 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -32,9 +32,11 @@ Public subpath API exposed for Node consumers: - `createLocalArtifactAdapter(options?)` - `AppError`, `isAgentDeviceError(error)`, `normalizeAgentDeviceError(error)` - `centerOfRect(rect)` - - root types are limited to the typed client contracts used by hosted adapters, such as `AppListOptions`, `BackCommandOptions`, `ScrollOptions`, and command result types. - `agent-device/io` - - artifact adapter types, file input refs, and file output refs + - `createLocalArtifactAdapter(options?)` + - types: `ArtifactAdapter`, `ArtifactDescriptor`, `CreateTempFileOptions`, `FileInputRef`, + `FileOutputRef`, `LocalArtifactAdapterOptions`, `OutputVisibility`, `ReserveOutputOptions`, + `ReservedOutputFile`, `ResolveInputOptions`, `ResolvedInputFile`, `TemporaryFile` - `agent-device/metro` - `buildBundleUrl(baseUrl, platform)` - `normalizeBaseUrl(baseUrl)` @@ -65,6 +67,7 @@ Public subpath API exposed for Node consumers: - `parseFindArgs(args)` - types: `FindMatchOptions` - `agent-device/install-source` + - `ARCHIVE_EXTENSIONS` - `isTrustedInstallSourceUrl(sourceUrl)` - `validateDownloadSourceUrl(url)` - types: `MaterializeInstallSource` @@ -85,8 +88,10 @@ Public subpath API exposed for Node consumers: - `new LimrunRuntime(options)` - `runtime.getDeviceSession(device)` - types: `LimrunRuntimeOptions`, `LimrunDeviceSession`, `LimrunAndroidDeviceSession`, - `LimrunIosDeviceSession`, `AndroidAdbProvider`, `AndroidKeyboardState`, - `AndroidKeyboardDismissResult` + `LimrunIosDeviceSession`, `LimrunForegroundApp`, `LimrunInstalledApp`, + `LimrunIosCommandExecution`, `LimrunIosCommandResult`, `LimrunRecordingQuality`, + `LimrunIosRemoteInstallOptions`, `LimrunIosRemoteInstallResult`, `AndroidAdbProvider`, + `AndroidKeyboardState`, `AndroidKeyboardDismissResult` The `contracts`, `selectors`, `finders`, `install-source`, `android-adb`, `limrun`, `artifacts`, `batch`, `metro`, `remote-config`, and `io` subpaths are the supported Node entry points. The former compatibility subpaths `agent-device/android-apps` and `agent-device/daemon`, plus hosted-runtime subpaths `agent-device/cloud-webdriver`, `agent-device/commands`, `agent-device/backend`, `agent-device/testing/conformance`, and `agent-device/observability`, are not published. @@ -254,6 +259,7 @@ Vega OS client support is currently VVD-only and covers device discovery, app op Supported command methods: - `wait` +- `alert` - `appState` - `back` - `home` @@ -262,20 +268,27 @@ Supported command methods: - `keyboard` - `clipboard` - `tvRemote` -- `alert` - -Additional CLI-backed methods are exposed on their domain groups with typed option objects so Node consumers do not need to build raw daemon requests: - -- `client.devices.boot()` -- `client.devices.capabilities()` -- `client.devices.shutdown()` -- `client.apps.push()` -- `client.apps.triggerEvent()` -- `client.capture.diff()` -- `client.interactions.click()`, `press()`, `longPress()`, `swipe()`, `pan()`, `fling()`, `focus()`, `type()`, `fill()`, `scroll()`, `pinch()`, `rotateGesture()`, `transformGesture()`, `get()`, `is()`, `find()` +- `reactNative` +- `doctor` +- `prepare` +- `viewport` + +The deprecated `rotate()` alias remains available for compatibility; use `orientation()` in new integrations. + +The complete domain-client method map is: + +- `client.devices.list()`, `capabilities()`, `boot()`, `shutdown()` +- `client.sessions.list()`, `stateDir()`, `close()`, `saveScript()`, `artifacts()` +- `client.apps.install()`, `reinstall()`, `installFromSource()`, `list()`, `open()`, `close()`, `push()`, `triggerEvent()` +- `client.materializations.release()` +- `client.leases.allocate()`, `heartbeat()`, `release()` +- `client.metro.prepare()`, `reload()` +- `client.capture.snapshot()`, `screenshot()`, `diff()` +- `client.interactions.click()`, `press()`, `longPress()`, `swipe()`, `pan()`, `fling()`, `swipeGesture()`, `focus()`, `type()`, `fill()`, `scroll()`, `pinch()`, `rotateGesture()`, `transformGesture()`, `get()`, `is()`, `find()` - `client.replay.run()` and `client.replay.test()` - `client.batch.run()` - `client.observability.perf()`, `logs()`, `events()`, `network()`, and `audio()` +- `client.debug.symbols()` - `client.recording.record()` and `client.recording.trace()` - `client.settings.update()` From 74a2e43983768276e981f7a823ec8e0b249681d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 18:28:14 +0200 Subject: [PATCH 6/7] docs: clarify Node.js API reference --- src/__tests__/client-api-examples-drift.test.ts | 4 ++-- website/docs/docs/client-api.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/__tests__/client-api-examples-drift.test.ts b/src/__tests__/client-api-examples-drift.test.ts index 7f9bbea4a0..183606dcc2 100644 --- a/src/__tests__/client-api-examples-drift.test.ts +++ b/src/__tests__/client-api-examples-drift.test.ts @@ -38,7 +38,7 @@ const REQUIRED_EXAMPLE_SYMBOLS: readonly { subpath: string; symbol: string }[] = { subpath: 'agent-device/batch', symbol: 'runBatch' }, ]; -// Parses the "Public subpath API" bullet list: a top-level `- \`agent-device...\`` +// Parses the "API reference" bullet list: a top-level `- \`agent-device...\`` // bullet starts a subpath section; backtick-quoted identifiers on its nested // bullet lines (stripping a trailing `(...)` call signature) are that // subpath's documented symbols, until the next top-level bullet. @@ -116,7 +116,7 @@ describe('examples/sdk vs client-api.md drift guard', () => { assert.ok( manifest.size > 0, `${CLIENT_API_DOC_PATH} did not yield a parseable subpath API manifest; ` + - 'has the "Public subpath API exposed for Node consumers" list moved or changed format?', + 'has the "API reference" entry-point list moved or changed format?', ); }); diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index c386ad060d..388e017600 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -25,7 +25,9 @@ The examples are checked against the source SDK using their dedicated [`tsconfig node --experimental-strip-types examples/sdk/client-session.ts ``` -Public subpath API exposed for Node consumers: +## API reference + +Supported public entry points for Node consumers: - `agent-device` - `createAgentDeviceClient(options?)` @@ -93,8 +95,6 @@ Public subpath API exposed for Node consumers: `LimrunIosRemoteInstallOptions`, `LimrunIosRemoteInstallResult`, `AndroidAdbProvider`, `AndroidKeyboardState`, `AndroidKeyboardDismissResult` -The `contracts`, `selectors`, `finders`, `install-source`, `android-adb`, `limrun`, `artifacts`, `batch`, `metro`, `remote-config`, and `io` subpaths are the supported Node entry points. The former compatibility subpaths `agent-device/android-apps` and `agent-device/daemon`, plus hosted-runtime subpaths `agent-device/cloud-webdriver`, `agent-device/commands`, `agent-device/backend`, `agent-device/testing/conformance`, and `agent-device/observability`, are not published. - ## Basic usage The canonical client example is embedded below. It is also runnable from [`examples/sdk/client-session.ts`](https://github.com/callstack/agent-device/blob/main/examples/sdk/client-session.ts). From edf2209d14e81a3f49c28f6a4f3eb86bf673eba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 18:33:57 +0200 Subject: [PATCH 7/7] docs: add Node agent integrations --- website/docs/docs/_meta.json | 10 ++++ website/docs/docs/ai-sdk.md | 78 +++++++++++++++++++++++++++++++ website/docs/docs/client-api.md | 4 +- website/docs/docs/eve.md | 83 +++++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 website/docs/docs/ai-sdk.md create mode 100644 website/docs/docs/eve.md diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index 602468d34b..b950d0cd5a 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -83,6 +83,16 @@ "label": "Node.js API", "link": "/docs/client-api" }, + { + "type": "custom-link", + "label": "AI SDK", + "link": "/docs/ai-sdk" + }, + { + "type": "custom-link", + "label": "Eve", + "link": "/docs/eve" + }, { "type": "custom-link", "label": "Remote Proxy", diff --git a/website/docs/docs/ai-sdk.md b/website/docs/docs/ai-sdk.md new file mode 100644 index 0000000000..736c2c91f8 --- /dev/null +++ b/website/docs/docs/ai-sdk.md @@ -0,0 +1,78 @@ +--- +title: AI SDK +--- + +# AI SDK + +[Vercel's AI SDK](https://ai-sdk.dev/) can expose `agent-device` client methods as typed tools in a Node.js agent. `ToolLoopAgent` owns the model loop, while `createAgentDeviceClient()` owns the device session and keeps tool implementations aligned with the CLI command contracts. + +Install the integration dependencies: + +```bash +pnpm add agent-device ai zod +``` + +The example below gives the model two deliberately small tools: one for observing the current UI and one for pressing an element returned by that observation. + +```ts +import { ToolLoopAgent, tool } from 'ai'; +import { createAgentDeviceClient } from 'agent-device'; +import { z } from 'zod'; + +const client = createAgentDeviceClient({ + session: 'ai-sdk-agent', + lockPolicy: 'reject', +}); + +const agent = new ToolLoopAgent({ + model: process.env.AI_MODEL!, + instructions: [ + 'Inspect the current UI before acting.', + 'Only press an element ref returned by the latest snapshot.', + 'Stop and explain when the requested state cannot be verified.', + ].join('\n'), + tools: { + snapshot: tool({ + description: 'Return the interactive elements in the current device UI.', + inputSchema: z.object({}), + execute: async () => await client.capture.snapshot({ interactiveOnly: true }), + }), + press: tool({ + description: 'Press an element from the latest snapshot by its @e ref.', + inputSchema: z.object({ + ref: z.string().regex(/^@e\d+$/), + }), + execute: async ({ ref }) => await client.interactions.press({ ref }), + }), + }, +}); + +try { + await client.apps.open({ + app: 'com.example.app', + platform: 'ios', + }); + + const result = await agent.generate({ + prompt: 'Navigate to Notifications and verify that notifications are enabled.', + }); + + console.log(result.text); +} finally { + await client.sessions.close(); +} +``` + +Set `AI_MODEL` to a model available through your configured AI SDK provider. See the AI SDK references for [`ToolLoopAgent`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool-loop-agent) and [`tool()`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool). + +## Designing device tools + +Prefer focused tools over a single tool that accepts an arbitrary command name and arguments: + +- Validate tool input with a schema. In particular, constrain element refs to values such as `@e12` and make the model observe before acting. +- Keep the `agent-device` client outside tool execution so calls share one named session. +- Return typed client results directly unless the result needs an application-specific projection. +- Let the host application own session cleanup with `try`/`finally`; do not rely on the model to close the session. +- Use AI SDK's `needsApproval` option for actions that require human confirmation in your product. + +See [Node.js API](/docs/client-api) for the complete client surface and runnable, typechecked `agent-device` examples. diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 388e017600..dc383644cd 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -4,9 +4,9 @@ title: Node.js API # Node.js API -Use `createAgentDeviceClient()` when you want to drive the daemon from application code instead of shelling out to the CLI. +Use `createAgentDeviceClient()` to give a Node.js agent typed access to device automation instead of shelling out to the CLI. Its methods can be exposed as model tools, called from deterministic orchestration code, or combined with another Node.js agent framework. -For remote Metro-backed flows, import the reusable Node APIs instead of spawning the `agent-device` binary. The CLI uses the same helpers internally. +Start with the [AI SDK](/docs/ai-sdk) or [Eve](/docs/eve) integration guide for complete tool-calling examples. The client is framework-neutral, so the same pattern works with other solutions that accept JavaScript or TypeScript functions as tools. ## Runnable examples diff --git a/website/docs/docs/eve.md b/website/docs/docs/eve.md new file mode 100644 index 0000000000..5e6138ba9f --- /dev/null +++ b/website/docs/docs/eve.md @@ -0,0 +1,83 @@ +--- +title: Eve +--- + +# Eve + +[Eve](https://eve.dev/) is Vercel's filesystem-first framework for durable agents. Files under `agent/tools/` become typed model tools, which makes them a natural place to adapt the `agent-device` Node.js client. + +Create an Eve project, then add `agent-device`: + +```bash +npx eve@latest init mobile-agent +cd mobile-agent +pnpm add agent-device +``` + +Add `agent/tools/agent_device.ts`: + +```ts +import { createAgentDeviceClient } from 'agent-device'; +import { defineTool } from 'eve/tools'; +import { z } from 'zod'; + +const client = createAgentDeviceClient({ + session: 'eve-agent', + lockPolicy: 'reject', +}); + +export default defineTool({ + description: 'Inspect or interact with the current device UI.', + inputSchema: z.discriminatedUnion('action', [ + z.object({ + action: z.literal('open'), + app: z.string().min(1), + platform: z.enum(['ios', 'android']), + }), + z.object({ + action: z.literal('snapshot'), + }), + z.object({ + action: z.literal('press'), + ref: z.string().regex(/^@e\d+$/), + }), + z.object({ + action: z.literal('close'), + }), + ]), + async execute(input) { + switch (input.action) { + case 'open': + return await client.apps.open({ app: input.app, platform: input.platform }); + case 'snapshot': + return await client.capture.snapshot({ interactiveOnly: true }); + case 'press': + return await client.interactions.press({ ref: input.ref }); + case 'close': + return await client.sessions.close(); + } + }, +}); +``` + +Eve discovers the file automatically; no tool registry is required. Tell the agent how to use it in `agent/instructions.md`: + +```md +Use the agent_device tool to inspect and operate the app. + +- Open the requested app before inspecting it. +- Call snapshot before every interaction. +- Only press an @e ref from the latest snapshot. +- Verify the requested outcome with another snapshot. +- Call close when the device task is complete. +``` + +For CI or another short-lived host, the outer runner should still close the named `agent-device` session in its own cleanup path. Model-directed `close` is useful during the normal tool loop, but it is not a replacement for deterministic cleanup after errors or cancellation. + +## Runtime placement + +Run the tool in Eve's app runtime when it needs local access to simulators, emulators, platform tooling, and daemon state. If Eve is hosted separately from the devices, connect through an [agent-device remote proxy](/docs/remote-proxy) instead. + +Eve is currently beta, so check its [current documentation](https://eve.dev/) when upgrading. For a production example, read [Building Mobile QA Agents With Vercel Eve](https://www.callstack.com/blog/building-reviewable-mobile-qa-agents-with-vercel-eve), which covers a PR QA agent using an `agent_device` tool and a deterministic CI runner. + +See [Node.js API](/docs/client-api) for the complete client surface.