Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

Runnable, typechecked Node.js examples for the `agent-device` SDK surface exposed to Node
consumers. Source of truth for the API itself is
[Typed Client](../website/docs/docs/client-api.md). Two guards keep these files in sync with that
[Node.js API](../website/docs/docs/client-api.md). Two guards keep these files in sync with that
doc: `src/__tests__/client-api-examples-drift.test.ts` checks the doc's subpath API manifest against
what these examples import, and `test/integration/client-api-doc-snippets.test.ts` compiles every
fenced TypeScript code block in the doc itself against the real `agent-device/*` sources.
Expand Down
19 changes: 17 additions & 2 deletions src/__tests__/client-api-examples-drift.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import { describe, test } from 'vitest';

const CLIENT_API_DOC_PATH = 'website/docs/docs/client-api.md';
const EXAMPLES_SDK_DIR = 'examples/sdk';
const PACKAGE_JSON_PATH = 'package.json';

type SubpathManifest = ReadonlyMap<string, ReadonlySet<string>>;

Expand All @@ -37,7 +38,7 @@ const REQUIRED_EXAMPLE_SYMBOLS: readonly { subpath: string; symbol: string }[] =
{ subpath: 'agent-device/batch', symbol: 'runBatch' },
];

// Parses the "Public subpath API" bullet list: a top-level `- \`agent-device...\``
// Parses the "API reference" bullet list: a top-level `- \`agent-device...\``
// bullet starts a subpath section; backtick-quoted identifiers on its nested
// bullet lines (stripping a trailing `(...)` call signature) are that
// subpath's documented symbols, until the next top-level bullet.
Expand Down Expand Up @@ -100,6 +101,12 @@ function listExampleFiles(dir: string): string[] {

const manifest = parseSubpathManifest(fs.readFileSync(CLIENT_API_DOC_PATH, 'utf8'));
const exampleFiles = listExampleFiles(EXAMPLES_SDK_DIR);
const packageExports = Object.keys(
(JSON.parse(fs.readFileSync(PACKAGE_JSON_PATH, 'utf8')) as { exports?: Record<string, unknown> })
.exports ?? {},
)
.map((subpath) => (subpath === '.' ? 'agent-device' : `agent-device${subpath.slice(1)}`))
.sort();
const importsByFile = new Map(
exampleFiles.map((file) => [file, extractImportedSymbols(fs.readFileSync(file, 'utf8'))]),
);
Expand All @@ -109,7 +116,15 @@ describe('examples/sdk vs client-api.md drift guard', () => {
assert.ok(
manifest.size > 0,
`${CLIENT_API_DOC_PATH} did not yield a parseable subpath API manifest; ` +
'has the "Public subpath API exposed for Node consumers" list moved or changed format?',
'has the "API reference" entry-point list moved or changed format?',
);
});

test('client-api.md documents every published package entry point', () => {
assert.deepEqual(
[...manifest.keys()].sort(),
packageExports,
`${CLIENT_API_DOC_PATH}'s public subpath manifest must match ${PACKAGE_JSON_PATH}#exports`,
);
});

Expand Down
219 changes: 126 additions & 93 deletions website/docs/docs/_meta.json
Original file line number Diff line number Diff line change
@@ -1,97 +1,130 @@
[
{
"name": "introduction",
"type": "file",
"label": "Introduction"
},
{
"name": "installation",
"type": "file",
"label": "Installation"
},
{
"name": "agent-setup",
"type": "file",
"label": "AI Agent Setup"
},
{
"name": "quick-start",
"type": "file",
"label": "Quick Start"
},
{
"name": "debugging-profiling",
"type": "file",
"label": "Debugging & Profiling"
},
{
"name": "client-api",
"type": "file",
"label": "Typed Client"
},
{
"name": "commands",
"type": "file",
"label": "Commands"
},
{
"name": "configuration",
"type": "file",
"label": "Configuration"
},
{
"name": "remote-proxy",
"type": "file",
"label": "Remote Proxy"
},
{
"name": "device-clouds",
"type": "file",
"label": "Device Clouds"
},
{
"name": "security-trust",
"type": "file",
"label": "Security & Trust"
},
{
"name": "batching",
"type": "file",
"label": "Batching"
},
{
"name": "selectors",
"type": "file",
"label": "Selectors"
},
{
"name": "sessions",
"type": "file",
"label": "Sessions"
},
{
"name": "replay-e2e",
"type": "file",
"label": "Replay & E2E"
},
{
"name": "snapshots",
"type": "file",
"label": "Snapshots"
},
{
"name": "ios-snapshot-elements",
"type": "file",
"label": "iOS Snapshot Elements"
},
{
"name": "known-limitations",
"type": "file",
"label": "Known Limitations"
},
{
"name": "migrating-gestures",
"type": "file",
"label": "Migrating Gestures"
"type": "custom-link",
"label": "Getting Started",
"collapsible": false,
"items": [
{
"type": "custom-link",
"label": "Introduction",
"link": "/docs/introduction"
},
{
"type": "custom-link",
"label": "Installation",
"link": "/docs/installation"
},
{
"type": "custom-link",
"label": "AI Agent Setup",
"link": "/docs/agent-setup"
},
{
"type": "custom-link",
"label": "Quick Start",
"link": "/docs/quick-start"
}
]
},
{
"type": "custom-link",
"label": "Using agent-device",
"collapsible": false,
"items": [
{
"type": "custom-link",
"label": "Commands",
"link": "/docs/commands"
},
{
"type": "custom-link",
"label": "Configuration",
"link": "/docs/configuration"
},
{
"type": "custom-link",
"label": "Selectors",
"link": "/docs/selectors"
},
{
"type": "custom-link",
"label": "Sessions",
"link": "/docs/sessions"
},
{
"type": "custom-link",
"label": "Batching",
"link": "/docs/batching"
},
{
"type": "custom-link",
"label": "Snapshots",
"link": "/docs/snapshots"
},
{
"type": "custom-link",
"label": "Replay & E2E testing",
"link": "/docs/replay-e2e"
},
{
"type": "custom-link",
"label": "Debugging & Profiling",
"link": "/docs/debugging-profiling"
}
]
},
{
"type": "custom-link",
"label": "Integrations",
"collapsible": false,
"items": [
{
"type": "custom-link",
"label": "Node.js API",
"link": "/docs/client-api"
},
{
"type": "custom-link",
"label": "AI SDK",
"link": "/docs/ai-sdk"
},
{
"type": "custom-link",
"label": "Eve",
"link": "/docs/eve"
},
{
"type": "custom-link",
"label": "Remote Proxy",
"link": "/docs/remote-proxy"
},
{
"type": "custom-link",
"label": "Device Clouds",
"link": "/docs/device-clouds"
}
]
},
{
"type": "custom-link",
"label": "Reference",
"collapsible": false,
"items": [
{
"type": "custom-link",
"label": "Security & Trust",
"link": "/docs/security-trust"
},
{
"type": "custom-link",
"label": "Known Limitations",
"link": "/docs/known-limitations"
},
{
"type": "custom-link",
"label": "Migrating Gestures",
"link": "/docs/migrating-gestures"
}
]
}
]
78 changes: 78 additions & 0 deletions website/docs/docs/ai-sdk.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
---
title: AI SDK
---

# AI SDK

[Vercel's AI SDK](https://ai-sdk.dev/) can expose `agent-device` client methods as typed tools in a Node.js agent. `ToolLoopAgent` owns the model loop, while `createAgentDeviceClient()` owns the device session and keeps tool implementations aligned with the CLI command contracts.

Install the integration dependencies:

```bash
pnpm add agent-device ai zod
```

The example below gives the model two deliberately small tools: one for observing the current UI and one for pressing an element returned by that observation.

```ts
import { ToolLoopAgent, tool } from 'ai';
import { createAgentDeviceClient } from 'agent-device';
import { z } from 'zod';

const client = createAgentDeviceClient({
session: 'ai-sdk-agent',
lockPolicy: 'reject',
});

const agent = new ToolLoopAgent({
model: process.env.AI_MODEL!,
instructions: [
'Inspect the current UI before acting.',
'Only press an element ref returned by the latest snapshot.',
'Stop and explain when the requested state cannot be verified.',
].join('\n'),
tools: {
snapshot: tool({
description: 'Return the interactive elements in the current device UI.',
inputSchema: z.object({}),
execute: async () => await client.capture.snapshot({ interactiveOnly: true }),
}),
press: tool({
description: 'Press an element from the latest snapshot by its @e ref.',
inputSchema: z.object({
ref: z.string().regex(/^@e\d+$/),
}),
execute: async ({ ref }) => await client.interactions.press({ ref }),
}),
},
});

try {
await client.apps.open({
app: 'com.example.app',
platform: 'ios',
});

const result = await agent.generate({
prompt: 'Navigate to Notifications and verify that notifications are enabled.',
});

console.log(result.text);
} finally {
await client.sessions.close();
}
```

Set `AI_MODEL` to a model available through your configured AI SDK provider. See the AI SDK references for [`ToolLoopAgent`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool-loop-agent) and [`tool()`](https://ai-sdk.dev/docs/reference/ai-sdk-core/tool).

## Designing device tools

Prefer focused tools over a single tool that accepts an arbitrary command name and arguments:

- Validate tool input with a schema. In particular, constrain element refs to values such as `@e12` and make the model observe before acting.
- Keep the `agent-device` client outside tool execution so calls share one named session.
- Return typed client results directly unless the result needs an application-specific projection.
- Let the host application own session cleanup with `try`/`finally`; do not rely on the model to close the session.
- Use AI SDK's `needsApproval` option for actions that require human confirmation in your product.

See [Node.js API](/docs/client-api) for the complete client surface and runnable, typechecked `agent-device` examples.
Loading
Loading