From cfd8dc196b84fe84d67988d779f95cbbdedad428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 24 Jul 2026 15:26:49 +0200 Subject: [PATCH 1/3] docs: bless the works-today iOS SpringBoard/widget workflow (#1296 PR A) Live probe on iOS 26.2/Xcode 26.2 proved open com.apple.springboard already binds a driveable SpringBoard session with zero code changes: the full widget add/edit/remove flow is selector-driven from a fresh snapshot, aside from two documented coordinate fallbacks. Add a help ios-system-ui topic (and cross-links from physical-device/workflow, the skill router, and docs/commands.md) so agents can use it today, ahead of the --ui-target contract and gallery-capture fix landing. --- skills/agent-device/SKILL.md | 1 + .../parser/__tests__/cli-help-topics.test.ts | 15 +++++++ src/cli/parser/cli-help.ts | 41 ++++++++++++++++++- website/docs/docs/commands.md | 10 +++++ 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index 51c6a6bca2..fe93725bce 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -35,6 +35,7 @@ agent-device help remote agent-device help macos agent-device help dogfood agent-device help tv +agent-device help ios-system-ui # iOS SpringBoard, widgets, and system-UI surfaces ``` Default loop: `open -> snapshot/-i -> get/is/find or press/fill/scroll/wait -> verify -> close`. When target-specific help says capture or selectors are unsupported, use its control-only loop and the device display as visual truth. diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 666e58cb7d..93d446abfd 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -183,6 +183,10 @@ test('usage includes agent workflows, config, environment, and examples footers' usageText, /agent-device help physical-device\s+Use when using a connected phone\/tablet or iOS signing setup/, ); + assert.match( + usageText, + /agent-device help ios-system-ui\s+Use when driving iOS SpringBoard, widgets, or other system-UI surfaces/, + ); assert.match( usageText, /agent-device help react-native\s+Use when the target app is React Native, Expo, or a dev client/, @@ -502,6 +506,17 @@ test('usageForCommand resolves physical-device help topic', async () => { ); }); +test('usageForCommand resolves ios-system-ui help topic', async () => { + const help = await usageForCommand('ios-system-ui'); + if (help === null) throw new Error('Expected ios-system-ui help text'); + assert.match(help, /agent-device help ios-system-ui/); + assert.match(help, /agent-device open com\.apple\.springboard --platform ios/); + assert.match(help, /longpress on an empty area of the home screen/); + assert.match(help, /discover them from the current snapshot/); + assert.match(help, /Do not hard-code Edit\/Done\/Add Widget or other SpringBoard label text/); + assert.match(help, /Reopen the app bundle under test/); +}); + test('usageForCommand resolves manual QA help topic', async () => { const help = await usageForCommand('manual-qa'); if (help === null) throw new Error('Expected manual QA help text'); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 9e5e00df2e..114c24dfb1 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -57,6 +57,10 @@ const AGENT_WORKFLOWS = [ label: 'agent-device help physical-device', description: 'Use when using a connected phone/tablet or iOS signing setup', }, + { + label: 'agent-device help ios-system-ui', + description: 'Use when driving iOS SpringBoard, widgets, or other system-UI surfaces', + }, { label: 'agent-device help remote', description: 'Use when working through cloud config, tenants, leases, or local tunnels', @@ -438,7 +442,8 @@ Escalate: help react-devtools React Native performance, profiling, props/state/hooks, slow renders, rerenders help react-native React Native app automation hazards, overlays, Metro/Re.Pack, and routing help remote remote/cloud config, tenant, lease, local service tunnels - help macos desktop, frontmost-app, menu bar surfaces`, + help macos desktop, frontmost-app, menu bar surfaces + help ios-system-ui iOS SpringBoard, widget add/edit/remove, system-UI surfaces`, }, tv: { summary: 'Android TV, tvOS, and Vega VVD focus-first remote navigation', @@ -842,7 +847,39 @@ iOS physical-device prerequisites: Android physical-device prerequisites: Enable USB debugging and confirm the device appears in agent-device devices --platform android. - Android does not need the iOS runner signing setup. For React Native/Expo Metro reachability, read help react-native.`, + Android does not need the iOS runner signing setup. For React Native/Expo Metro reachability, read help react-native. + +For iOS SpringBoard, widget, or other system-UI surfaces, read agent-device help ios-system-ui.`, + }, + 'ios-system-ui': { + summary: 'iOS SpringBoard, widget, and system-surface workflow', + body: `agent-device help ios-system-ui + +Use this when a task needs iOS SpringBoard (home screen), widget add/edit/remove, or other system-UI surfaces instead of the app under test. + +This works today by opening SpringBoard as the session app; there is no separate widget/system command. System labels vary by iOS version and locale, so discover them from the current snapshot instead of relying on the literal strings shown below. + +Core loop: + 1. Reach the app state you want to prepare (for example, arrange the widget/Live Activity data the app should show) with normal app automation, then agent-device open com.apple.springboard --platform ios. From an existing app session, agent-device home first also lands on the home screen, but open com.apple.springboard is what actually binds the session to SpringBoard for selector-driven commands. + 2. agent-device snapshot -i to read the current localized SpringBoard controls. + 3. agent-device longpress on an empty area of the home screen to enter edit mode. This is the one deliberate coordinate step; there is no reliable non-coordinate way to trigger it. + 4. Re-snapshot and use selectors from the fresh tree to drive the Edit menu -> widget gallery -> search -> size picker -> Add Widget, for example: + agent-device snapshot -i + agent-device press 'label="Add Widget"' + agent-device fill 'label="Search"' "Calendar" + 5. The widget-gallery search-result rows currently fall back to unlabeled nodes (a known capture gap), so tap the result by coordinates read from a screenshot until that is fixed: + agent-device screenshot + agent-device press + 6. Continue with the semantic size picker and agent-device press 'label="Add Widget"' to place it. + 7. To edit or remove an installed widget, longpress it, then re-snapshot and use the fresh context-menu selectors (Edit Widget / Remove Widget). + 8. Use screenshots for visual assertions, and as the fallback wherever a system surface exposes sparse accessibility, not only in the gallery step. + 9. Reopen the app bundle under test (agent-device open --platform ios) to return to normal app automation; leaving SpringBoard bound does not resume the app session on its own. + +Rules: + Do not hard-code Edit/Done/Add Widget or other SpringBoard label text into a plan as a fixed assumption; take them from the latest snapshot -i so the plan survives iOS version/locale differences. + A real system permission alert can appear mid-flow; it composes with this workflow normally, so handle it with alert wait/accept/dismiss or by pressing the visible label like any other step. + Prefer refs/selectors from the fresh snapshot for every step except the two documented coordinate fallbacks (empty-space long-press to enter edit mode, and the gallery search-result tap). + This topic covers what already works by opening SpringBoard as the session app. It does not yet cover keeping an app session open while alternating individual commands against SpringBoard, or Live Activity/Dynamic Island semantics; those land separately.`, }, remote: { summary: 'Direct proxy, cloud profiles, and remote config', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 6476b41ba6..68dff92a76 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -20,6 +20,7 @@ agent-device help remote agent-device help web agent-device help macos agent-device help dogfood +agent-device help ios-system-ui ``` Skills are recommended for auto-routing when your agent runtime supports them, but they are not required. The CLI help topics are the version-matched operating contract. @@ -1037,3 +1038,12 @@ For CLI-discoverable setup guidance, run `agent-device help physical-device`. - If you override the iOS runner derived-data path and also force cleanup, keep `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` under the project `.tmp/` directory. Other cleanup override paths are rejected with a recovery hint. - For daemon startup troubleshooting: - follow stale metadata hints for `/daemon.json` and `/daemon.lock` (`state-dir` defaults to `~/.agent-device` for packaged installs, or a worktree-scoped dir under `~/.agent-device/dev/` from source) + +## iOS SpringBoard, widgets, and system-UI surfaces + +For CLI-discoverable workflow guidance, run `agent-device help ios-system-ui`. + +- `agent-device open com.apple.springboard --platform ios` binds the session to SpringBoard on a simulator or physical device today; there is no separate widget/system command. +- The full widget add/edit/remove flow is selector-driven from a fresh `snapshot -i`, except two coordinate-based steps: the empty-space long-press that enters edit mode, and (until fixed) the widget-gallery search-result rows, which currently return unlabeled accessibility nodes. +- SpringBoard labels vary by iOS version and locale; discover them from the current snapshot rather than hard-coding strings like `Edit` or `Add Widget`. +- Reopen the app bundle under test to return to normal app automation after a SpringBoard step. From 41157ea157e4205da3c3c9a7c4374fa13ac5117f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 13:37:17 +0200 Subject: [PATCH 2/3] docs: scope the SpringBoard claim to verified iOS simulator support #1296 explicitly leaves physical-iPhone SpringBoard unverified; the only evidence so far is an iPhone simulator run, whose private-AX fallback is simulator-only. Applied to both the CLI help topic and website/docs/docs/commands.md, with a link to the tracking issue. Dropped the SkillGym case from this PR per thymikee: most of that harness is being removed in #1411, so it's not worth iterating on here. --- src/cli/parser/__tests__/cli-help-topics.test.ts | 4 ++++ src/cli/parser/cli-help.ts | 2 +- website/docs/docs/commands.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 93d446abfd..3ee8e4f9f3 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -513,6 +513,10 @@ test('usageForCommand resolves ios-system-ui help topic', async () => { assert.match(help, /agent-device open com\.apple\.springboard --platform ios/); assert.match(help, /longpress on an empty area of the home screen/); assert.match(help, /discover them from the current snapshot/); + assert.match( + help, + /verified on iOS simulator; physical-iPhone SpringBoard support is not yet verified/, + ); assert.match(help, /Do not hard-code Edit\/Done\/Add Widget or other SpringBoard label text/); assert.match(help, /Reopen the app bundle under test/); }); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 114c24dfb1..710f8b8f17 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -857,7 +857,7 @@ For iOS SpringBoard, widget, or other system-UI surfaces, read agent-device help Use this when a task needs iOS SpringBoard (home screen), widget add/edit/remove, or other system-UI surfaces instead of the app under test. -This works today by opening SpringBoard as the session app; there is no separate widget/system command. System labels vary by iOS version and locale, so discover them from the current snapshot instead of relying on the literal strings shown below. +This works today by opening SpringBoard as the session app; there is no separate widget/system command. System labels vary by iOS version and locale, so discover them from the current snapshot instead of relying on the literal strings shown below. This workflow is verified on iOS simulator; physical-iPhone SpringBoard support is not yet verified. Core loop: 1. Reach the app state you want to prepare (for example, arrange the widget/Live Activity data the app should show) with normal app automation, then agent-device open com.apple.springboard --platform ios. From an existing app session, agent-device home first also lands on the home screen, but open com.apple.springboard is what actually binds the session to SpringBoard for selector-driven commands. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 68dff92a76..3b6e1f2727 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1043,7 +1043,7 @@ For CLI-discoverable setup guidance, run `agent-device help physical-device`. For CLI-discoverable workflow guidance, run `agent-device help ios-system-ui`. -- `agent-device open com.apple.springboard --platform ios` binds the session to SpringBoard on a simulator or physical device today; there is no separate widget/system command. +- `agent-device open com.apple.springboard --platform ios` binds the session to SpringBoard today; this is verified on iOS simulator only. Physical-iPhone SpringBoard support is not yet verified — see [#1296](https://github.com/callstack/agent-device/issues/1296). - The full widget add/edit/remove flow is selector-driven from a fresh `snapshot -i`, except two coordinate-based steps: the empty-space long-press that enters edit mode, and (until fixed) the widget-gallery search-result rows, which currently return unlabeled accessibility nodes. - SpringBoard labels vary by iOS version and locale; discover them from the current snapshot rather than hard-coding strings like `Edit` or `Add Widget`. - Reopen the app bundle under test to return to normal app automation after a SpringBoard step. From f482975610efbf1a64f6b750556f0dc0f05761d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 13:40:14 +0200 Subject: [PATCH 3/3] fix(test): keep help ios-system-ui out of the 30-line first-screen budget #1404 added a benchmark gate requiring every Agent Workflows pointer to stay within the first 30 lines of bare `agent-device help` output. Adding ios-system-ui to that list pushed help macos to line 31. Drop the AGENT_WORKFLOWS entry; the topic stays fully reachable via its cross-references from help physical-device, help workflow's Escalate section, the agent-device skill router, and website/docs/docs/commands.md. --- src/cli/parser/__tests__/cli-help-topics.test.ts | 4 ---- src/cli/parser/cli-help.ts | 4 ---- 2 files changed, 8 deletions(-) diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 3ee8e4f9f3..576ddf97d1 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -183,10 +183,6 @@ test('usage includes agent workflows, config, environment, and examples footers' usageText, /agent-device help physical-device\s+Use when using a connected phone\/tablet or iOS signing setup/, ); - assert.match( - usageText, - /agent-device help ios-system-ui\s+Use when driving iOS SpringBoard, widgets, or other system-UI surfaces/, - ); assert.match( usageText, /agent-device help react-native\s+Use when the target app is React Native, Expo, or a dev client/, diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 710f8b8f17..3f109b61f2 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -57,10 +57,6 @@ const AGENT_WORKFLOWS = [ label: 'agent-device help physical-device', description: 'Use when using a connected phone/tablet or iOS signing setup', }, - { - label: 'agent-device help ios-system-ui', - description: 'Use when driving iOS SpringBoard, widgets, or other system-UI surfaces', - }, { label: 'agent-device help remote', description: 'Use when working through cloud config, tenants, leases, or local tunnels',