From 7bc9b3516a8090bd5c83206282e990935e2b01c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 11 Jun 2026 16:48:21 +0200 Subject: [PATCH 1/3] refactor(ios): snapshot capture plans with a structured quality verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements ADR 0004's explicit-strategies decision as architecture (candidates 1+2 of the snapshot pipeline review): - Snapshot backend seam: three adapters (recursive tree, query sweep, private AX) behind one captureWithBackend dispatch. Each strategy declares its chain as data (regular: tree→queries→private-ax, compact: queries→private-ax, raw: tree→private-ax) and one plan runner walks it under a 20s umbrella budget so chained recovery can never stack past the 30s main-thread watchdog. Terminal policy is per-plan: raw rethrows AX failures (diagnostics preserve errors), interactive fails closed with runnerFatal invalidation. - Single quality classifier: one sparsePayloadReason predicate (with reason codes), one collapsed-leaf detector, replacing the three divergent sparse detectors (Swift structural, daemon count==1, CLI count<=3) that each patched a different failure shape. - Structured snapshot quality verdict on the wire (state, backend, reason, reasonCode, effectiveDepth, collapsedLeafIndexes): the daemon and CLI render warnings from it instead of re-deriving degradation from node shapes; budget starvation is no longer blamed on the app's accessibility. Legacy runner messages and daemon-side detectors stay behind a verdict-absent gate for mixed-version compat. - The verdict surfaces in --json (snapshotQuality) for agents; the generic sparse CLI hint is suppressed when a verdict explains it. Threading the verdict exposed two more hand-copy field drops (captureInteractionOutcomeAwareSnapshot, serializeSnapshotResult, client response mapping) - now carried alongside warnings everywhere. Verified live: Settings healthy (tree, no warnings), Settings compact under load (recovered/private-ax/budget), production login (sparse best-effort with honest warning), collapse fixture (healthy + collapsedLeafIndexes -> @ref warning), Bluesky Home (recovered/ private-ax, 24 nodes in 2s). Full unit suite 2327 passed, fallow clean, runner builds. --- CONTEXT.md | 2 + .../adr/0004-ios-snapshot-backend-strategy.md | 4 +- .../RunnerTests+AXSnapshotFallback.swift | 25 +- .../RunnerTests+Models.swift | 3 + .../RunnerTests+Snapshot.swift | 462 ++---------------- .../RunnerTests+SnapshotCapturePlan.swift | 362 ++++++++++++++ src/__tests__/runtime-snapshot.test.ts | 29 ++ src/backend.ts | 2 + src/client-shared.ts | 1 + src/client-types.ts | 2 + src/client.ts | 8 +- src/commands/capture-snapshot.ts | 17 +- src/core/interactors/apple.ts | 13 +- src/daemon/handlers/snapshot-capture.ts | 7 + src/daemon/snapshot-runtime.ts | 2 + src/utils/output.ts | 7 +- src/utils/snapshot-quality.ts | 90 ++++ 17 files changed, 599 insertions(+), 437 deletions(-) create mode 100644 ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift create mode 100644 src/utils/snapshot-quality.ts diff --git a/CONTEXT.md b/CONTEXT.md index 020b2307d1..e04c377cb3 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -17,6 +17,8 @@ - Daemon command registry: daemon-side source of truth for command route ownership and request-policy traits, including admission exemptions, session locking, selector validation, replay-scoped actions, recording invalidation, Android dialog guards, and request provider device resolution. - Runner command traits: the iOS XCTest runner's per-command-type classification across three independent axes — interaction (gates the foreground-guard and stabilization preflight), read-only (gates the session-invalidating retry; the alert command is read-only only for its `get` action), and runner-lifecycle (skips the app-activation preflight). One source of truth keyed by command type, distinct from the public command surface and daemon command registry. - Coordinate-first resolved element activation: iOS/macOS runner interaction pattern where a selector or text query resolves the semantic `XCUIElement`, then activation uses the element's resolved center coordinate when a frame is available. This keeps target selection semantic while avoiding `XCUIElement.tap()` post-action element re-resolution after normal navigation. tvOS remains focus/remote-driven. +- Snapshot capture plan: per-strategy ordered chain of iOS snapshot capture backends (recursive tree, query sweep, private AX) run by one plan runner under a shared wall-clock budget; recovery ordering is declared data, never a per-call-site branch. +- Snapshot quality verdict: structured outcome (state, backend, reason code, effective depth, collapsed leaves) computed once by the plan runner and shipped with every planned snapshot payload; the daemon and CLI render it instead of re-deriving degradation from node shapes. - AX-unavailable target invalidation: iOS/macOS runner behavior where a root accessibility snapshot failure such as `kAXErrorIllegalArgument` marks the cached `XCUIApplication` target handle suspect. The runner fails closed for degraded interactive snapshots, clears the cached target, and lets the next command reacquire the app through normal activation. ## Testing Principles diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index b3ced94848..428accc893 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -2,7 +2,9 @@ ## Status -Accepted +Accepted — implemented by the snapshot capture plan runner (RunnerTests+SnapshotCapturePlan.swift): +each strategy declares its backend chain, and a structured snapshot quality verdict makes +degraded or recovered output observable end to end. ## Context diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 671fbb4157..d8774f924e 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -10,11 +10,10 @@ extension RunnerTests { /// apps where the AX surface is genuinely unavailable. static let privateAXSnapshotDepthLadder = [56, 40, 24, 12] - func privateAXSnapshotFallback( + func privateAXSnapshotCapture( app: XCUIApplication, - options: SnapshotOptions, - reason: String - ) -> DataPayload? { + options: SnapshotOptions + ) -> SnapshotBackendCapture? { #if os(iOS) && targetEnvironment(simulator) let requestedDepth = options.depth ?? 64 var attemptDepths = [requestedDepth] @@ -74,21 +73,15 @@ extension RunnerTests { } let depthLimited = effectiveDepth < requestedDepth - let truncated = (response["truncated"] as? Bool) == true || depthLimited - var message = - "Recovered this snapshot with the fallback accessibility backend after \(reason). This usually means the app publishes an unhealthy accessibility tree (too large or deep to serialize, or containers that hide their children) — fixing the app's accessibility is the real cure. The fallback is simulator-only and may expose a partial tree; treat screenshot as visual truth when this warning appears." - if depthLimited { - message += - " The accessibility server rejected deeper requests; this tree is capped at depth \(effectiveDepth) — re-run with --depth \(effectiveDepth) --scope to inspect deeper content." - } NSLog( - "AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_USED reason=%@ nodes=%ld depth=%ld truncated=%@", - reason, + "AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_USED nodes=%ld depth=%ld", nodes.count, - effectiveDepth, - truncated ? "true" : "false" + effectiveDepth + ) + return SnapshotBackendCapture( + payload: DataPayload(nodes: nodes, truncated: (response["truncated"] as? Bool) == true), + effectiveDepth: depthLimited ? effectiveDepth : nil ) - return DataPayload(message: message, nodes: nodes, truncated: truncated) #else return nil #endif diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 726a451a50..7e329c6001 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -205,6 +205,7 @@ struct DataPayload: Codable { let items: [String]? let nodes: [SnapshotNode]? let truncated: Bool? + let snapshotQuality: SnapshotQuality? let gestureStartUptimeMs: Double? let gestureEndUptimeMs: Double? let x: Double? @@ -242,6 +243,7 @@ struct DataPayload: Codable { items: [String]? = nil, nodes: [SnapshotNode]? = nil, truncated: Bool? = nil, + snapshotQuality: SnapshotQuality? = nil, gestureStartUptimeMs: Double? = nil, gestureEndUptimeMs: Double? = nil, x: Double? = nil, @@ -278,6 +280,7 @@ struct DataPayload: Codable { self.items = items self.nodes = nodes self.truncated = truncated + self.snapshotQuality = snapshotQuality self.gestureStartUptimeMs = gestureStartUptimeMs self.gestureEndUptimeMs = gestureEndUptimeMs self.x = x diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index d3c2c773b1..d695cddfab 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -11,29 +11,7 @@ extension RunnerTests { private static let rawSnapshotMaxNodes = 5_000 private static let rawSnapshotTooLargeHint = "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree." - private static let publicQueryRecoveryMessage = - "Recovered iOS snapshot through XCTest accessibility element queries after the public snapshot tree was sparse. This usually means the app publishes an unhealthy accessibility tree - fixing the app accessibility is the real cure. The recovered nodes are a flattened view of on-screen controls; treat screenshot as visual truth when this warning appears." - private static let structuralOnlyNodeTypes: Set = [ - "Application", - "Window", - "Other", - "ScrollView" - ] - private static let collapsedTabCandidateTypes: Set = [ - .button, - .link, - .menuItem, - .other, - .staticText - ] - static let scrollContainerTypes: Set = [ - .collectionView, - .scrollView, - .table - ] - private static let flatInteractiveFallbackBudget: TimeInterval = 1.0 - - private struct SnapshotTraversalContext { + struct SnapshotTraversalContext { let queryRoot: XCUIElement let rootSnapshot: XCUIElementSnapshot let viewport: CGRect @@ -52,12 +30,6 @@ extension RunnerTests { let visible: Bool } - private enum SnapshotTraversalCapture { - case context(SnapshotTraversalContext) - case fallback(DataPayload) - case empty - } - struct SnapshotCaptureFailure: Error { let code: String let message: String @@ -106,35 +78,48 @@ extension RunnerTests { } } + static let structuralOnlyNodeTypes: Set = [ + "Application", + "Window", + "Other", + "ScrollView" + ] + + private static let collapsedTabCandidateTypes: Set = [ + .button, + .link, + .menuItem, + .other, + .staticText + ] + + static let scrollContainerTypes: Set = [ + .collectionView, + .scrollView, + .table + ] + + private static let flatInteractiveFallbackBudget: TimeInterval = 1.0 + func snapshotFast(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload { if let blocking = blockingSystemAlertSnapshot() { return blocking } - if options.interactiveOnly && options.compact { - let payload = snapshotFlatInteractive(app: app, options: options) - return snapshotWithPrivateAXFallbackIfSparse( - payload, - app: app, - options: options, - reason: "compact interactive XCTest snapshot was sparse" - ) - } - - let capture = try captureSnapshotTraversalContext( + let plan = options.interactiveOnly && options.compact + ? Self.compactInteractivePlan + : Self.regularVisiblePlan + return try runSnapshotCapturePlan( + plan, app: app, options: options, - allowInteractiveUnavailableFallback: true + terminal: .sparseWithFatalOnAXFailure ) - let context: SnapshotTraversalContext - switch capture { - case .context(let traversalContext): - context = traversalContext - case .fallback(let fallback): - return fallback - case .empty: - return DataPayload(nodes: [], truncated: false) - } + } + func recursiveTreeSnapshotPayload( + context: SnapshotTraversalContext, + options: SnapshotOptions + ) -> DataPayload { var cachedDescendantElements: [XCUIElement]? func collapsedTabDescendants() -> [XCUIElement] { if let cachedDescendantElements { @@ -257,150 +242,28 @@ extension RunnerTests { } - let payload = DataPayload( + return DataPayload( nodes: applyHiddenContentHints(hiddenContentHintsByNodeIndex, to: nodes), truncated: false ) - return snapshotWithFallbackIfSparse( - payload, - app: app, - options: options, - reason: "XCTest snapshot returned a sparse application/window tree" - ) - } - - private func snapshotWithFallbackIfSparse( - _ payload: DataPayload, - app: XCUIApplication, - options: SnapshotOptions, - reason: String - ) -> DataPayload { - guard Self.snapshotPayloadNeedsRecovery(payload) else { - return payload - } - if let fallback = publicQuerySnapshotFallback( - app: app, - options: options, - reason: reason - ) { - return fallback - } - return betterSnapshotPayload( - payload, - recovered: privateAXSnapshotFallback(app: app, options: options, reason: reason) - ) - } - - private func snapshotWithPrivateAXFallbackIfSparse( - _ payload: DataPayload, - app: XCUIApplication, - options: SnapshotOptions, - reason: String - ) -> DataPayload { - guard Self.snapshotPayloadNeedsRecovery(payload) else { - return payload - } - return betterSnapshotPayload( - payload, - recovered: privateAXSnapshotFallback(app: app, options: options, reason: reason) - ) - } - - /// A payload needs recovery when the tree is structural-only, OR when the capture was cut - /// off by a budget/deadline with almost nothing collected. The second condition matters on - /// large React Native trees: the typed-query sweep can resolve one or two stray controls - /// before its deadline, which defeats an all-structural check while the payload is still - /// useless in practice. A legitimately minimal screen finishes the sweep without truncation, - /// so it never pays for recovery. - static let sparseRecoveryTruncatedNodeThreshold = 8 - - static func snapshotPayloadNeedsRecovery(_ payload: DataPayload) -> Bool { - guard let nodes = payload.nodes, !nodes.isEmpty else { return false } - if isSparseApplicationWindowTree(nodes) { return true } - return payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold - } - - /// Keeps the original payload unless the recovered tree actually carries more nodes — - /// recovery must never replace a partial-but-real capture with something thinner. - private func betterSnapshotPayload( - _ payload: DataPayload, - recovered: DataPayload? - ) -> DataPayload { - guard let recovered, let recoveredNodes = recovered.nodes, - recoveredNodes.count > (payload.nodes?.count ?? 0) - else { - return payload - } - return recovered - } - - private static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool { - guard !nodes.isEmpty else { return false } - return nodes.allSatisfy { node in - // Application/Window labels are just the app/window name, and full-screen roots - // compute as hittable; neither says anything about tree health, so neither counts - // as content for these types (a labeled app+window pair is still a sparse tree). - let isRootContainer = node.type == "Application" || node.type == "Window" - let hasContent = (!isRootContainer && node.label?.isEmpty == false) - || node.identifier?.isEmpty == false - || node.value?.isEmpty == false - return !hasContent - && (isRootContainer || !node.hittable) - && Self.structuralOnlyNodeTypes.contains(node.type) - } - } - - private func publicQuerySnapshotFallback( - app: XCUIApplication, - options: SnapshotOptions, - reason: String - ) -> DataPayload? { - let fallback = snapshotFlatInteractive( - app: app, - options: SnapshotOptions( - interactiveOnly: false, - compact: options.compact, - depth: options.depth, - scope: options.scope, - raw: false - ) - ) - guard let nodes = fallback.nodes, !Self.isSparseApplicationWindowTree(nodes) else { - return nil - } - NSLog( - "AGENT_DEVICE_RUNNER_PUBLIC_QUERY_SNAPSHOT_USED reason=%@ nodes=%ld truncated=%@", - reason, - nodes.count, - fallback.truncated == true ? "true" : "false" - ) - return DataPayload( - message: Self.publicQueryRecoveryMessage, - nodes: nodes, - truncated: true - ) } func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload { if let blocking = blockingSystemAlertSnapshot() { return blocking } - - let capture = try captureSnapshotTraversalContext( + return try runSnapshotCapturePlan( + Self.rawDiagnosticPlan, app: app, options: options, - allowInteractiveUnavailableFallback: false + terminal: .throwOnAXFailure ) - let context: SnapshotTraversalContext - switch capture { - case .context(let traversalContext): - context = traversalContext - case .fallback(let fallback): - return fallback - case .empty: - return DataPayload(nodes: [], truncated: false) - } + } + func rawTreeSnapshotPayload( + context: SnapshotTraversalContext, + options: SnapshotOptions + ) throws -> DataPayload { var nodes: [SnapshotNode] = [] func walk(_ snapshot: XCUIElementSnapshot, depth: Int, parentIndex: Int?) throws { @@ -439,15 +302,10 @@ extension RunnerTests { } try walk(context.rootSnapshot, depth: 0, parentIndex: nil) - return snapshotWithPrivateAXFallbackIfSparse( - DataPayload(nodes: nodes, truncated: false), - app: app, - options: options, - reason: "XCTest raw snapshot returned a sparse application/window tree" - ) + return DataPayload(nodes: nodes, truncated: false) } - private func snapshotFlatInteractive(app: XCUIApplication, options: SnapshotOptions) -> DataPayload { + func snapshotFlatInteractive(app: XCUIApplication, options: SnapshotOptions) -> DataPayload { var nodes: [SnapshotNode] = [ compactInteractiveRootNode(rect: .zero) ] @@ -517,7 +375,7 @@ extension RunnerTests { return DataPayload(nodes: nodes, truncated: truncated) } - private func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { + func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) return sparseTruncatedSnapshotPayload( @@ -527,77 +385,6 @@ extension RunnerTests { ) } - private func captureSnapshotTraversalContext( - app: XCUIApplication, - options: SnapshotOptions, - allowInteractiveUnavailableFallback: Bool - ) throws -> SnapshotTraversalCapture { - do { - guard let context = try makeSnapshotTraversalContext(app: app, options: options) else { - return .empty - } - return .context(context) - } catch let failure as SnapshotCaptureFailure { - if Self.isAxSnapshotFailure(failure), - let fallback = privateAXSnapshotFallback( - app: app, - options: options, - reason: failure.message - ) - { - return .fallback(fallback) - } - if let fallback = snapshotDepthLimitedAccessibilityFallback( - app: app, - options: options, - failure: failure - ) { - return .fallback(fallback) - } - if allowInteractiveUnavailableFallback && options.interactiveOnly { - return .fallback(snapshotAccessibilityUnavailable(failure: failure)) - } - throw failure - } - } - - private func snapshotDepthLimitedAccessibilityFallback( - app: XCUIApplication, - options: SnapshotOptions, - failure: SnapshotCaptureFailure - ) -> DataPayload? { - guard let requestedDepth = options.depth else { - return nil - } - - NSLog( - "AGENT_DEVICE_RUNNER_SNAPSHOT_DEPTH_FALLBACK=%@", - failure.message - ) - - if requestedDepth <= 0 { - return sparseTruncatedSnapshotPayload(message: recoveredSnapshotMessage(failure)) - } - - // Raw depth-limited recovery intentionally falls back to sparse interactive discovery because - // the raw AX tree is the failed operation. - let fallback = snapshotFlatInteractive( - app: app, - options: SnapshotOptions( - interactiveOnly: true, - compact: options.compact, - depth: requestedDepth, - scope: options.scope, - raw: false - ) - ) - return DataPayload( - message: recoveredSnapshotMessage(failure), - nodes: fallback.nodes, - truncated: true - ) - } - private func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String { return "\(failure.message) Hint: \(failure.hint)" } @@ -610,8 +397,8 @@ extension RunnerTests { ) } - private func sparseTruncatedSnapshotPayload( - message: String, + func sparseTruncatedSnapshotPayload( + message: String? = nil, runnerFatal: Bool? = nil, runnerFatalReason: String? = nil ) -> DataPayload { @@ -659,120 +446,6 @@ extension RunnerTests { XCTAssertTrue(message.contains(Self.axSnapshotHint)) } - func testSparseApplicationWindowTreeDetectionIsConservative() { - let root = compactInteractiveRootNode(rect: .zero) - func node( - index: Int, - type: String, - label: String? = nil, - identifier: String? = nil, - value: String? = nil, - hittable: Bool = false - ) -> SnapshotNode { - SnapshotNode( - index: index, - type: type, - label: label, - identifier: identifier, - value: value, - rect: snapshotRect(from: .zero), - enabled: true, - focused: nil, - selected: nil, - hittable: hittable, - depth: 1, - parentIndex: 0, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - let window = node(index: 1, type: "Window") - let structuralOther = node(index: 2, type: "Other") - let structuralScroll = node(index: 3, type: "ScrollView") - let labeledOther = node(index: 4, type: "Other", label: "Visible content") - let identifiedOther = node(index: 5, type: "Other", identifier: "test-id") - let valuedOther = node(index: 6, type: "Other", value: "Selected") - let hittableOther = node(index: 7, type: "Other", hittable: true) - let button = node( - index: 8, - type: "Button", - label: "Sign in", - hittable: true - ) - - XCTAssertTrue(Self.isSparseApplicationWindowTree([root])) - XCTAssertTrue(Self.isSparseApplicationWindowTree([root, window])) - XCTAssertTrue(Self.isSparseApplicationWindowTree([root, window, structuralOther, structuralScroll])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, labeledOther])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, identifiedOther])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, valuedOther])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, hittableOther])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, button])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, window, button])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([])) - // App/window name labels and full-screen-root hittability are not content: a labeled, - // hittable Application root over a bare Window is still a sparse tree (a shape seen on - // production React Native login screens behind full-screen modal overlays). - let labeledHittableRoot = node( - index: 0, type: "Application", label: "Example App", hittable: true) - XCTAssertTrue(Self.isSparseApplicationWindowTree([labeledHittableRoot, window])) - XCTAssertFalse( - Self.isSparseApplicationWindowTree([ - labeledHittableRoot, node(index: 1, type: "Application", identifier: "custom-id"), - ]) - ) - } - - func testSnapshotPayloadNeedsRecoveryOnDeadlineTruncatedNearEmptySweep() { - let root = compactInteractiveRootNode(rect: .zero) - func node(index: Int, label: String) -> SnapshotNode { - SnapshotNode( - index: index, - type: "Button", - label: label, - identifier: nil, - value: nil, - rect: snapshotRect(from: .zero), - enabled: true, - focused: nil, - selected: nil, - hittable: true, - depth: 1, - parentIndex: 0, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - let button = node(index: 1, label: "Home") - - // Deadline-truncated sweep with a stray control: still needs recovery. - XCTAssertTrue( - Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: [root, button], truncated: true)) - ) - // Structural-only tree needs recovery regardless of truncation. - XCTAssertTrue( - Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: [root], truncated: false)) - ) - // A completed sweep on a legitimately minimal screen does not. - XCTAssertFalse( - Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: [root, button], truncated: false)) - ) - // A truncated but reasonably populated sweep does not. - var populated: [SnapshotNode] = [root] - for index in 1...Self.sparseRecoveryTruncatedNodeThreshold { - populated.append(node(index: index, label: "b\(index)")) - } - XCTAssertFalse( - Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: populated, truncated: true)) - ) - XCTAssertFalse(Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: [], truncated: true))) - } - - func testPublicQueryRecoveryMessageExplainsFlattenedFallback() { - XCTAssertTrue(Self.publicQueryRecoveryMessage.contains("XCTest accessibility element queries")) - XCTAssertTrue(Self.publicQueryRecoveryMessage.contains("flattened")) - } - func testRawSnapshotTooLargeFailureIsStructured() { let failure = rawSnapshotTooLargeFailure(nodeCount: Self.rawSnapshotMaxNodes + 1) @@ -781,39 +454,6 @@ extension RunnerTests { XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint) } - func testDepthLimitedSnapshotFailureReturnsNonFatalFallback() { - currentApp = app - currentBundleId = "com.example.app" - - let payload = snapshotDepthLimitedAccessibilityFallback( - app: app, - options: SnapshotOptions( - interactiveOnly: false, - compact: false, - depth: 0, - scope: nil, - raw: false - ), - failure: SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: "\(Self.axSnapshotFailureMessage) kAXErrorIllegalArgument.", - hint: Self.axSnapshotHint - ) - ) - - XCTAssertEqual( - payload?.message, - "\(Self.axSnapshotFailureMessage) kAXErrorIllegalArgument. Hint: \(Self.axSnapshotHint)" - ) - XCTAssertEqual(payload?.nodes?.count, 1) - XCTAssertEqual(payload?.nodes?.first?.type, "Application") - XCTAssertEqual(payload?.truncated, true) - XCTAssertNil(payload?.runnerFatal) - XCTAssertNil(payload?.runnerFatalReason) - XCTAssertNotNil(currentApp) - XCTAssertEqual(currentBundleId, "com.example.app") - } - private func compactInteractiveRootNode(rect: CGRect) -> SnapshotNode { SnapshotNode( index: 0, @@ -909,7 +549,7 @@ extension RunnerTests { return true } - private func makeSnapshotTraversalContext( + func makeSnapshotTraversalContext( app: XCUIApplication, options: SnapshotOptions ) throws -> SnapshotTraversalContext? { @@ -985,7 +625,7 @@ extension RunnerTests { || (normalized.contains("illegal argument") && normalized.contains("snapshot")) } - private static func isAxSnapshotFailure(_ failure: SnapshotCaptureFailure) -> Bool { + static func isAxSnapshotFailure(_ failure: SnapshotCaptureFailure) -> Bool { failure.code == Self.axSnapshotErrorCode || isAxIllegalArgument(failure.message) } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift new file mode 100644 index 0000000000..5ac9044ea8 --- /dev/null +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -0,0 +1,362 @@ +import XCTest + +// MARK: - Snapshot capture plans (ADR 0004) +// +// Each snapshot strategy declares an ordered chain of capture backends. One runner walks the +// chain: capture, classify, accept the first payload the quality classifier calls usable, and +// stamp the outcome with a structured quality verdict so the daemon renders state instead of +// re-deriving it from node shapes. Recovery ordering is data here, never a per-call-site branch. + +/// Structured quality verdict shipped with every iOS snapshot payload. +struct SnapshotQuality: Codable { + /// healthy: first backend produced a usable tree. recovered: a later backend did. + /// sparse: no backend produced a usable tree; the best attempt is returned as-is. + let state: String + /// Backend that produced the returned payload: tree | queries | private-ax. + let backend: String + /// Why recovery ran (first failure) or why the payload is degraded. + let reason: String? + /// Machine-readable reason: ax-rejected | sparse-tree | budget | no-nodes. + let reasonCode: String? + /// Private AX ladder cap when the accepted tree is shallower than requested. + let effectiveDepth: Int? + /// Leaves that merge many labels — a container marked accessible hides its descendants. + let collapsedLeafIndexes: [Int]? +} + +enum SnapshotBackendKind: String, CaseIterable { + case recursiveTree = "tree" + case querySweep = "queries" + case privateAX = "private-ax" +} + +/// What the plan runner does when every backend failed or stayed sparse. +enum SnapshotCaptureTerminalPolicy { + /// Return the best sparse payload; if the tree backend hit a real AX serialization failure + /// on an interactive request, fail closed: invalidate the cached target and mark runnerFatal + /// (AX-unavailable target invalidation, CONTEXT.md). + case sparseWithFatalOnAXFailure + /// Re-throw the tree backend's AX failure (raw diagnostics preserve errors, ADR 0004). + case throwOnAXFailure +} + +struct SnapshotBackendCapture { + let payload: DataPayload + /// Set by the private AX backend when the ladder accepted a shallower depth than requested. + let effectiveDepth: Int? +} + +extension RunnerTests { + static let sparseRecoveryTruncatedNodeThreshold = 8 + /// Umbrella wall-clock budget for one capture plan. Individual backends bound themselves, + /// but chained recovery tiers must never stack past the 30s main-thread watchdog: when the + /// budget is spent, remaining tiers are skipped and the best payload so far is returned. + static let snapshotPlanBudget: TimeInterval = 20 + static let collapsedLeafMinimumSegments = 10 + + static func payloadNodeCount(_ payload: DataPayload?) -> Int { + payload?.nodes?.count ?? 0 + } + + // MARK: Plan definitions + + static let regularVisiblePlan: [SnapshotBackendKind] = [.recursiveTree, .querySweep, .privateAX] + static let compactInteractivePlan: [SnapshotBackendKind] = [.querySweep, .privateAX] + static let rawDiagnosticPlan: [SnapshotBackendKind] = [.recursiveTree, .privateAX] + + // MARK: Plan runner + + func runSnapshotCapturePlan( + _ plan: [SnapshotBackendKind], + app: XCUIApplication, + options: SnapshotOptions, + terminal: SnapshotCaptureTerminalPolicy + ) throws -> DataPayload { + var best: (kind: SnapshotBackendKind, capture: SnapshotBackendCapture)? + var firstFailure: (reason: String, code: String)? + var axFailure: SnapshotCaptureFailure? + let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) + + for kind in plan { + if kind != plan.first && Date() >= deadline { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_PLAN_BUDGET_EXHAUSTED skipped=%@", kind.rawValue) + if firstFailure == nil { + firstFailure = ("the capture plan ran out of its time budget", "budget") + } + break + } + let capture: SnapshotBackendCapture + do { + guard let result = try captureWithBackend(kind, app: app, options: options) else { + continue + } + capture = result + } catch let failure as SnapshotCaptureFailure { + if Self.isAxSnapshotFailure(failure) { axFailure = failure } + if firstFailure == nil { + firstFailure = (failure.message, Self.isAxSnapshotFailure(failure) ? "ax-rejected" : "capture-failed") + } + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_BACKEND_FAILED backend=%@ error=%@", + kind.rawValue, + failure.message + ) + continue + } + + if let sparseReason = Self.sparsePayloadReason(capture.payload) { + if firstFailure == nil { firstFailure = sparseReason } + if Self.payloadNodeCount(capture.payload) > Self.payloadNodeCount(best?.capture.payload) { + best = (kind, capture) + } + continue + } + + let recovered = kind != plan.first + if recovered { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_RECOVERED backend=%@ reason=%@", + kind.rawValue, + firstFailure?.reason ?? "sparse tree" + ) + } + return stampedSnapshotPayload( + capture, + backend: kind, + state: recovered ? "recovered" : "healthy", + reason: recovered ? firstFailure : nil + ) + } + + if let axFailure { + switch terminal { + case .throwOnAXFailure: + throw axFailure + case .sparseWithFatalOnAXFailure where options.interactiveOnly && best == nil: + return snapshotAccessibilityUnavailable(failure: axFailure) + default: + break + } + } + + let fallbackPayload = + best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: "sparse", reason: firstFailure) } + ?? stampedSnapshotPayload( + SnapshotBackendCapture(payload: sparseTruncatedSnapshotPayload(), effectiveDepth: nil), + backend: plan.last ?? .recursiveTree, + state: "sparse", + reason: firstFailure + ) + return fallbackPayload + } + + private func captureWithBackend( + _ kind: SnapshotBackendKind, + app: XCUIApplication, + options: SnapshotOptions + ) throws -> SnapshotBackendCapture? { + switch kind { + case .recursiveTree: + guard let context = try makeSnapshotTraversalContext(app: app, options: options) else { + return nil + } + let payload = options.raw + ? try rawTreeSnapshotPayload(context: context, options: options) + : recursiveTreeSnapshotPayload(context: context, options: options) + return SnapshotBackendCapture(payload: payload, effectiveDepth: nil) + case .querySweep: + return SnapshotBackendCapture( + payload: snapshotFlatInteractive(app: app, options: options), + effectiveDepth: nil + ) + case .privateAX: + return privateAXSnapshotCapture(app: app, options: options) + } + } + + // MARK: Quality classifier (the single source of "is this snapshot degraded") + + /// Returns a degradation reason + machine code when the payload is too degraded to accept. + static func sparsePayloadReason(_ payload: DataPayload) -> (reason: String, code: String)? { + guard let nodes = payload.nodes, !nodes.isEmpty else { + return ("snapshot returned no nodes", "no-nodes") + } + if isSparseApplicationWindowTree(nodes) { + return ("snapshot returned only structural application/window nodes", "sparse-tree") + } + if payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold { + return ("snapshot was cut off by its budget with almost nothing collected", "budget") + } + return nil + } + + static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool { + guard !nodes.isEmpty else { return false } + return nodes.allSatisfy { node in + // Application/Window labels are just the app/window name, and full-screen roots + // compute as hittable; neither says anything about tree health. + let isRootContainer = node.type == "Application" || node.type == "Window" + let hasContent = (!isRootContainer && node.label?.isEmpty == false) + || node.identifier?.isEmpty == false + || node.value?.isEmpty == false + return !hasContent + && (isRootContainer || !node.hittable) + && Self.structuralOnlyNodeTypes.contains(node.type) + } + } + + /// A leaf whose label joins many short segments is a container marked as an accessibility + /// element: the platform folds every descendant into one merged node. Nothing below it can + /// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it. + static func collapsedLeafIndexes(_ nodes: [SnapshotNode]) -> [Int]? { + let parents = Set(nodes.compactMap { $0.parentIndex }) + let collapsed = nodes.filter { node in + guard !parents.contains(node.index) else { return false } + guard !(node.type.lowercased().contains("text")) else { return false } + let label = node.label ?? "" + return label.split(separator: ",").count > collapsedLeafMinimumSegments + } + return collapsed.isEmpty ? nil : collapsed.map(\.index) + } + + // MARK: Outcome stamping + + private func stampedSnapshotPayload( + _ capture: SnapshotBackendCapture, + backend: SnapshotBackendKind, + state: String, + reason: (reason: String, code: String)? + ) -> DataPayload { + let payload = capture.payload + let quality = SnapshotQuality( + state: state, + backend: backend.rawValue, + reason: reason?.reason, + reasonCode: reason?.code, + effectiveDepth: capture.effectiveDepth, + collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? []) + ) + return DataPayload( + // Legacy human text for older daemons that read message instead of snapshotQuality. + message: Self.legacyQualityMessage(quality) ?? payload.message, + nodes: payload.nodes, + truncated: payload.truncated == true || state != "healthy" || capture.effectiveDepth != nil, + snapshotQuality: quality, + runnerFatal: payload.runnerFatal, + runnerFatalReason: payload.runnerFatalReason + ) + } + + static func legacyQualityMessage(_ quality: SnapshotQuality) -> String? { + guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil else { return nil } + var parts: [String] = [] + if quality.state == "recovered" { + let meaning = quality.reasonCode == "budget" + ? " The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen." + : " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears." + parts.append( + "Recovered this snapshot with the \(quality.backend) accessibility backend" + + (quality.reason.map { " after: \($0)." } ?? ".") + + meaning + ) + } + if quality.state == "sparse" { + parts.append( + "No snapshot backend could read this screen" + + (quality.reason.map { " (\($0))" } ?? "") + + ". Use screenshot as visual truth and coordinate taps." + ) + } + if let depth = quality.effectiveDepth { + parts.append( + "The accessibility server rejected deeper requests; this tree is capped at depth \(depth) — re-run with --depth \(depth) --scope for deeper content." + ) + } + return parts.isEmpty ? nil : parts.joined(separator: " ") + } +} + +// MARK: - In-bundle unit tests + +extension RunnerTests { + private func planTestNode( + index: Int, + type: String, + label: String? = nil, + identifier: String? = nil, + hittable: Bool = false, + parentIndex: Int? = nil + ) -> SnapshotNode { + SnapshotNode( + index: index, + type: type, + label: label, + identifier: identifier, + value: nil, + rect: snapshotRect(from: .zero), + enabled: true, + focused: nil, + selected: nil, + hittable: hittable, + depth: parentIndex == nil ? 0 : 1, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + func testSparsePayloadReasonMatrix() { + let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) + let window = planTestNode(index: 1, type: "Window", parentIndex: 0) + let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) + + // Labeled, hittable root over a bare window is still sparse. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) + // Deadline-truncated near-empty sweep needs recovery even with one real control. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) + // The same tiny tree from a completed sweep is a legitimately minimal screen. + XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) + // Empty payloads are degraded. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) + } + + func testCollapsedLeafIndexesFlagsMergedContainersOnly() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "), + parentIndex: 0 + ) + let prose = planTestNode( + index: 2, + type: "StaticText", + label: (0...30).map { "clause \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1]) + XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) + } + + func testLegacyQualityMessageStatesFallbackMeaning() { + let recovered = SnapshotQuality( + state: "recovered", + backend: "queries", + reason: "snapshot returned only structural application/window nodes", + reasonCode: "sparse-tree", + effectiveDepth: nil, + collapsedLeafIndexes: nil + ) + let message = Self.legacyQualityMessage(recovered) + XCTAssertTrue(message?.contains("queries accessibility backend") == true) + XCTAssertTrue(message?.contains("fixing the app's accessibility") == true) + XCTAssertTrue(message?.contains("screenshot as visual truth") == true) + XCTAssertNil( + Self.legacyQualityMessage( + SnapshotQuality( + state: "healthy", backend: "tree", reason: nil, reasonCode: nil, effectiveDepth: nil, + collapsedLeafIndexes: nil) + ) + ) + } +} diff --git a/src/__tests__/runtime-snapshot.test.ts b/src/__tests__/runtime-snapshot.test.ts index b4c7f5288f..4496d864c8 100644 --- a/src/__tests__/runtime-snapshot.test.ts +++ b/src/__tests__/runtime-snapshot.test.ts @@ -191,6 +191,35 @@ test('runtime snapshot does not flag prose text or labeled containers with child assert.deepEqual(result.warnings ?? [], []); }); +test('runtime snapshot renders the structured quality verdict and skips legacy detectors', async () => { + const mergedLabel = Array.from({ length: 30 }, (_, i) => `Row ${i}, Tap`).join(', '); + const device = createSnapshotOnlyDevice({ + nodes: [ + { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'App' }, + { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Other', label: mergedLabel }, + ], + truncated: true, + backend: 'xctest', + quality: { + state: 'recovered', + backend: 'queries', + reason: 'snapshot returned only structural application/window nodes', + collapsedLeafIndexes: [1], + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.equal(result.warnings?.length, 2); + assert.match( + String(result.warnings?.[0]), + /Recovered this snapshot with the queries accessibility backend/, + ); + assert.match(String(result.warnings?.[0]), /fixing the app's accessibility is the real cure/); + assert.match(String(result.warnings?.[1]), /@e2 \[Other\] merges many labels/); + assert.deepEqual(result.snapshotQuality?.state, 'recovered'); +}); + test('runtime snapshot does not warn for a normal iOS compact interactive output', async () => { const device = createSnapshotOnlyDevice({ nodes: [ diff --git a/src/backend.ts b/src/backend.ts index 3f98026deb..b57d644f43 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,3 +1,4 @@ +import type { SnapshotQualityVerdict } from './utils/snapshot-quality.ts'; import type { AndroidSnapshotBackendMetadata } from './platforms/android/snapshot-types.ts'; import type { AlertAction, AlertInfo } from './alert-contract.ts'; import type { AppsFilter } from './contracts/app-inventory.ts'; @@ -47,6 +48,7 @@ export type BackendSnapshotResult = { androidSnapshot?: AndroidSnapshotBackendMetadata; freshness?: BackendSnapshotFreshness; warnings?: string[]; + quality?: SnapshotQualityVerdict; appName?: string; appBundleId?: string; }; diff --git a/src/client-shared.ts b/src/client-shared.ts index 13b3b5d682..f2c00afd9f 100644 --- a/src/client-shared.ts +++ b/src/client-shared.ts @@ -173,6 +173,7 @@ export function serializeSnapshotResult(result: CaptureSnapshotResult): Record 0 ? { warnings: result.warnings } : {}), ...(result.unchanged ? { unchanged: result.unchanged } : {}), }; diff --git a/src/client-types.ts b/src/client-types.ts index 507268d973..cbeb403db6 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -1,3 +1,4 @@ +import type { SnapshotQualityVerdict } from './utils/snapshot-quality.ts'; import type { DaemonResponseData, DaemonInstallSource, @@ -338,6 +339,7 @@ export type CaptureSnapshotResult = { warnings?: string[]; unchanged?: SnapshotUnchanged; identifiers: AgentDeviceIdentifiers; + snapshotQuality?: SnapshotQualityVerdict; }; export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { diff --git a/src/client.ts b/src/client.ts index 9127cd5ec9..352b192091 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,3 +1,4 @@ +import { readSnapshotQualityVerdict } from './utils/snapshot-quality.ts'; import { sendToDaemon } from './daemon-client.ts'; import { prepareMetroRuntime, reloadMetro } from './client-metro.ts'; import { resolveDaemonPaths } from './daemon/config.ts'; @@ -335,7 +336,10 @@ function normalizeSnapshotResult( function optionalSnapshotResponseFields( data: Record, ): Partial< - Pick + Pick< + CaptureSnapshotResult, + 'androidSnapshot' | 'unchanged' | 'visibility' | 'warnings' | 'snapshotQuality' + > > { const visibility = readObject(data.visibility); const androidSnapshot = readObject(data.androidSnapshot); @@ -343,8 +347,10 @@ function optionalSnapshotResponseFields( const warnings = Array.isArray(data.warnings) ? data.warnings.filter((entry): entry is string => typeof entry === 'string') : undefined; + const snapshotQuality = readSnapshotQualityVerdict(data.snapshotQuality); return { ...(visibility ? { visibility: visibility as CaptureSnapshotResult['visibility'] } : {}), + ...(snapshotQuality ? { snapshotQuality } : {}), ...(androidSnapshot ? { androidSnapshot: androidSnapshot as CaptureSnapshotResult['androidSnapshot'] } : {}), diff --git a/src/commands/capture-snapshot.ts b/src/commands/capture-snapshot.ts index 9896a703d2..407a528063 100644 --- a/src/commands/capture-snapshot.ts +++ b/src/commands/capture-snapshot.ts @@ -1,6 +1,10 @@ import type { BackendSnapshotResult } from '../backend.ts'; import type { AndroidSnapshotBackendMetadata } from '../platforms/android/snapshot-types.ts'; import type { AgentDeviceRuntime, CommandSessionRecord } from '../runtime-contract.ts'; +import { + renderSnapshotQualityWarnings, + type SnapshotQualityVerdict, +} from '../utils/snapshot-quality.ts'; import { AppError } from '../utils/errors.ts'; import { buildSnapshotDiff, countSnapshotComparableLines } from '../utils/snapshot-diff.ts'; import type { SnapshotDiffLine, SnapshotDiffSummary } from '../utils/snapshot-diff.ts'; @@ -34,6 +38,7 @@ export type SnapshotCommandResult = { androidSnapshot?: AndroidSnapshotBackendMetadata; warnings?: string[]; unchanged?: SnapshotUnchanged; + snapshotQuality?: SnapshotQualityVerdict; }; export type DiffSnapshotCommandResult = { @@ -75,6 +80,7 @@ export const snapshotCommand: RuntimeCommand< snapshotRaw: options.raw, }), ...(capture.result.androidSnapshot ? { androidSnapshot: capture.result.androidSnapshot } : {}), + ...(capture.result.quality ? { snapshotQuality: capture.result.quality } : {}), ...(capture.warnings.length > 0 ? { warnings: capture.warnings } : {}), ...(unchanged ? { unchanged } : {}), ...snapshotAppFields(capture), @@ -216,8 +222,15 @@ function buildSnapshotWarnings(params: { runtimeNow: number; }): string[] { const warnings = [...(params.result.warnings ?? [])]; + if (params.result.quality) { + warnings.push(...renderSnapshotQualityWarnings(params.result.quality, params.snapshot.nodes)); + } warnings.push(...buildEmptyAndroidInteractiveWarnings(params)); - warnings.push(...buildSparseIosInteractiveWarnings(params)); + if (!params.result.quality) { + // Legacy runners without a structured verdict keep the old daemon-side heuristics. + warnings.push(...buildSparseIosInteractiveWarnings(params)); + warnings.push(...buildMergedAccessibilityLeafWarnings(params.snapshot.nodes)); + } const helperFallbackWarning = formatAndroidHelperFallbackWarning(params.result.androidSnapshot); if (helperFallbackWarning) warnings.push(helperFallbackWarning); @@ -225,8 +238,6 @@ function buildSnapshotWarnings(params: { const reactNativeOverlayWarning = formatReactNativeOverlayWarning(params.snapshot.nodes); if (reactNativeOverlayWarning) warnings.push(reactNativeOverlayWarning); - warnings.push(...buildMergedAccessibilityLeafWarnings(params.snapshot.nodes)); - const recentDropWarning = formatRecentSnapshotDropWarning(params); if (recentDropWarning) warnings.push(recentDropWarning); diff --git a/src/core/interactors/apple.ts b/src/core/interactors/apple.ts index 46f56d8b07..d8f3060625 100644 --- a/src/core/interactors/apple.ts +++ b/src/core/interactors/apple.ts @@ -19,6 +19,10 @@ import type { DeviceInfo } from '../../utils/device.ts'; import { AppError } from '../../utils/errors.ts'; import type { RawSnapshotNode } from '../../utils/snapshot.ts'; import type { Interactor, RunnerContext } from '../interactor-types.ts'; +import { + readSnapshotQualityVerdict, + type SnapshotQualityVerdict, +} from '../../utils/snapshot-quality.ts'; export function createAppleInteractor( device: DeviceInfo, @@ -74,7 +78,9 @@ export function createAppleInteractor( nodes, truncated: result.truncated ?? false, backend: 'xctest', - ...(result.message ? { warnings: [result.message] } : {}), + ...(result.quality ? { quality: result.quality } : {}), + // Legacy runners without a quality verdict still surface their message text. + ...(!result.quality && result.message ? { warnings: [result.message] } : {}), }; }, back: async (mode) => { @@ -136,12 +142,13 @@ function readAppleSnapshotResult(result: Record): { nodes?: RawSnapshotNode[]; truncated?: boolean; message?: string; + quality?: SnapshotQualityVerdict; } { return { nodes: Array.isArray(result.nodes) ? (result.nodes as RawSnapshotNode[]) : undefined, truncated: typeof result.truncated === 'boolean' ? result.truncated : undefined, - // Runner-attached context (e.g. "recovered with the fallback accessibility backend") - // surfaces as a snapshot warning so fallbacks are never silent. + quality: readSnapshotQualityVerdict(result.snapshotQuality), + // Legacy runner context for builds that predate the structured verdict. message: typeof result.message === 'string' && result.message.trim().length > 0 ? result.message diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 8691c962ea..036bcb69c8 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -64,6 +64,7 @@ type SnapshotData = { analysis?: AndroidSnapshotAnalysis; androidSnapshot?: AndroidSnapshotBackendMetadata; warnings?: string[]; + quality?: unknown; }; type SnapshotAttempt = { @@ -78,6 +79,7 @@ type CaptureSnapshotResult = { androidSnapshot?: AndroidSnapshotBackendMetadata; freshness?: AndroidFreshnessCaptureMeta; warnings?: string[]; + quality?: unknown; }; type AndroidFreshnessReason = 'empty-interactive' | 'sharp-drop' | 'stuck-route'; @@ -97,6 +99,7 @@ export async function captureSnapshot( analysis: data.analysis, androidSnapshot: data.androidSnapshot, warnings: data.warnings, + quality: data.quality, }; } @@ -184,6 +187,8 @@ async function captureInteractionOutcomeAwareSnapshot( analysis: latest.data.analysis, androidSnapshot: latest.data.androidSnapshot, freshness: latest.freshness, + warnings: latest.data.warnings, + quality: latest.data.quality, }; } @@ -249,6 +254,7 @@ async function captureAndroidFreshnessAwareSnapshot( androidSnapshot: latest.data.androidSnapshot, freshness: latest.freshness, warnings: latest.data.warnings, + quality: latest.data.quality, }; } @@ -303,6 +309,7 @@ async function capturePostGestureAwareSnapshot( androidSnapshot: latest.data.androidSnapshot, freshness: latest.freshness, warnings: latest.data.warnings, + quality: latest.data.quality, }; } diff --git a/src/daemon/snapshot-runtime.ts b/src/daemon/snapshot-runtime.ts index f652b8376c..607667911f 100644 --- a/src/daemon/snapshot-runtime.ts +++ b/src/daemon/snapshot-runtime.ts @@ -8,6 +8,7 @@ import type { DaemonRequest, DaemonResponse, DaemonResponseData, SessionState } import { SessionStore } from './session-store.ts'; import { errorResponse } from './handlers/response.ts'; import { captureSnapshot, resolveSnapshotScope } from './handlers/snapshot-capture.ts'; +import { readSnapshotQualityVerdict } from '../utils/snapshot-quality.ts'; import { buildSnapshotSession, resolveSessionDevice, @@ -291,6 +292,7 @@ function createDaemonSnapshotBackend(params: { androidSnapshot: capture.androidSnapshot, freshness: capture.freshness, warnings: capture.warnings, + quality: readSnapshotQualityVerdict(capture.quality), appName: session?.appBundleId ? (session.appName ?? session.appBundleId) : undefined, appBundleId: session?.appBundleId, }; diff --git a/src/utils/output.ts b/src/utils/output.ts index 45d166b087..7d7d249b3f 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -611,8 +611,11 @@ function buildSnapshotNotices( helperPresentation: AndroidHelperPresentationInput = { nodes, filteredCount: 0 }, ): string[] { const notices = readSnapshotWarnings(data); - const sparseSnapshotHint = formatSparseSnapshotHint(nodes, options); - if (sparseSnapshotHint) notices.push(sparseSnapshotHint); + // The structured snapshot quality verdict already carries a sharper version of this hint. + if (!data.snapshotQuality) { + const sparseSnapshotHint = formatSparseSnapshotHint(nodes, options); + if (sparseSnapshotHint) notices.push(sparseSnapshotHint); + } if (!options.raw && helperPresentation.filteredCount > 0) { notices.push( `Collapsed ${helperPresentation.filteredCount} Android helper node${helperPresentation.filteredCount === 1 ? '' : 's'} from the agent-facing text snapshot; use --raw or --json for the full hierarchy.`, diff --git a/src/utils/snapshot-quality.ts b/src/utils/snapshot-quality.ts new file mode 100644 index 0000000000..73a35becfe --- /dev/null +++ b/src/utils/snapshot-quality.ts @@ -0,0 +1,90 @@ +import type { SnapshotNode } from './snapshot.ts'; + +/** + * Structured quality verdict computed once by the iOS runner's snapshot capture plan. + * The daemon renders it; it never re-derives degradation from node shapes. + */ +export type SnapshotQualityVerdict = { + state: 'healthy' | 'recovered' | 'sparse'; + backend: 'tree' | 'queries' | 'private-ax'; + reason?: string; + reasonCode?: 'ax-rejected' | 'sparse-tree' | 'budget' | 'no-nodes' | 'capture-failed'; + effectiveDepth?: number; + collapsedLeafIndexes?: number[]; +}; + +export function readSnapshotQualityVerdict(value: unknown): SnapshotQualityVerdict | undefined { + if (!value || typeof value !== 'object') return undefined; + const raw = value as Record; + if (typeof raw.state !== 'string' || typeof raw.backend !== 'string') return undefined; + return { + state: raw.state as SnapshotQualityVerdict['state'], + backend: raw.backend as SnapshotQualityVerdict['backend'], + reason: typeof raw.reason === 'string' ? raw.reason : undefined, + reasonCode: + typeof raw.reasonCode === 'string' + ? (raw.reasonCode as SnapshotQualityVerdict['reasonCode']) + : undefined, + effectiveDepth: typeof raw.effectiveDepth === 'number' ? raw.effectiveDepth : undefined, + collapsedLeafIndexes: Array.isArray(raw.collapsedLeafIndexes) + ? raw.collapsedLeafIndexes.filter((entry): entry is number => typeof entry === 'number') + : undefined, + }; +} + +/** Canonical warning lines for a verdict; the single place degradation is worded. */ +export function renderSnapshotQualityWarnings( + verdict: SnapshotQualityVerdict, + nodes: Pick[], +): string[] { + return [ + ...stateWarning(verdict), + ...depthWarning(verdict), + ...collapsedLeafWarnings(verdict, nodes), + ]; +} + +function stateWarning(verdict: SnapshotQualityVerdict): string[] { + if (verdict.state === 'recovered') { + const meaning = + verdict.reasonCode === 'budget' + ? ' The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen.' + : " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears."; + return [ + `Recovered this snapshot with the ${verdict.backend} accessibility backend` + + (verdict.reason ? ` after: ${verdict.reason}.` : '.') + + meaning, + ]; + } + if (verdict.state === 'sparse') { + return [ + 'No snapshot backend could read this screen' + + (verdict.reason ? ` (${verdict.reason})` : '') + + '. Use screenshot as visual truth and coordinate taps; retry snapshot after navigating.', + ]; + } + return []; +} + +function depthWarning(verdict: SnapshotQualityVerdict): string[] { + if (verdict.effectiveDepth === undefined) return []; + return [ + `The accessibility server rejected deeper requests; this tree is capped at depth ${verdict.effectiveDepth} — re-run with --depth ${verdict.effectiveDepth} --scope for deeper content.`, + ]; +} + +function collapsedLeafWarnings( + verdict: SnapshotQualityVerdict, + nodes: Pick[], +): string[] { + const warnings: string[] = []; + for (const index of verdict.collapsedLeafIndexes ?? []) { + const node = nodes.find((entry) => entry.index === index); + if (!node) continue; + const name = node.identifier ? ` (${node.identifier})` : ''; + warnings.push( + `@${node.ref} [${node.type ?? 'element'}]${name} merges many labels into a single accessibility element. The app likely marks a container as accessible, which hides every descendant from assistive tech and automation — the children cannot be addressed individually. Fix the app's accessibility (mark the rows, not the container); until then use screenshot as visual truth and coordinate taps.`, + ); + } + return warnings; +} From 3a868efa6e31b9c0b78e93df99f9e4ad832bbbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 11 Jun 2026 21:14:51 +0200 Subject: [PATCH 2/3] fix(ios): correct recovered-snapshot viewport and private-AX scope semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the capture-plan refactor: - The query-sweep synthetic root doubles as the daemon's viewport (find.ts prefers on-screen matches inside nodes[0].rect), but it was built from candidate bounds, so off-screen controls below the screen could inflate it and win duplicate-label resolution. The root now uses the real finite viewport, falling back to candidate bounds only when viewport capture failed. - The private-AX backend applied --scope as a per-node text filter, hiding the matched container's children — diverging from regular snapshot scope semantics and contradicting the depth-cap hint that recommends scoped re-runs. Scope now selects the matched subtree: descendants inherit the match and only the normal option filters apply to them (in-bundle test covers a non-matching descendant). Verified live on Bluesky Home: scope homeScreen returns the 52-node subtree including non-matching descendants; compact root rect equals the screen (0,0,402,874). --- .../RunnerTests+AXSnapshotFallback.swift | 63 +++++++++++++++++-- .../RunnerTests+Snapshot.swift | 8 ++- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index d8774f924e..ac37434bd1 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -65,7 +65,8 @@ extension RunnerTests { options: options, viewport: viewport, depth: 0, - parentIndex: nil + parentIndex: nil, + insideMatchedScope: false ) if nodes.count <= 1 { NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_SPARSE=%ld", nodes.count) @@ -93,7 +94,8 @@ extension RunnerTests { options: SnapshotOptions, viewport: CGRect, depth: Int, - parentIndex: Int? + parentIndex: Int?, + insideMatchedScope: Bool ) { if let limit = options.depth, depth > limit { return } @@ -108,14 +110,26 @@ extension RunnerTests { let hasContent = !label.isEmpty || !identifier.isEmpty || !value.isEmpty let isRoot = parentIndex == nil + // Scope selects a subtree, matching regular snapshot semantics: once a node matches, + // every descendant is inside scope and only the normal option filters apply to it. + let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines) + let scopeActive = (scope?.isEmpty == false) + let matchesScope: Bool + if scopeActive, let scope { + let haystack = [label, identifier, value].joined(separator: "\n") + matchesScope = haystack.localizedCaseInsensitiveContains(scope) + } else { + matchesScope = false + } + let nowInsideScope = insideMatchedScope || matchesScope + let include: Bool if isRoot { include = true + } else if scopeActive && !nowInsideScope { + include = false } else if options.interactiveOnly && !visible { include = false - } else if let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines), !scope.isEmpty { - let haystack = [label, identifier, value].joined(separator: "\n") - include = haystack.localizedCaseInsensitiveContains(scope) } else if options.compact { include = hasContent || privateAXLikelyInteractive(rawElementType: rawType) } else { @@ -157,7 +171,8 @@ extension RunnerTests { options: options, viewport: viewport, depth: depth + 1, - parentIndex: currentIndex + parentIndex: currentIndex, + insideMatchedScope: nowInsideScope ) } } @@ -218,3 +233,39 @@ extension RunnerTests { return nil } } + +// MARK: - In-bundle unit tests + +extension RunnerTests { + func testPrivateAXScopeSelectsSubtreeNotMatchingLabels() { + let tree: [String: Any] = [ + "type": 1, "label": "App", + "children": [ + [ + "type": 9, "identifier": "homeScreen", + "children": [ + ["type": 48, "label": "Post body without the scope text", "children": []] + ], + ], + ["type": 9, "label": "unrelated sibling", "children": []], + ], + ] + var nodes: [SnapshotNode] = [] + appendPrivateAXNode( + tree, + to: &nodes, + options: SnapshotOptions( + interactiveOnly: false, compact: false, depth: nil, scope: "homeScreen", raw: false), + viewport: .infinite, + depth: 0, + parentIndex: nil, + insideMatchedScope: false + ) + + let labels = nodes.compactMap { $0.label ?? $0.identifier } + XCTAssertTrue(labels.contains("homeScreen")) + // Descendants of the matched scope are included even when they do not contain the text. + XCTAssertTrue(labels.contains("Post body without the scope text")) + XCTAssertFalse(labels.contains("unrelated sibling")) + } +} diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index d695cddfab..87189b77a7 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -351,7 +351,13 @@ extension RunnerTests { return left.type < right.type } - nodes[0] = compactInteractiveRootNode(rect: compactInteractiveRootFrame(for: candidates)) + // The synthetic root doubles as the daemon's viewport (find.ts prefers on-screen matches + // inside nodes[0].rect): use the real screen viewport when capture produced a finite one, + // so off-screen candidates can never inflate the root and masquerade as on-screen. + let rootRect = viewport.isInfinite || viewport.isNull || viewport.isEmpty + ? compactInteractiveRootFrame(for: candidates) + : viewport + nodes[0] = compactInteractiveRootNode(rect: rootRect) for candidate in candidates { nodes.append( SnapshotNode( From 427b604c8ccdfe5277251b5865f96820e95e4fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 12 Jun 2026 11:50:54 +0200 Subject: [PATCH 3/3] fix(ios): fail closed on interactive AX failure, stamp fatal verdict, validate parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings on the capture-plan terminal path: - P1: the fail-closed guard required `best == nil`, but the query-sweep tier always returns a synthetic-root sparse payload that sets `best` — so an interactive recursive-tree AX serialization failure that no backend recovered returned a sparse snapshot instead of invalidating the cached target. Reaching the terminal already means no backend produced a usable tree, so the sparse `best` must not suppress the fail-closed path. Extracted the decision into a pure, unit-tested `resolveSnapshotPlanTerminal` (closes the terminal-ordering testability gap the architecture review flagged). - P2: `snapshotAccessibilityUnavailable` returned a payload with no `snapshotQuality`, leaving one planned sparse result on the legacy-message path. It now carries a sparse/ax-rejected verdict like every other planned snapshot, so downstream sparse handling keys off the verdict. - P2: `readSnapshotQualityVerdict` cast any string state/backend into the union, so a malformed object suppressed the legacy node-shape detectors. State and backend are now validated against their unions (unknown → verdict-absent → legacy detectors run); an unknown reasonCode is dropped rather than rejecting the whole verdict, so a forward-version runner still yields a usable verdict. Unit-covered: Swift resolveSnapshotPlanTerminal matrix + fatal-verdict assertion; TS parser accept/reject/forward-compat. Full suite 249 files / 2449 tests, fallow, lint, runner build green. --- .../RunnerTests+Snapshot.swift | 12 ++++ .../RunnerTests+SnapshotCapturePlan.swift | 72 +++++++++++++++++-- src/utils/__tests__/snapshot-quality.test.ts | 43 +++++++++++ src/utils/snapshot-quality.ts | 41 ++++++++++- 4 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 src/utils/__tests__/snapshot-quality.test.ts diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 87189b77a7..7d2adb8462 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -384,8 +384,18 @@ extension RunnerTests { func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) + // This is a planned terminal result, so it carries the structured verdict like every other + // planned snapshot — downstream sparse handling keys off the verdict, not node shapes. return sparseTruncatedSnapshotPayload( message: recoveredSnapshotMessage(failure), + snapshotQuality: SnapshotQuality( + state: "sparse", + backend: SnapshotBackendKind.recursiveTree.rawValue, + reason: failure.message, + reasonCode: "ax-rejected", + effectiveDepth: nil, + collapsedLeafIndexes: nil + ), runnerFatal: true, runnerFatalReason: Self.axSnapshotUnavailableReason ) @@ -405,6 +415,7 @@ extension RunnerTests { func sparseTruncatedSnapshotPayload( message: String? = nil, + snapshotQuality: SnapshotQuality? = nil, runnerFatal: Bool? = nil, runnerFatalReason: String? = nil ) -> DataPayload { @@ -412,6 +423,7 @@ extension RunnerTests { message: message, nodes: [compactInteractiveRootNode(rect: .zero)], truncated: true, + snapshotQuality: snapshotQuality, runnerFatal: runnerFatal, runnerFatalReason: runnerFatalReason ) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 5ac9044ea8..d7601ebe10 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -129,12 +129,20 @@ extension RunnerTests { } if let axFailure { - switch terminal { - case .throwOnAXFailure: + switch Self.resolveSnapshotPlanTerminal( + terminal: terminal, + interactiveOnly: options.interactiveOnly + ) { + case .throwAxFailure: throw axFailure - case .sparseWithFatalOnAXFailure where options.interactiveOnly && best == nil: + case .failClosed: + // Fail closed on any interactive AX serialization failure that no backend recovered: + // invalidate the cached target so the next command reacquires it (AX-unavailable target + // invalidation, CONTEXT.md). A sparse `best` from a later tier (e.g. the query sweep's + // synthetic root) must NOT suppress this — reaching the terminal already means no backend + // produced a usable tree. return snapshotAccessibilityUnavailable(failure: axFailure) - default: + case .sparseBest: break } } @@ -190,6 +198,27 @@ extension RunnerTests { return nil } + /// Terminal action when a capture plan exhausted every backend with an AX serialization + /// failure still pending. Pure so the fail-closed-vs-sparse policy is unit-testable without + /// a live app (the ordering gap the architecture review flagged). + enum SnapshotPlanTerminalAction: Equatable { + case throwAxFailure + case failClosed + case sparseBest + } + + static func resolveSnapshotPlanTerminal( + terminal: SnapshotCaptureTerminalPolicy, + interactiveOnly: Bool + ) -> SnapshotPlanTerminalAction { + switch terminal { + case .throwOnAXFailure: + return .throwAxFailure + case .sparseWithFatalOnAXFailure: + return interactiveOnly ? .failClosed : .sparseBest + } + } + static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool { guard !nodes.isEmpty else { return false } return nodes.allSatisfy { node in @@ -359,4 +388,39 @@ extension RunnerTests { ) ) } + func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { + // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root + // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true), + .failClosed + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false), + .sparseBest + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true), + .throwAxFailure + ) + } + + func testSnapshotAccessibilityUnavailableCarriesSparseVerdict() { + currentApp = app + currentBundleId = "com.example.app" + defer { + currentApp = nil + currentBundleId = nil + } + let payload = snapshotAccessibilityUnavailable( + failure: SnapshotCaptureFailure( + code: "IOS_AX_SNAPSHOT_FAILED", + message: "kAXErrorIllegalArgument", + hint: "use screenshot" + ) + ) + XCTAssertEqual(payload.runnerFatal, true) + XCTAssertEqual(payload.snapshotQuality?.state, "sparse") + XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected") + } } diff --git a/src/utils/__tests__/snapshot-quality.test.ts b/src/utils/__tests__/snapshot-quality.test.ts new file mode 100644 index 0000000000..f101cfc674 --- /dev/null +++ b/src/utils/__tests__/snapshot-quality.test.ts @@ -0,0 +1,43 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; + +import { readSnapshotQualityVerdict } from '../snapshot-quality.ts'; + +test('readSnapshotQualityVerdict accepts a well-formed verdict', () => { + const verdict = readSnapshotQualityVerdict({ + state: 'recovered', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'budget', + effectiveDepth: 56, + collapsedLeafIndexes: [3], + }); + assert.deepEqual(verdict, { + state: 'recovered', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'budget', + effectiveDepth: 56, + collapsedLeafIndexes: [3], + }); +}); + +test('readSnapshotQualityVerdict rejects unknown state or backend as verdict-absent', () => { + // A malformed object must not be treated as an authoritative verdict — it has to fall through + // so legacy node-shape detectors still run instead of being silently suppressed. + assert.equal(readSnapshotQualityVerdict({ state: 'bogus', backend: 'tree' }), undefined); + assert.equal(readSnapshotQualityVerdict({ state: 'sparse', backend: 'mystery' }), undefined); + assert.equal(readSnapshotQualityVerdict({ backend: 'tree' }), undefined); + assert.equal(readSnapshotQualityVerdict(null), undefined); +}); + +test('readSnapshotQualityVerdict keeps the verdict but drops an unknown reasonCode', () => { + // Forward-compat: a newer runner adding a reasonCode must still yield a usable verdict. + const verdict = readSnapshotQualityVerdict({ + state: 'sparse', + backend: 'queries', + reasonCode: 'future-code', + }); + assert.equal(verdict?.state, 'sparse'); + assert.equal(verdict?.reasonCode, undefined); +}); diff --git a/src/utils/snapshot-quality.ts b/src/utils/snapshot-quality.ts index 73a35becfe..fd9404a14a 100644 --- a/src/utils/snapshot-quality.ts +++ b/src/utils/snapshot-quality.ts @@ -13,16 +13,53 @@ export type SnapshotQualityVerdict = { collapsedLeafIndexes?: number[]; }; +const SNAPSHOT_QUALITY_STATES = new Set([ + 'healthy', + 'recovered', + 'sparse', +]); +const SNAPSHOT_QUALITY_BACKENDS = new Set([ + 'tree', + 'queries', + 'private-ax', +]); +const SNAPSHOT_QUALITY_REASON_CODES = new Set>([ + 'ax-rejected', + 'sparse-tree', + 'budget', + 'no-nodes', + 'capture-failed', +]); + export function readSnapshotQualityVerdict(value: unknown): SnapshotQualityVerdict | undefined { if (!value || typeof value !== 'object') return undefined; const raw = value as Record; - if (typeof raw.state !== 'string' || typeof raw.backend !== 'string') return undefined; + // Validate the load-bearing union fields: an object with an unknown state/backend is not a + // verdict this version understands, so it falls through as verdict-absent and the legacy + // node-shape detectors run instead of being silently suppressed by a malformed payload. + if ( + typeof raw.state !== 'string' || + !SNAPSHOT_QUALITY_STATES.has(raw.state as SnapshotQualityVerdict['state']) + ) { + return undefined; + } + if ( + typeof raw.backend !== 'string' || + !SNAPSHOT_QUALITY_BACKENDS.has(raw.backend as SnapshotQualityVerdict['backend']) + ) { + return undefined; + } return { state: raw.state as SnapshotQualityVerdict['state'], backend: raw.backend as SnapshotQualityVerdict['backend'], reason: typeof raw.reason === 'string' ? raw.reason : undefined, + // An unknown reasonCode is dropped, not rejected: a forward-version runner that adds one + // still yields a usable verdict (only the budget-specific wording is keyed off it). reasonCode: - typeof raw.reasonCode === 'string' + typeof raw.reasonCode === 'string' && + SNAPSHOT_QUALITY_REASON_CODES.has( + raw.reasonCode as NonNullable, + ) ? (raw.reasonCode as SnapshotQualityVerdict['reasonCode']) : undefined, effectiveDepth: typeof raw.effectiveDepth === 'number' ? raw.effectiveDepth : undefined,