diff --git a/CONTEXT.md b/CONTEXT.md index 020b2307d1..e04c377cb3 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -17,6 +17,8 @@ - Daemon command registry: daemon-side source of truth for command route ownership and request-policy traits, including admission exemptions, session locking, selector validation, replay-scoped actions, recording invalidation, Android dialog guards, and request provider device resolution. - Runner command traits: the iOS XCTest runner's per-command-type classification across three independent axes — interaction (gates the foreground-guard and stabilization preflight), read-only (gates the session-invalidating retry; the alert command is read-only only for its `get` action), and runner-lifecycle (skips the app-activation preflight). One source of truth keyed by command type, distinct from the public command surface and daemon command registry. - Coordinate-first resolved element activation: iOS/macOS runner interaction pattern where a selector or text query resolves the semantic `XCUIElement`, then activation uses the element's resolved center coordinate when a frame is available. This keeps target selection semantic while avoiding `XCUIElement.tap()` post-action element re-resolution after normal navigation. tvOS remains focus/remote-driven. +- Snapshot capture plan: per-strategy ordered chain of iOS snapshot capture backends (recursive tree, query sweep, private AX) run by one plan runner under a shared wall-clock budget; recovery ordering is declared data, never a per-call-site branch. +- Snapshot quality verdict: structured outcome (state, backend, reason code, effective depth, collapsed leaves) computed once by the plan runner and shipped with every planned snapshot payload; the daemon and CLI render it instead of re-deriving degradation from node shapes. - AX-unavailable target invalidation: iOS/macOS runner behavior where a root accessibility snapshot failure such as `kAXErrorIllegalArgument` marks the cached `XCUIApplication` target handle suspect. The runner fails closed for degraded interactive snapshots, clears the cached target, and lets the next command reacquire the app through normal activation. ## Testing Principles diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index b3ced94848..428accc893 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -2,7 +2,9 @@ ## Status -Accepted +Accepted — implemented by the snapshot capture plan runner (RunnerTests+SnapshotCapturePlan.swift): +each strategy declares its backend chain, and a structured snapshot quality verdict makes +degraded or recovered output observable end to end. ## Context diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 671fbb4157..ac37434bd1 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -10,11 +10,10 @@ extension RunnerTests { /// apps where the AX surface is genuinely unavailable. static let privateAXSnapshotDepthLadder = [56, 40, 24, 12] - func privateAXSnapshotFallback( + func privateAXSnapshotCapture( app: XCUIApplication, - options: SnapshotOptions, - reason: String - ) -> DataPayload? { + options: SnapshotOptions + ) -> SnapshotBackendCapture? { #if os(iOS) && targetEnvironment(simulator) let requestedDepth = options.depth ?? 64 var attemptDepths = [requestedDepth] @@ -66,7 +65,8 @@ extension RunnerTests { options: options, viewport: viewport, depth: 0, - parentIndex: nil + parentIndex: nil, + insideMatchedScope: false ) if nodes.count <= 1 { NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_SPARSE=%ld", nodes.count) @@ -74,21 +74,15 @@ extension RunnerTests { } let depthLimited = effectiveDepth < requestedDepth - let truncated = (response["truncated"] as? Bool) == true || depthLimited - var message = - "Recovered this snapshot with the fallback accessibility backend after \(reason). This usually means the app publishes an unhealthy accessibility tree (too large or deep to serialize, or containers that hide their children) — fixing the app's accessibility is the real cure. The fallback is simulator-only and may expose a partial tree; treat screenshot as visual truth when this warning appears." - if depthLimited { - message += - " The accessibility server rejected deeper requests; this tree is capped at depth \(effectiveDepth) — re-run with --depth \(effectiveDepth) --scope to inspect deeper content." - } NSLog( - "AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_USED reason=%@ nodes=%ld depth=%ld truncated=%@", - reason, + "AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_USED nodes=%ld depth=%ld", nodes.count, - effectiveDepth, - truncated ? "true" : "false" + effectiveDepth + ) + return SnapshotBackendCapture( + payload: DataPayload(nodes: nodes, truncated: (response["truncated"] as? Bool) == true), + effectiveDepth: depthLimited ? effectiveDepth : nil ) - return DataPayload(message: message, nodes: nodes, truncated: truncated) #else return nil #endif @@ -100,7 +94,8 @@ extension RunnerTests { options: SnapshotOptions, viewport: CGRect, depth: Int, - parentIndex: Int? + parentIndex: Int?, + insideMatchedScope: Bool ) { if let limit = options.depth, depth > limit { return } @@ -115,14 +110,26 @@ extension RunnerTests { let hasContent = !label.isEmpty || !identifier.isEmpty || !value.isEmpty let isRoot = parentIndex == nil + // Scope selects a subtree, matching regular snapshot semantics: once a node matches, + // every descendant is inside scope and only the normal option filters apply to it. + let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines) + let scopeActive = (scope?.isEmpty == false) + let matchesScope: Bool + if scopeActive, let scope { + let haystack = [label, identifier, value].joined(separator: "\n") + matchesScope = haystack.localizedCaseInsensitiveContains(scope) + } else { + matchesScope = false + } + let nowInsideScope = insideMatchedScope || matchesScope + let include: Bool if isRoot { include = true + } else if scopeActive && !nowInsideScope { + include = false } else if options.interactiveOnly && !visible { include = false - } else if let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines), !scope.isEmpty { - let haystack = [label, identifier, value].joined(separator: "\n") - include = haystack.localizedCaseInsensitiveContains(scope) } else if options.compact { include = hasContent || privateAXLikelyInteractive(rawElementType: rawType) } else { @@ -164,7 +171,8 @@ extension RunnerTests { options: options, viewport: viewport, depth: depth + 1, - parentIndex: currentIndex + parentIndex: currentIndex, + insideMatchedScope: nowInsideScope ) } } @@ -225,3 +233,39 @@ extension RunnerTests { return nil } } + +// MARK: - In-bundle unit tests + +extension RunnerTests { + func testPrivateAXScopeSelectsSubtreeNotMatchingLabels() { + let tree: [String: Any] = [ + "type": 1, "label": "App", + "children": [ + [ + "type": 9, "identifier": "homeScreen", + "children": [ + ["type": 48, "label": "Post body without the scope text", "children": []] + ], + ], + ["type": 9, "label": "unrelated sibling", "children": []], + ], + ] + var nodes: [SnapshotNode] = [] + appendPrivateAXNode( + tree, + to: &nodes, + options: SnapshotOptions( + interactiveOnly: false, compact: false, depth: nil, scope: "homeScreen", raw: false), + viewport: .infinite, + depth: 0, + parentIndex: nil, + insideMatchedScope: false + ) + + let labels = nodes.compactMap { $0.label ?? $0.identifier } + XCTAssertTrue(labels.contains("homeScreen")) + // Descendants of the matched scope are included even when they do not contain the text. + XCTAssertTrue(labels.contains("Post body without the scope text")) + XCTAssertFalse(labels.contains("unrelated sibling")) + } +} diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 726a451a50..7e329c6001 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -205,6 +205,7 @@ struct DataPayload: Codable { let items: [String]? let nodes: [SnapshotNode]? let truncated: Bool? + let snapshotQuality: SnapshotQuality? let gestureStartUptimeMs: Double? let gestureEndUptimeMs: Double? let x: Double? @@ -242,6 +243,7 @@ struct DataPayload: Codable { items: [String]? = nil, nodes: [SnapshotNode]? = nil, truncated: Bool? = nil, + snapshotQuality: SnapshotQuality? = nil, gestureStartUptimeMs: Double? = nil, gestureEndUptimeMs: Double? = nil, x: Double? = nil, @@ -278,6 +280,7 @@ struct DataPayload: Codable { self.items = items self.nodes = nodes self.truncated = truncated + self.snapshotQuality = snapshotQuality self.gestureStartUptimeMs = gestureStartUptimeMs self.gestureEndUptimeMs = gestureEndUptimeMs self.x = x diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index d3c2c773b1..7d2adb8462 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -11,29 +11,7 @@ extension RunnerTests { private static let rawSnapshotMaxNodes = 5_000 private static let rawSnapshotTooLargeHint = "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree." - private static let publicQueryRecoveryMessage = - "Recovered iOS snapshot through XCTest accessibility element queries after the public snapshot tree was sparse. This usually means the app publishes an unhealthy accessibility tree - fixing the app accessibility is the real cure. The recovered nodes are a flattened view of on-screen controls; treat screenshot as visual truth when this warning appears." - private static let structuralOnlyNodeTypes: Set = [ - "Application", - "Window", - "Other", - "ScrollView" - ] - private static let collapsedTabCandidateTypes: Set = [ - .button, - .link, - .menuItem, - .other, - .staticText - ] - static let scrollContainerTypes: Set = [ - .collectionView, - .scrollView, - .table - ] - private static let flatInteractiveFallbackBudget: TimeInterval = 1.0 - - private struct SnapshotTraversalContext { + struct SnapshotTraversalContext { let queryRoot: XCUIElement let rootSnapshot: XCUIElementSnapshot let viewport: CGRect @@ -52,12 +30,6 @@ extension RunnerTests { let visible: Bool } - private enum SnapshotTraversalCapture { - case context(SnapshotTraversalContext) - case fallback(DataPayload) - case empty - } - struct SnapshotCaptureFailure: Error { let code: String let message: String @@ -106,35 +78,48 @@ extension RunnerTests { } } + static let structuralOnlyNodeTypes: Set = [ + "Application", + "Window", + "Other", + "ScrollView" + ] + + private static let collapsedTabCandidateTypes: Set = [ + .button, + .link, + .menuItem, + .other, + .staticText + ] + + static let scrollContainerTypes: Set = [ + .collectionView, + .scrollView, + .table + ] + + private static let flatInteractiveFallbackBudget: TimeInterval = 1.0 + func snapshotFast(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload { if let blocking = blockingSystemAlertSnapshot() { return blocking } - if options.interactiveOnly && options.compact { - let payload = snapshotFlatInteractive(app: app, options: options) - return snapshotWithPrivateAXFallbackIfSparse( - payload, - app: app, - options: options, - reason: "compact interactive XCTest snapshot was sparse" - ) - } - - let capture = try captureSnapshotTraversalContext( + let plan = options.interactiveOnly && options.compact + ? Self.compactInteractivePlan + : Self.regularVisiblePlan + return try runSnapshotCapturePlan( + plan, app: app, options: options, - allowInteractiveUnavailableFallback: true + terminal: .sparseWithFatalOnAXFailure ) - let context: SnapshotTraversalContext - switch capture { - case .context(let traversalContext): - context = traversalContext - case .fallback(let fallback): - return fallback - case .empty: - return DataPayload(nodes: [], truncated: false) - } + } + func recursiveTreeSnapshotPayload( + context: SnapshotTraversalContext, + options: SnapshotOptions + ) -> DataPayload { var cachedDescendantElements: [XCUIElement]? func collapsedTabDescendants() -> [XCUIElement] { if let cachedDescendantElements { @@ -257,150 +242,28 @@ extension RunnerTests { } - let payload = DataPayload( + return DataPayload( nodes: applyHiddenContentHints(hiddenContentHintsByNodeIndex, to: nodes), truncated: false ) - return snapshotWithFallbackIfSparse( - payload, - app: app, - options: options, - reason: "XCTest snapshot returned a sparse application/window tree" - ) - } - - private func snapshotWithFallbackIfSparse( - _ payload: DataPayload, - app: XCUIApplication, - options: SnapshotOptions, - reason: String - ) -> DataPayload { - guard Self.snapshotPayloadNeedsRecovery(payload) else { - return payload - } - if let fallback = publicQuerySnapshotFallback( - app: app, - options: options, - reason: reason - ) { - return fallback - } - return betterSnapshotPayload( - payload, - recovered: privateAXSnapshotFallback(app: app, options: options, reason: reason) - ) - } - - private func snapshotWithPrivateAXFallbackIfSparse( - _ payload: DataPayload, - app: XCUIApplication, - options: SnapshotOptions, - reason: String - ) -> DataPayload { - guard Self.snapshotPayloadNeedsRecovery(payload) else { - return payload - } - return betterSnapshotPayload( - payload, - recovered: privateAXSnapshotFallback(app: app, options: options, reason: reason) - ) - } - - /// A payload needs recovery when the tree is structural-only, OR when the capture was cut - /// off by a budget/deadline with almost nothing collected. The second condition matters on - /// large React Native trees: the typed-query sweep can resolve one or two stray controls - /// before its deadline, which defeats an all-structural check while the payload is still - /// useless in practice. A legitimately minimal screen finishes the sweep without truncation, - /// so it never pays for recovery. - static let sparseRecoveryTruncatedNodeThreshold = 8 - - static func snapshotPayloadNeedsRecovery(_ payload: DataPayload) -> Bool { - guard let nodes = payload.nodes, !nodes.isEmpty else { return false } - if isSparseApplicationWindowTree(nodes) { return true } - return payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold - } - - /// Keeps the original payload unless the recovered tree actually carries more nodes — - /// recovery must never replace a partial-but-real capture with something thinner. - private func betterSnapshotPayload( - _ payload: DataPayload, - recovered: DataPayload? - ) -> DataPayload { - guard let recovered, let recoveredNodes = recovered.nodes, - recoveredNodes.count > (payload.nodes?.count ?? 0) - else { - return payload - } - return recovered - } - - private static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool { - guard !nodes.isEmpty else { return false } - return nodes.allSatisfy { node in - // Application/Window labels are just the app/window name, and full-screen roots - // compute as hittable; neither says anything about tree health, so neither counts - // as content for these types (a labeled app+window pair is still a sparse tree). - let isRootContainer = node.type == "Application" || node.type == "Window" - let hasContent = (!isRootContainer && node.label?.isEmpty == false) - || node.identifier?.isEmpty == false - || node.value?.isEmpty == false - return !hasContent - && (isRootContainer || !node.hittable) - && Self.structuralOnlyNodeTypes.contains(node.type) - } - } - - private func publicQuerySnapshotFallback( - app: XCUIApplication, - options: SnapshotOptions, - reason: String - ) -> DataPayload? { - let fallback = snapshotFlatInteractive( - app: app, - options: SnapshotOptions( - interactiveOnly: false, - compact: options.compact, - depth: options.depth, - scope: options.scope, - raw: false - ) - ) - guard let nodes = fallback.nodes, !Self.isSparseApplicationWindowTree(nodes) else { - return nil - } - NSLog( - "AGENT_DEVICE_RUNNER_PUBLIC_QUERY_SNAPSHOT_USED reason=%@ nodes=%ld truncated=%@", - reason, - nodes.count, - fallback.truncated == true ? "true" : "false" - ) - return DataPayload( - message: Self.publicQueryRecoveryMessage, - nodes: nodes, - truncated: true - ) } func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload { if let blocking = blockingSystemAlertSnapshot() { return blocking } - - let capture = try captureSnapshotTraversalContext( + return try runSnapshotCapturePlan( + Self.rawDiagnosticPlan, app: app, options: options, - allowInteractiveUnavailableFallback: false + terminal: .throwOnAXFailure ) - let context: SnapshotTraversalContext - switch capture { - case .context(let traversalContext): - context = traversalContext - case .fallback(let fallback): - return fallback - case .empty: - return DataPayload(nodes: [], truncated: false) - } + } + func rawTreeSnapshotPayload( + context: SnapshotTraversalContext, + options: SnapshotOptions + ) throws -> DataPayload { var nodes: [SnapshotNode] = [] func walk(_ snapshot: XCUIElementSnapshot, depth: Int, parentIndex: Int?) throws { @@ -439,15 +302,10 @@ extension RunnerTests { } try walk(context.rootSnapshot, depth: 0, parentIndex: nil) - return snapshotWithPrivateAXFallbackIfSparse( - DataPayload(nodes: nodes, truncated: false), - app: app, - options: options, - reason: "XCTest raw snapshot returned a sparse application/window tree" - ) + return DataPayload(nodes: nodes, truncated: false) } - private func snapshotFlatInteractive(app: XCUIApplication, options: SnapshotOptions) -> DataPayload { + func snapshotFlatInteractive(app: XCUIApplication, options: SnapshotOptions) -> DataPayload { var nodes: [SnapshotNode] = [ compactInteractiveRootNode(rect: .zero) ] @@ -493,7 +351,13 @@ extension RunnerTests { return left.type < right.type } - nodes[0] = compactInteractiveRootNode(rect: compactInteractiveRootFrame(for: candidates)) + // The synthetic root doubles as the daemon's viewport (find.ts prefers on-screen matches + // inside nodes[0].rect): use the real screen viewport when capture produced a finite one, + // so off-screen candidates can never inflate the root and masquerade as on-screen. + let rootRect = viewport.isInfinite || viewport.isNull || viewport.isEmpty + ? compactInteractiveRootFrame(for: candidates) + : viewport + nodes[0] = compactInteractiveRootNode(rect: rootRect) for candidate in candidates { nodes.append( SnapshotNode( @@ -517,87 +381,26 @@ extension RunnerTests { return DataPayload(nodes: nodes, truncated: truncated) } - private func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { + func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) + // This is a planned terminal result, so it carries the structured verdict like every other + // planned snapshot — downstream sparse handling keys off the verdict, not node shapes. return sparseTruncatedSnapshotPayload( message: recoveredSnapshotMessage(failure), + snapshotQuality: SnapshotQuality( + state: "sparse", + backend: SnapshotBackendKind.recursiveTree.rawValue, + reason: failure.message, + reasonCode: "ax-rejected", + effectiveDepth: nil, + collapsedLeafIndexes: nil + ), runnerFatal: true, runnerFatalReason: Self.axSnapshotUnavailableReason ) } - private func captureSnapshotTraversalContext( - app: XCUIApplication, - options: SnapshotOptions, - allowInteractiveUnavailableFallback: Bool - ) throws -> SnapshotTraversalCapture { - do { - guard let context = try makeSnapshotTraversalContext(app: app, options: options) else { - return .empty - } - return .context(context) - } catch let failure as SnapshotCaptureFailure { - if Self.isAxSnapshotFailure(failure), - let fallback = privateAXSnapshotFallback( - app: app, - options: options, - reason: failure.message - ) - { - return .fallback(fallback) - } - if let fallback = snapshotDepthLimitedAccessibilityFallback( - app: app, - options: options, - failure: failure - ) { - return .fallback(fallback) - } - if allowInteractiveUnavailableFallback && options.interactiveOnly { - return .fallback(snapshotAccessibilityUnavailable(failure: failure)) - } - throw failure - } - } - - private func snapshotDepthLimitedAccessibilityFallback( - app: XCUIApplication, - options: SnapshotOptions, - failure: SnapshotCaptureFailure - ) -> DataPayload? { - guard let requestedDepth = options.depth else { - return nil - } - - NSLog( - "AGENT_DEVICE_RUNNER_SNAPSHOT_DEPTH_FALLBACK=%@", - failure.message - ) - - if requestedDepth <= 0 { - return sparseTruncatedSnapshotPayload(message: recoveredSnapshotMessage(failure)) - } - - // Raw depth-limited recovery intentionally falls back to sparse interactive discovery because - // the raw AX tree is the failed operation. - let fallback = snapshotFlatInteractive( - app: app, - options: SnapshotOptions( - interactiveOnly: true, - compact: options.compact, - depth: requestedDepth, - scope: options.scope, - raw: false - ) - ) - return DataPayload( - message: recoveredSnapshotMessage(failure), - nodes: fallback.nodes, - truncated: true - ) - } - private func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String { return "\(failure.message) Hint: \(failure.hint)" } @@ -610,8 +413,9 @@ extension RunnerTests { ) } - private func sparseTruncatedSnapshotPayload( - message: String, + func sparseTruncatedSnapshotPayload( + message: String? = nil, + snapshotQuality: SnapshotQuality? = nil, runnerFatal: Bool? = nil, runnerFatalReason: String? = nil ) -> DataPayload { @@ -619,6 +423,7 @@ extension RunnerTests { message: message, nodes: [compactInteractiveRootNode(rect: .zero)], truncated: true, + snapshotQuality: snapshotQuality, runnerFatal: runnerFatal, runnerFatalReason: runnerFatalReason ) @@ -659,120 +464,6 @@ extension RunnerTests { XCTAssertTrue(message.contains(Self.axSnapshotHint)) } - func testSparseApplicationWindowTreeDetectionIsConservative() { - let root = compactInteractiveRootNode(rect: .zero) - func node( - index: Int, - type: String, - label: String? = nil, - identifier: String? = nil, - value: String? = nil, - hittable: Bool = false - ) -> SnapshotNode { - SnapshotNode( - index: index, - type: type, - label: label, - identifier: identifier, - value: value, - rect: snapshotRect(from: .zero), - enabled: true, - focused: nil, - selected: nil, - hittable: hittable, - depth: 1, - parentIndex: 0, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - let window = node(index: 1, type: "Window") - let structuralOther = node(index: 2, type: "Other") - let structuralScroll = node(index: 3, type: "ScrollView") - let labeledOther = node(index: 4, type: "Other", label: "Visible content") - let identifiedOther = node(index: 5, type: "Other", identifier: "test-id") - let valuedOther = node(index: 6, type: "Other", value: "Selected") - let hittableOther = node(index: 7, type: "Other", hittable: true) - let button = node( - index: 8, - type: "Button", - label: "Sign in", - hittable: true - ) - - XCTAssertTrue(Self.isSparseApplicationWindowTree([root])) - XCTAssertTrue(Self.isSparseApplicationWindowTree([root, window])) - XCTAssertTrue(Self.isSparseApplicationWindowTree([root, window, structuralOther, structuralScroll])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, labeledOther])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, identifiedOther])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, valuedOther])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, hittableOther])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, button])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([root, window, button])) - XCTAssertFalse(Self.isSparseApplicationWindowTree([])) - // App/window name labels and full-screen-root hittability are not content: a labeled, - // hittable Application root over a bare Window is still a sparse tree (a shape seen on - // production React Native login screens behind full-screen modal overlays). - let labeledHittableRoot = node( - index: 0, type: "Application", label: "Example App", hittable: true) - XCTAssertTrue(Self.isSparseApplicationWindowTree([labeledHittableRoot, window])) - XCTAssertFalse( - Self.isSparseApplicationWindowTree([ - labeledHittableRoot, node(index: 1, type: "Application", identifier: "custom-id"), - ]) - ) - } - - func testSnapshotPayloadNeedsRecoveryOnDeadlineTruncatedNearEmptySweep() { - let root = compactInteractiveRootNode(rect: .zero) - func node(index: Int, label: String) -> SnapshotNode { - SnapshotNode( - index: index, - type: "Button", - label: label, - identifier: nil, - value: nil, - rect: snapshotRect(from: .zero), - enabled: true, - focused: nil, - selected: nil, - hittable: true, - depth: 1, - parentIndex: 0, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - let button = node(index: 1, label: "Home") - - // Deadline-truncated sweep with a stray control: still needs recovery. - XCTAssertTrue( - Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: [root, button], truncated: true)) - ) - // Structural-only tree needs recovery regardless of truncation. - XCTAssertTrue( - Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: [root], truncated: false)) - ) - // A completed sweep on a legitimately minimal screen does not. - XCTAssertFalse( - Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: [root, button], truncated: false)) - ) - // A truncated but reasonably populated sweep does not. - var populated: [SnapshotNode] = [root] - for index in 1...Self.sparseRecoveryTruncatedNodeThreshold { - populated.append(node(index: index, label: "b\(index)")) - } - XCTAssertFalse( - Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: populated, truncated: true)) - ) - XCTAssertFalse(Self.snapshotPayloadNeedsRecovery(DataPayload(nodes: [], truncated: true))) - } - - func testPublicQueryRecoveryMessageExplainsFlattenedFallback() { - XCTAssertTrue(Self.publicQueryRecoveryMessage.contains("XCTest accessibility element queries")) - XCTAssertTrue(Self.publicQueryRecoveryMessage.contains("flattened")) - } - func testRawSnapshotTooLargeFailureIsStructured() { let failure = rawSnapshotTooLargeFailure(nodeCount: Self.rawSnapshotMaxNodes + 1) @@ -781,39 +472,6 @@ extension RunnerTests { XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint) } - func testDepthLimitedSnapshotFailureReturnsNonFatalFallback() { - currentApp = app - currentBundleId = "com.example.app" - - let payload = snapshotDepthLimitedAccessibilityFallback( - app: app, - options: SnapshotOptions( - interactiveOnly: false, - compact: false, - depth: 0, - scope: nil, - raw: false - ), - failure: SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: "\(Self.axSnapshotFailureMessage) kAXErrorIllegalArgument.", - hint: Self.axSnapshotHint - ) - ) - - XCTAssertEqual( - payload?.message, - "\(Self.axSnapshotFailureMessage) kAXErrorIllegalArgument. Hint: \(Self.axSnapshotHint)" - ) - XCTAssertEqual(payload?.nodes?.count, 1) - XCTAssertEqual(payload?.nodes?.first?.type, "Application") - XCTAssertEqual(payload?.truncated, true) - XCTAssertNil(payload?.runnerFatal) - XCTAssertNil(payload?.runnerFatalReason) - XCTAssertNotNil(currentApp) - XCTAssertEqual(currentBundleId, "com.example.app") - } - private func compactInteractiveRootNode(rect: CGRect) -> SnapshotNode { SnapshotNode( index: 0, @@ -909,7 +567,7 @@ extension RunnerTests { return true } - private func makeSnapshotTraversalContext( + func makeSnapshotTraversalContext( app: XCUIApplication, options: SnapshotOptions ) throws -> SnapshotTraversalContext? { @@ -985,7 +643,7 @@ extension RunnerTests { || (normalized.contains("illegal argument") && normalized.contains("snapshot")) } - private static func isAxSnapshotFailure(_ failure: SnapshotCaptureFailure) -> Bool { + static func isAxSnapshotFailure(_ failure: SnapshotCaptureFailure) -> Bool { failure.code == Self.axSnapshotErrorCode || isAxIllegalArgument(failure.message) } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift new file mode 100644 index 0000000000..d7601ebe10 --- /dev/null +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -0,0 +1,426 @@ +import XCTest + +// MARK: - Snapshot capture plans (ADR 0004) +// +// Each snapshot strategy declares an ordered chain of capture backends. One runner walks the +// chain: capture, classify, accept the first payload the quality classifier calls usable, and +// stamp the outcome with a structured quality verdict so the daemon renders state instead of +// re-deriving it from node shapes. Recovery ordering is data here, never a per-call-site branch. + +/// Structured quality verdict shipped with every iOS snapshot payload. +struct SnapshotQuality: Codable { + /// healthy: first backend produced a usable tree. recovered: a later backend did. + /// sparse: no backend produced a usable tree; the best attempt is returned as-is. + let state: String + /// Backend that produced the returned payload: tree | queries | private-ax. + let backend: String + /// Why recovery ran (first failure) or why the payload is degraded. + let reason: String? + /// Machine-readable reason: ax-rejected | sparse-tree | budget | no-nodes. + let reasonCode: String? + /// Private AX ladder cap when the accepted tree is shallower than requested. + let effectiveDepth: Int? + /// Leaves that merge many labels — a container marked accessible hides its descendants. + let collapsedLeafIndexes: [Int]? +} + +enum SnapshotBackendKind: String, CaseIterable { + case recursiveTree = "tree" + case querySweep = "queries" + case privateAX = "private-ax" +} + +/// What the plan runner does when every backend failed or stayed sparse. +enum SnapshotCaptureTerminalPolicy { + /// Return the best sparse payload; if the tree backend hit a real AX serialization failure + /// on an interactive request, fail closed: invalidate the cached target and mark runnerFatal + /// (AX-unavailable target invalidation, CONTEXT.md). + case sparseWithFatalOnAXFailure + /// Re-throw the tree backend's AX failure (raw diagnostics preserve errors, ADR 0004). + case throwOnAXFailure +} + +struct SnapshotBackendCapture { + let payload: DataPayload + /// Set by the private AX backend when the ladder accepted a shallower depth than requested. + let effectiveDepth: Int? +} + +extension RunnerTests { + static let sparseRecoveryTruncatedNodeThreshold = 8 + /// Umbrella wall-clock budget for one capture plan. Individual backends bound themselves, + /// but chained recovery tiers must never stack past the 30s main-thread watchdog: when the + /// budget is spent, remaining tiers are skipped and the best payload so far is returned. + static let snapshotPlanBudget: TimeInterval = 20 + static let collapsedLeafMinimumSegments = 10 + + static func payloadNodeCount(_ payload: DataPayload?) -> Int { + payload?.nodes?.count ?? 0 + } + + // MARK: Plan definitions + + static let regularVisiblePlan: [SnapshotBackendKind] = [.recursiveTree, .querySweep, .privateAX] + static let compactInteractivePlan: [SnapshotBackendKind] = [.querySweep, .privateAX] + static let rawDiagnosticPlan: [SnapshotBackendKind] = [.recursiveTree, .privateAX] + + // MARK: Plan runner + + func runSnapshotCapturePlan( + _ plan: [SnapshotBackendKind], + app: XCUIApplication, + options: SnapshotOptions, + terminal: SnapshotCaptureTerminalPolicy + ) throws -> DataPayload { + var best: (kind: SnapshotBackendKind, capture: SnapshotBackendCapture)? + var firstFailure: (reason: String, code: String)? + var axFailure: SnapshotCaptureFailure? + let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) + + for kind in plan { + if kind != plan.first && Date() >= deadline { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_PLAN_BUDGET_EXHAUSTED skipped=%@", kind.rawValue) + if firstFailure == nil { + firstFailure = ("the capture plan ran out of its time budget", "budget") + } + break + } + let capture: SnapshotBackendCapture + do { + guard let result = try captureWithBackend(kind, app: app, options: options) else { + continue + } + capture = result + } catch let failure as SnapshotCaptureFailure { + if Self.isAxSnapshotFailure(failure) { axFailure = failure } + if firstFailure == nil { + firstFailure = (failure.message, Self.isAxSnapshotFailure(failure) ? "ax-rejected" : "capture-failed") + } + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_BACKEND_FAILED backend=%@ error=%@", + kind.rawValue, + failure.message + ) + continue + } + + if let sparseReason = Self.sparsePayloadReason(capture.payload) { + if firstFailure == nil { firstFailure = sparseReason } + if Self.payloadNodeCount(capture.payload) > Self.payloadNodeCount(best?.capture.payload) { + best = (kind, capture) + } + continue + } + + let recovered = kind != plan.first + if recovered { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_RECOVERED backend=%@ reason=%@", + kind.rawValue, + firstFailure?.reason ?? "sparse tree" + ) + } + return stampedSnapshotPayload( + capture, + backend: kind, + state: recovered ? "recovered" : "healthy", + reason: recovered ? firstFailure : nil + ) + } + + if let axFailure { + switch Self.resolveSnapshotPlanTerminal( + terminal: terminal, + interactiveOnly: options.interactiveOnly + ) { + case .throwAxFailure: + throw axFailure + case .failClosed: + // Fail closed on any interactive AX serialization failure that no backend recovered: + // invalidate the cached target so the next command reacquires it (AX-unavailable target + // invalidation, CONTEXT.md). A sparse `best` from a later tier (e.g. the query sweep's + // synthetic root) must NOT suppress this — reaching the terminal already means no backend + // produced a usable tree. + return snapshotAccessibilityUnavailable(failure: axFailure) + case .sparseBest: + break + } + } + + let fallbackPayload = + best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: "sparse", reason: firstFailure) } + ?? stampedSnapshotPayload( + SnapshotBackendCapture(payload: sparseTruncatedSnapshotPayload(), effectiveDepth: nil), + backend: plan.last ?? .recursiveTree, + state: "sparse", + reason: firstFailure + ) + return fallbackPayload + } + + private func captureWithBackend( + _ kind: SnapshotBackendKind, + app: XCUIApplication, + options: SnapshotOptions + ) throws -> SnapshotBackendCapture? { + switch kind { + case .recursiveTree: + guard let context = try makeSnapshotTraversalContext(app: app, options: options) else { + return nil + } + let payload = options.raw + ? try rawTreeSnapshotPayload(context: context, options: options) + : recursiveTreeSnapshotPayload(context: context, options: options) + return SnapshotBackendCapture(payload: payload, effectiveDepth: nil) + case .querySweep: + return SnapshotBackendCapture( + payload: snapshotFlatInteractive(app: app, options: options), + effectiveDepth: nil + ) + case .privateAX: + return privateAXSnapshotCapture(app: app, options: options) + } + } + + // MARK: Quality classifier (the single source of "is this snapshot degraded") + + /// Returns a degradation reason + machine code when the payload is too degraded to accept. + static func sparsePayloadReason(_ payload: DataPayload) -> (reason: String, code: String)? { + guard let nodes = payload.nodes, !nodes.isEmpty else { + return ("snapshot returned no nodes", "no-nodes") + } + if isSparseApplicationWindowTree(nodes) { + return ("snapshot returned only structural application/window nodes", "sparse-tree") + } + if payload.truncated == true && nodes.count <= sparseRecoveryTruncatedNodeThreshold { + return ("snapshot was cut off by its budget with almost nothing collected", "budget") + } + return nil + } + + /// Terminal action when a capture plan exhausted every backend with an AX serialization + /// failure still pending. Pure so the fail-closed-vs-sparse policy is unit-testable without + /// a live app (the ordering gap the architecture review flagged). + enum SnapshotPlanTerminalAction: Equatable { + case throwAxFailure + case failClosed + case sparseBest + } + + static func resolveSnapshotPlanTerminal( + terminal: SnapshotCaptureTerminalPolicy, + interactiveOnly: Bool + ) -> SnapshotPlanTerminalAction { + switch terminal { + case .throwOnAXFailure: + return .throwAxFailure + case .sparseWithFatalOnAXFailure: + return interactiveOnly ? .failClosed : .sparseBest + } + } + + static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool { + guard !nodes.isEmpty else { return false } + return nodes.allSatisfy { node in + // Application/Window labels are just the app/window name, and full-screen roots + // compute as hittable; neither says anything about tree health. + let isRootContainer = node.type == "Application" || node.type == "Window" + let hasContent = (!isRootContainer && node.label?.isEmpty == false) + || node.identifier?.isEmpty == false + || node.value?.isEmpty == false + return !hasContent + && (isRootContainer || !node.hittable) + && Self.structuralOnlyNodeTypes.contains(node.type) + } + } + + /// A leaf whose label joins many short segments is a container marked as an accessibility + /// element: the platform folds every descendant into one merged node. Nothing below it can + /// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it. + static func collapsedLeafIndexes(_ nodes: [SnapshotNode]) -> [Int]? { + let parents = Set(nodes.compactMap { $0.parentIndex }) + let collapsed = nodes.filter { node in + guard !parents.contains(node.index) else { return false } + guard !(node.type.lowercased().contains("text")) else { return false } + let label = node.label ?? "" + return label.split(separator: ",").count > collapsedLeafMinimumSegments + } + return collapsed.isEmpty ? nil : collapsed.map(\.index) + } + + // MARK: Outcome stamping + + private func stampedSnapshotPayload( + _ capture: SnapshotBackendCapture, + backend: SnapshotBackendKind, + state: String, + reason: (reason: String, code: String)? + ) -> DataPayload { + let payload = capture.payload + let quality = SnapshotQuality( + state: state, + backend: backend.rawValue, + reason: reason?.reason, + reasonCode: reason?.code, + effectiveDepth: capture.effectiveDepth, + collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? []) + ) + return DataPayload( + // Legacy human text for older daemons that read message instead of snapshotQuality. + message: Self.legacyQualityMessage(quality) ?? payload.message, + nodes: payload.nodes, + truncated: payload.truncated == true || state != "healthy" || capture.effectiveDepth != nil, + snapshotQuality: quality, + runnerFatal: payload.runnerFatal, + runnerFatalReason: payload.runnerFatalReason + ) + } + + static func legacyQualityMessage(_ quality: SnapshotQuality) -> String? { + guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil else { return nil } + var parts: [String] = [] + if quality.state == "recovered" { + let meaning = quality.reasonCode == "budget" + ? " The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen." + : " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears." + parts.append( + "Recovered this snapshot with the \(quality.backend) accessibility backend" + + (quality.reason.map { " after: \($0)." } ?? ".") + + meaning + ) + } + if quality.state == "sparse" { + parts.append( + "No snapshot backend could read this screen" + + (quality.reason.map { " (\($0))" } ?? "") + + ". Use screenshot as visual truth and coordinate taps." + ) + } + if let depth = quality.effectiveDepth { + parts.append( + "The accessibility server rejected deeper requests; this tree is capped at depth \(depth) — re-run with --depth \(depth) --scope for deeper content." + ) + } + return parts.isEmpty ? nil : parts.joined(separator: " ") + } +} + +// MARK: - In-bundle unit tests + +extension RunnerTests { + private func planTestNode( + index: Int, + type: String, + label: String? = nil, + identifier: String? = nil, + hittable: Bool = false, + parentIndex: Int? = nil + ) -> SnapshotNode { + SnapshotNode( + index: index, + type: type, + label: label, + identifier: identifier, + value: nil, + rect: snapshotRect(from: .zero), + enabled: true, + focused: nil, + selected: nil, + hittable: hittable, + depth: parentIndex == nil ? 0 : 1, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + func testSparsePayloadReasonMatrix() { + let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) + let window = planTestNode(index: 1, type: "Window", parentIndex: 0) + let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) + + // Labeled, hittable root over a bare window is still sparse. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) + // Deadline-truncated near-empty sweep needs recovery even with one real control. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) + // The same tiny tree from a completed sweep is a legitimately minimal screen. + XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) + // Empty payloads are degraded. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) + } + + func testCollapsedLeafIndexesFlagsMergedContainersOnly() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "), + parentIndex: 0 + ) + let prose = planTestNode( + index: 2, + type: "StaticText", + label: (0...30).map { "clause \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1]) + XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) + } + + func testLegacyQualityMessageStatesFallbackMeaning() { + let recovered = SnapshotQuality( + state: "recovered", + backend: "queries", + reason: "snapshot returned only structural application/window nodes", + reasonCode: "sparse-tree", + effectiveDepth: nil, + collapsedLeafIndexes: nil + ) + let message = Self.legacyQualityMessage(recovered) + XCTAssertTrue(message?.contains("queries accessibility backend") == true) + XCTAssertTrue(message?.contains("fixing the app's accessibility") == true) + XCTAssertTrue(message?.contains("screenshot as visual truth") == true) + XCTAssertNil( + Self.legacyQualityMessage( + SnapshotQuality( + state: "healthy", backend: "tree", reason: nil, reasonCode: nil, effectiveDepth: nil, + collapsedLeafIndexes: nil) + ) + ) + } + func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { + // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root + // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true), + .failClosed + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false), + .sparseBest + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true), + .throwAxFailure + ) + } + + func testSnapshotAccessibilityUnavailableCarriesSparseVerdict() { + currentApp = app + currentBundleId = "com.example.app" + defer { + currentApp = nil + currentBundleId = nil + } + let payload = snapshotAccessibilityUnavailable( + failure: SnapshotCaptureFailure( + code: "IOS_AX_SNAPSHOT_FAILED", + message: "kAXErrorIllegalArgument", + hint: "use screenshot" + ) + ) + XCTAssertEqual(payload.runnerFatal, true) + XCTAssertEqual(payload.snapshotQuality?.state, "sparse") + XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected") + } +} diff --git a/src/__tests__/runtime-snapshot.test.ts b/src/__tests__/runtime-snapshot.test.ts index b4c7f5288f..4496d864c8 100644 --- a/src/__tests__/runtime-snapshot.test.ts +++ b/src/__tests__/runtime-snapshot.test.ts @@ -191,6 +191,35 @@ test('runtime snapshot does not flag prose text or labeled containers with child assert.deepEqual(result.warnings ?? [], []); }); +test('runtime snapshot renders the structured quality verdict and skips legacy detectors', async () => { + const mergedLabel = Array.from({ length: 30 }, (_, i) => `Row ${i}, Tap`).join(', '); + const device = createSnapshotOnlyDevice({ + nodes: [ + { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'App' }, + { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Other', label: mergedLabel }, + ], + truncated: true, + backend: 'xctest', + quality: { + state: 'recovered', + backend: 'queries', + reason: 'snapshot returned only structural application/window nodes', + collapsedLeafIndexes: [1], + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.equal(result.warnings?.length, 2); + assert.match( + String(result.warnings?.[0]), + /Recovered this snapshot with the queries accessibility backend/, + ); + assert.match(String(result.warnings?.[0]), /fixing the app's accessibility is the real cure/); + assert.match(String(result.warnings?.[1]), /@e2 \[Other\] merges many labels/); + assert.deepEqual(result.snapshotQuality?.state, 'recovered'); +}); + test('runtime snapshot does not warn for a normal iOS compact interactive output', async () => { const device = createSnapshotOnlyDevice({ nodes: [ diff --git a/src/backend.ts b/src/backend.ts index 3f98026deb..b57d644f43 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,3 +1,4 @@ +import type { SnapshotQualityVerdict } from './utils/snapshot-quality.ts'; import type { AndroidSnapshotBackendMetadata } from './platforms/android/snapshot-types.ts'; import type { AlertAction, AlertInfo } from './alert-contract.ts'; import type { AppsFilter } from './contracts/app-inventory.ts'; @@ -47,6 +48,7 @@ export type BackendSnapshotResult = { androidSnapshot?: AndroidSnapshotBackendMetadata; freshness?: BackendSnapshotFreshness; warnings?: string[]; + quality?: SnapshotQualityVerdict; appName?: string; appBundleId?: string; }; diff --git a/src/client-shared.ts b/src/client-shared.ts index 13b3b5d682..f2c00afd9f 100644 --- a/src/client-shared.ts +++ b/src/client-shared.ts @@ -173,6 +173,7 @@ export function serializeSnapshotResult(result: CaptureSnapshotResult): Record 0 ? { warnings: result.warnings } : {}), ...(result.unchanged ? { unchanged: result.unchanged } : {}), }; diff --git a/src/client-types.ts b/src/client-types.ts index 507268d973..cbeb403db6 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -1,3 +1,4 @@ +import type { SnapshotQualityVerdict } from './utils/snapshot-quality.ts'; import type { DaemonResponseData, DaemonInstallSource, @@ -338,6 +339,7 @@ export type CaptureSnapshotResult = { warnings?: string[]; unchanged?: SnapshotUnchanged; identifiers: AgentDeviceIdentifiers; + snapshotQuality?: SnapshotQualityVerdict; }; export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { diff --git a/src/client.ts b/src/client.ts index 9127cd5ec9..352b192091 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,3 +1,4 @@ +import { readSnapshotQualityVerdict } from './utils/snapshot-quality.ts'; import { sendToDaemon } from './daemon-client.ts'; import { prepareMetroRuntime, reloadMetro } from './client-metro.ts'; import { resolveDaemonPaths } from './daemon/config.ts'; @@ -335,7 +336,10 @@ function normalizeSnapshotResult( function optionalSnapshotResponseFields( data: Record, ): Partial< - Pick + Pick< + CaptureSnapshotResult, + 'androidSnapshot' | 'unchanged' | 'visibility' | 'warnings' | 'snapshotQuality' + > > { const visibility = readObject(data.visibility); const androidSnapshot = readObject(data.androidSnapshot); @@ -343,8 +347,10 @@ function optionalSnapshotResponseFields( const warnings = Array.isArray(data.warnings) ? data.warnings.filter((entry): entry is string => typeof entry === 'string') : undefined; + const snapshotQuality = readSnapshotQualityVerdict(data.snapshotQuality); return { ...(visibility ? { visibility: visibility as CaptureSnapshotResult['visibility'] } : {}), + ...(snapshotQuality ? { snapshotQuality } : {}), ...(androidSnapshot ? { androidSnapshot: androidSnapshot as CaptureSnapshotResult['androidSnapshot'] } : {}), diff --git a/src/commands/capture-snapshot.ts b/src/commands/capture-snapshot.ts index 9896a703d2..407a528063 100644 --- a/src/commands/capture-snapshot.ts +++ b/src/commands/capture-snapshot.ts @@ -1,6 +1,10 @@ import type { BackendSnapshotResult } from '../backend.ts'; import type { AndroidSnapshotBackendMetadata } from '../platforms/android/snapshot-types.ts'; import type { AgentDeviceRuntime, CommandSessionRecord } from '../runtime-contract.ts'; +import { + renderSnapshotQualityWarnings, + type SnapshotQualityVerdict, +} from '../utils/snapshot-quality.ts'; import { AppError } from '../utils/errors.ts'; import { buildSnapshotDiff, countSnapshotComparableLines } from '../utils/snapshot-diff.ts'; import type { SnapshotDiffLine, SnapshotDiffSummary } from '../utils/snapshot-diff.ts'; @@ -34,6 +38,7 @@ export type SnapshotCommandResult = { androidSnapshot?: AndroidSnapshotBackendMetadata; warnings?: string[]; unchanged?: SnapshotUnchanged; + snapshotQuality?: SnapshotQualityVerdict; }; export type DiffSnapshotCommandResult = { @@ -75,6 +80,7 @@ export const snapshotCommand: RuntimeCommand< snapshotRaw: options.raw, }), ...(capture.result.androidSnapshot ? { androidSnapshot: capture.result.androidSnapshot } : {}), + ...(capture.result.quality ? { snapshotQuality: capture.result.quality } : {}), ...(capture.warnings.length > 0 ? { warnings: capture.warnings } : {}), ...(unchanged ? { unchanged } : {}), ...snapshotAppFields(capture), @@ -216,8 +222,15 @@ function buildSnapshotWarnings(params: { runtimeNow: number; }): string[] { const warnings = [...(params.result.warnings ?? [])]; + if (params.result.quality) { + warnings.push(...renderSnapshotQualityWarnings(params.result.quality, params.snapshot.nodes)); + } warnings.push(...buildEmptyAndroidInteractiveWarnings(params)); - warnings.push(...buildSparseIosInteractiveWarnings(params)); + if (!params.result.quality) { + // Legacy runners without a structured verdict keep the old daemon-side heuristics. + warnings.push(...buildSparseIosInteractiveWarnings(params)); + warnings.push(...buildMergedAccessibilityLeafWarnings(params.snapshot.nodes)); + } const helperFallbackWarning = formatAndroidHelperFallbackWarning(params.result.androidSnapshot); if (helperFallbackWarning) warnings.push(helperFallbackWarning); @@ -225,8 +238,6 @@ function buildSnapshotWarnings(params: { const reactNativeOverlayWarning = formatReactNativeOverlayWarning(params.snapshot.nodes); if (reactNativeOverlayWarning) warnings.push(reactNativeOverlayWarning); - warnings.push(...buildMergedAccessibilityLeafWarnings(params.snapshot.nodes)); - const recentDropWarning = formatRecentSnapshotDropWarning(params); if (recentDropWarning) warnings.push(recentDropWarning); diff --git a/src/core/interactors/apple.ts b/src/core/interactors/apple.ts index 46f56d8b07..d8f3060625 100644 --- a/src/core/interactors/apple.ts +++ b/src/core/interactors/apple.ts @@ -19,6 +19,10 @@ import type { DeviceInfo } from '../../utils/device.ts'; import { AppError } from '../../utils/errors.ts'; import type { RawSnapshotNode } from '../../utils/snapshot.ts'; import type { Interactor, RunnerContext } from '../interactor-types.ts'; +import { + readSnapshotQualityVerdict, + type SnapshotQualityVerdict, +} from '../../utils/snapshot-quality.ts'; export function createAppleInteractor( device: DeviceInfo, @@ -74,7 +78,9 @@ export function createAppleInteractor( nodes, truncated: result.truncated ?? false, backend: 'xctest', - ...(result.message ? { warnings: [result.message] } : {}), + ...(result.quality ? { quality: result.quality } : {}), + // Legacy runners without a quality verdict still surface their message text. + ...(!result.quality && result.message ? { warnings: [result.message] } : {}), }; }, back: async (mode) => { @@ -136,12 +142,13 @@ function readAppleSnapshotResult(result: Record): { nodes?: RawSnapshotNode[]; truncated?: boolean; message?: string; + quality?: SnapshotQualityVerdict; } { return { nodes: Array.isArray(result.nodes) ? (result.nodes as RawSnapshotNode[]) : undefined, truncated: typeof result.truncated === 'boolean' ? result.truncated : undefined, - // Runner-attached context (e.g. "recovered with the fallback accessibility backend") - // surfaces as a snapshot warning so fallbacks are never silent. + quality: readSnapshotQualityVerdict(result.snapshotQuality), + // Legacy runner context for builds that predate the structured verdict. message: typeof result.message === 'string' && result.message.trim().length > 0 ? result.message diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 8691c962ea..036bcb69c8 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -64,6 +64,7 @@ type SnapshotData = { analysis?: AndroidSnapshotAnalysis; androidSnapshot?: AndroidSnapshotBackendMetadata; warnings?: string[]; + quality?: unknown; }; type SnapshotAttempt = { @@ -78,6 +79,7 @@ type CaptureSnapshotResult = { androidSnapshot?: AndroidSnapshotBackendMetadata; freshness?: AndroidFreshnessCaptureMeta; warnings?: string[]; + quality?: unknown; }; type AndroidFreshnessReason = 'empty-interactive' | 'sharp-drop' | 'stuck-route'; @@ -97,6 +99,7 @@ export async function captureSnapshot( analysis: data.analysis, androidSnapshot: data.androidSnapshot, warnings: data.warnings, + quality: data.quality, }; } @@ -184,6 +187,8 @@ async function captureInteractionOutcomeAwareSnapshot( analysis: latest.data.analysis, androidSnapshot: latest.data.androidSnapshot, freshness: latest.freshness, + warnings: latest.data.warnings, + quality: latest.data.quality, }; } @@ -249,6 +254,7 @@ async function captureAndroidFreshnessAwareSnapshot( androidSnapshot: latest.data.androidSnapshot, freshness: latest.freshness, warnings: latest.data.warnings, + quality: latest.data.quality, }; } @@ -303,6 +309,7 @@ async function capturePostGestureAwareSnapshot( androidSnapshot: latest.data.androidSnapshot, freshness: latest.freshness, warnings: latest.data.warnings, + quality: latest.data.quality, }; } diff --git a/src/daemon/snapshot-runtime.ts b/src/daemon/snapshot-runtime.ts index f652b8376c..607667911f 100644 --- a/src/daemon/snapshot-runtime.ts +++ b/src/daemon/snapshot-runtime.ts @@ -8,6 +8,7 @@ import type { DaemonRequest, DaemonResponse, DaemonResponseData, SessionState } import { SessionStore } from './session-store.ts'; import { errorResponse } from './handlers/response.ts'; import { captureSnapshot, resolveSnapshotScope } from './handlers/snapshot-capture.ts'; +import { readSnapshotQualityVerdict } from '../utils/snapshot-quality.ts'; import { buildSnapshotSession, resolveSessionDevice, @@ -291,6 +292,7 @@ function createDaemonSnapshotBackend(params: { androidSnapshot: capture.androidSnapshot, freshness: capture.freshness, warnings: capture.warnings, + quality: readSnapshotQualityVerdict(capture.quality), appName: session?.appBundleId ? (session.appName ?? session.appBundleId) : undefined, appBundleId: session?.appBundleId, }; diff --git a/src/utils/__tests__/snapshot-quality.test.ts b/src/utils/__tests__/snapshot-quality.test.ts new file mode 100644 index 0000000000..f101cfc674 --- /dev/null +++ b/src/utils/__tests__/snapshot-quality.test.ts @@ -0,0 +1,43 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; + +import { readSnapshotQualityVerdict } from '../snapshot-quality.ts'; + +test('readSnapshotQualityVerdict accepts a well-formed verdict', () => { + const verdict = readSnapshotQualityVerdict({ + state: 'recovered', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'budget', + effectiveDepth: 56, + collapsedLeafIndexes: [3], + }); + assert.deepEqual(verdict, { + state: 'recovered', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'budget', + effectiveDepth: 56, + collapsedLeafIndexes: [3], + }); +}); + +test('readSnapshotQualityVerdict rejects unknown state or backend as verdict-absent', () => { + // A malformed object must not be treated as an authoritative verdict — it has to fall through + // so legacy node-shape detectors still run instead of being silently suppressed. + assert.equal(readSnapshotQualityVerdict({ state: 'bogus', backend: 'tree' }), undefined); + assert.equal(readSnapshotQualityVerdict({ state: 'sparse', backend: 'mystery' }), undefined); + assert.equal(readSnapshotQualityVerdict({ backend: 'tree' }), undefined); + assert.equal(readSnapshotQualityVerdict(null), undefined); +}); + +test('readSnapshotQualityVerdict keeps the verdict but drops an unknown reasonCode', () => { + // Forward-compat: a newer runner adding a reasonCode must still yield a usable verdict. + const verdict = readSnapshotQualityVerdict({ + state: 'sparse', + backend: 'queries', + reasonCode: 'future-code', + }); + assert.equal(verdict?.state, 'sparse'); + assert.equal(verdict?.reasonCode, undefined); +}); diff --git a/src/utils/output.ts b/src/utils/output.ts index 45d166b087..7d7d249b3f 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -611,8 +611,11 @@ function buildSnapshotNotices( helperPresentation: AndroidHelperPresentationInput = { nodes, filteredCount: 0 }, ): string[] { const notices = readSnapshotWarnings(data); - const sparseSnapshotHint = formatSparseSnapshotHint(nodes, options); - if (sparseSnapshotHint) notices.push(sparseSnapshotHint); + // The structured snapshot quality verdict already carries a sharper version of this hint. + if (!data.snapshotQuality) { + const sparseSnapshotHint = formatSparseSnapshotHint(nodes, options); + if (sparseSnapshotHint) notices.push(sparseSnapshotHint); + } if (!options.raw && helperPresentation.filteredCount > 0) { notices.push( `Collapsed ${helperPresentation.filteredCount} Android helper node${helperPresentation.filteredCount === 1 ? '' : 's'} from the agent-facing text snapshot; use --raw or --json for the full hierarchy.`, diff --git a/src/utils/snapshot-quality.ts b/src/utils/snapshot-quality.ts new file mode 100644 index 0000000000..fd9404a14a --- /dev/null +++ b/src/utils/snapshot-quality.ts @@ -0,0 +1,127 @@ +import type { SnapshotNode } from './snapshot.ts'; + +/** + * Structured quality verdict computed once by the iOS runner's snapshot capture plan. + * The daemon renders it; it never re-derives degradation from node shapes. + */ +export type SnapshotQualityVerdict = { + state: 'healthy' | 'recovered' | 'sparse'; + backend: 'tree' | 'queries' | 'private-ax'; + reason?: string; + reasonCode?: 'ax-rejected' | 'sparse-tree' | 'budget' | 'no-nodes' | 'capture-failed'; + effectiveDepth?: number; + collapsedLeafIndexes?: number[]; +}; + +const SNAPSHOT_QUALITY_STATES = new Set([ + 'healthy', + 'recovered', + 'sparse', +]); +const SNAPSHOT_QUALITY_BACKENDS = new Set([ + 'tree', + 'queries', + 'private-ax', +]); +const SNAPSHOT_QUALITY_REASON_CODES = new Set>([ + 'ax-rejected', + 'sparse-tree', + 'budget', + 'no-nodes', + 'capture-failed', +]); + +export function readSnapshotQualityVerdict(value: unknown): SnapshotQualityVerdict | undefined { + if (!value || typeof value !== 'object') return undefined; + const raw = value as Record; + // Validate the load-bearing union fields: an object with an unknown state/backend is not a + // verdict this version understands, so it falls through as verdict-absent and the legacy + // node-shape detectors run instead of being silently suppressed by a malformed payload. + if ( + typeof raw.state !== 'string' || + !SNAPSHOT_QUALITY_STATES.has(raw.state as SnapshotQualityVerdict['state']) + ) { + return undefined; + } + if ( + typeof raw.backend !== 'string' || + !SNAPSHOT_QUALITY_BACKENDS.has(raw.backend as SnapshotQualityVerdict['backend']) + ) { + return undefined; + } + return { + state: raw.state as SnapshotQualityVerdict['state'], + backend: raw.backend as SnapshotQualityVerdict['backend'], + reason: typeof raw.reason === 'string' ? raw.reason : undefined, + // An unknown reasonCode is dropped, not rejected: a forward-version runner that adds one + // still yields a usable verdict (only the budget-specific wording is keyed off it). + reasonCode: + typeof raw.reasonCode === 'string' && + SNAPSHOT_QUALITY_REASON_CODES.has( + raw.reasonCode as NonNullable, + ) + ? (raw.reasonCode as SnapshotQualityVerdict['reasonCode']) + : undefined, + effectiveDepth: typeof raw.effectiveDepth === 'number' ? raw.effectiveDepth : undefined, + collapsedLeafIndexes: Array.isArray(raw.collapsedLeafIndexes) + ? raw.collapsedLeafIndexes.filter((entry): entry is number => typeof entry === 'number') + : undefined, + }; +} + +/** Canonical warning lines for a verdict; the single place degradation is worded. */ +export function renderSnapshotQualityWarnings( + verdict: SnapshotQualityVerdict, + nodes: Pick[], +): string[] { + return [ + ...stateWarning(verdict), + ...depthWarning(verdict), + ...collapsedLeafWarnings(verdict, nodes), + ]; +} + +function stateWarning(verdict: SnapshotQualityVerdict): string[] { + if (verdict.state === 'recovered') { + const meaning = + verdict.reasonCode === 'budget' + ? ' The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen.' + : " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears."; + return [ + `Recovered this snapshot with the ${verdict.backend} accessibility backend` + + (verdict.reason ? ` after: ${verdict.reason}.` : '.') + + meaning, + ]; + } + if (verdict.state === 'sparse') { + return [ + 'No snapshot backend could read this screen' + + (verdict.reason ? ` (${verdict.reason})` : '') + + '. Use screenshot as visual truth and coordinate taps; retry snapshot after navigating.', + ]; + } + return []; +} + +function depthWarning(verdict: SnapshotQualityVerdict): string[] { + if (verdict.effectiveDepth === undefined) return []; + return [ + `The accessibility server rejected deeper requests; this tree is capped at depth ${verdict.effectiveDepth} — re-run with --depth ${verdict.effectiveDepth} --scope for deeper content.`, + ]; +} + +function collapsedLeafWarnings( + verdict: SnapshotQualityVerdict, + nodes: Pick[], +): string[] { + const warnings: string[] = []; + for (const index of verdict.collapsedLeafIndexes ?? []) { + const node = nodes.find((entry) => entry.index === index); + if (!node) continue; + const name = node.identifier ? ` (${node.identifier})` : ''; + warnings.push( + `@${node.ref} [${node.type ?? 'element'}]${name} merges many labels into a single accessibility element. The app likely marks a container as accessible, which hides every descendant from assistive tech and automation — the children cannot be addressed individually. Fix the app's accessibility (mark the rows, not the container); until then use screenshot as visual truth and coordinate taps.`, + ); + } + return warnings; +}