diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index f00d559ec9..46e5fda965 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -394,6 +394,14 @@ test('snapshotAndroid derives hidden content hints for interactive snapshots fro const scrollArea = result.nodes.find((node) => node.type === 'android.widget.ScrollView'); assert.ok(scrollArea); + assert.equal( + result.nodes.some((node) => node.type === 'android.view.ViewGroup'), + false, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Offscreen message'), + false, + ); assert.equal(scrollArea?.hiddenContentAbove, undefined); assert.equal(scrollArea?.hiddenContentBelow, true); }); diff --git a/src/platforms/android/scroll-hints.ts b/src/platforms/android/scroll-hints.ts index 9b456dccc0..9212c78a39 100644 --- a/src/platforms/android/scroll-hints.ts +++ b/src/platforms/android/scroll-hints.ts @@ -1,4 +1,4 @@ -import type { RawSnapshotNode, Rect } from '../../utils/snapshot.ts'; +import type { HiddenContentHint, RawSnapshotNode, Rect } from '../../utils/snapshot.ts'; import { isScrollableType } from '../../utils/scrollable.ts'; type ViewNode = { @@ -17,16 +17,36 @@ export function annotateAndroidScrollableContentHints( nodes: RawSnapshotNode[], activityTopDump: string, ): void { + const hintsByIndex = deriveAndroidScrollableContentHints(nodes, activityTopDump); + for (const node of nodes) { + const hint = hintsByIndex.get(node.index); + if (!hint) { + continue; + } + if (hint.hiddenContentAbove) { + node.hiddenContentAbove = true; + } + if (hint.hiddenContentBelow) { + node.hiddenContentBelow = true; + } + } +} + +export function deriveAndroidScrollableContentHints( + nodes: RawSnapshotNode[], + activityTopDump: string, +): Map { const viewTree = parseActivityTopViewTree(activityTopDump); if (!viewTree) { - return; + return new Map(); } const nativeScrollViews = collectNativeScrollViews(viewTree); if (nativeScrollViews.length === 0) { - return; + return new Map(); } + const hintsByIndex = new Map(); for (const node of nodes) { if (!node.rect || !isScrollableType(node.type)) { continue; @@ -44,13 +64,18 @@ export function annotateAndroidScrollableContentHints( if (!hiddenContent) { continue; } + const hint: HiddenContentHint = {}; if (hiddenContent.above) { - node.hiddenContentAbove = true; + hint.hiddenContentAbove = true; } if (hiddenContent.below) { - node.hiddenContentBelow = true; + hint.hiddenContentBelow = true; + } + if (hint.hiddenContentAbove || hint.hiddenContentBelow) { + hintsByIndex.set(node.index, hint); } } + return hintsByIndex; } type NativeScrollView = { diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index 2952220b87..922fa33471 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -2,17 +2,24 @@ import { runCmd } from '../../utils/exec.ts'; import { withRetry } from '../../utils/retry.ts'; import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; -import { attachRefs, type RawSnapshotNode, type SnapshotOptions } from '../../utils/snapshot.ts'; +import { + attachRefs, + type HiddenContentHint, + type RawSnapshotNode, + type SnapshotOptions, +} from '../../utils/snapshot.ts'; import { isScrollableType } from '../../utils/scrollable.ts'; -import { buildMobileSnapshotPresentation } from '../../utils/mobile-snapshot-semantics.ts'; +import { deriveMobileSnapshotHiddenContentHints } from '../../utils/mobile-snapshot-semantics.ts'; import { buildUiHierarchySnapshot, parseUiHierarchy, parseUiHierarchyTree, + type AndroidBuiltSnapshot, type AndroidSnapshotAnalysis, + type AndroidUiHierarchy, } from './ui-hierarchy.ts'; import { adbArgs } from './adb.ts'; -import { annotateAndroidScrollableContentHints } from './scroll-hints.ts'; +import { deriveAndroidScrollableContentHints } from './scroll-hints.ts'; export async function snapshotAndroid( device: DeviceInfo, @@ -25,34 +32,38 @@ export async function snapshotAndroid( const xml = await dumpUiHierarchy(device); if (!options.interactiveOnly) { const parsed = parseUiHierarchy(xml, 800, options); - await annotateScrollableContentHintsIfNeeded(device, parsed.nodes); + const nativeHints = await deriveScrollableContentHintsIfNeeded(device, parsed.nodes); + applyHiddenContentHintsToNodes(nativeHints, parsed.nodes); return parsed; } const tree = parseUiHierarchyTree(xml); - const parsed = buildUiHierarchySnapshot(tree, 800, { ...options, interactiveOnly: false }); - await annotateScrollableContentHintsIfNeeded(device, parsed.nodes); - applyDerivedPresentationHiddenContentHints(parsed.nodes); - applyHiddenContentHintsToSourceNodes(parsed); - const { sourceNodes: _sourceNodes, ...interactiveSnapshot } = buildUiHierarchySnapshot( - tree, - 800, - options, - ); - return interactiveSnapshot; + const fullSnapshot = buildUiHierarchySnapshot(tree, 800, { ...options, interactiveOnly: false }); + const interactiveSnapshot = buildUiHierarchySnapshot(tree, 800, options); + const nativeHints = await deriveScrollableContentHintsIfNeeded(device, fullSnapshot.nodes); + applyHiddenContentHintsToInteractiveNodes(nativeHints, fullSnapshot, interactiveSnapshot); + if (nativeHints.size === 0) { + const presentationHints = deriveMobileSnapshotHiddenContentHints( + attachRefs(fullSnapshot.nodes), + ); + applyHiddenContentHintsToInteractiveNodes(presentationHints, fullSnapshot, interactiveSnapshot); + } + const { sourceNodes: _sourceNodes, ...snapshot } = interactiveSnapshot; + return snapshot; } -async function annotateScrollableContentHintsIfNeeded( +async function deriveScrollableContentHintsIfNeeded( device: DeviceInfo, nodes: RawSnapshotNode[], -): Promise { +): Promise> { if (!nodes.some((node) => isScrollableType(node.type))) { - return; + return new Map(); } const activityTopDump = await dumpActivityTop(device); - if (activityTopDump) { - annotateAndroidScrollableContentHints(nodes, activityTopDump); + if (!activityTopDump) { + return new Map(); } + return deriveAndroidScrollableContentHints(nodes, activityTopDump); } export async function dumpUiHierarchy(device: DeviceInfo): Promise { @@ -137,41 +148,50 @@ async function dumpActivityTop(device: DeviceInfo): Promise { } } -function applyHiddenContentHintsToSourceNodes( - parsed: ReturnType, +function applyHiddenContentHintsToInteractiveNodes( + hintsByFullNodeIndex: ReadonlyMap, + fullSnapshot: AndroidBuiltSnapshot, + interactiveSnapshot: AndroidBuiltSnapshot, ): void { - // `tree` is parsed fresh for each snapshot call, so mutating the paired source nodes here - // is scoped to this invocation and feeds the interactive rebuild below. - for (const [index, sourceNode] of parsed.sourceNodes.entries()) { - const snapshotNode = parsed.nodes[index]; - if (!snapshotNode) { + if (hintsByFullNodeIndex.size === 0) { + return; + } + + // Both snapshots come from one parsed hierarchy, so source node identity is the stable bridge + // between full geometry context and the pruned interactive output. + const interactiveNodesBySource = new Map(); + for (const [index, sourceNode] of interactiveSnapshot.sourceNodes.entries()) { + const node = interactiveSnapshot.nodes[index]; + if (node) { + interactiveNodesBySource.set(sourceNode, node); + } + } + + for (const [fullIndex, hint] of hintsByFullNodeIndex) { + const sourceNode = fullSnapshot.sourceNodes[fullIndex]; + if (!sourceNode) { continue; } - if (snapshotNode.hiddenContentAbove) { - sourceNode.hiddenContentAbove = true; + const interactiveNode = interactiveNodesBySource.get(sourceNode); + if (!interactiveNode) { + continue; } - if (snapshotNode.hiddenContentBelow) { - sourceNode.hiddenContentBelow = true; + if (hint.hiddenContentAbove) { + interactiveNode.hiddenContentAbove = true; + } + if (hint.hiddenContentBelow) { + interactiveNode.hiddenContentBelow = true; } } } -function applyDerivedPresentationHiddenContentHints(nodes: RawSnapshotNode[]): void { - if ( - nodes.length === 0 || - nodes.some((node) => node.hiddenContentAbove || node.hiddenContentBelow) - ) { - return; - } - const presentation = buildMobileSnapshotPresentation(attachRefs(nodes)); - const hintsByIndex = new Map( - presentation.nodes - .filter((node) => node.hiddenContentAbove || node.hiddenContentBelow) - .map((node) => [node.index, node] as const), - ); - for (const node of nodes) { - const hint = hintsByIndex.get(node.index); - if (!hint) { +function applyHiddenContentHintsToNodes( + hintsByIndex: ReadonlyMap, + nodes: RawSnapshotNode[], +): void { + for (const [index, hint] of hintsByIndex) { + const node = nodes[index]; + if (!node) { continue; } if (hint.hiddenContentAbove) { diff --git a/src/utils/mobile-snapshot-semantics.ts b/src/utils/mobile-snapshot-semantics.ts index 1fa852e713..16a1a43c5f 100644 --- a/src/utils/mobile-snapshot-semantics.ts +++ b/src/utils/mobile-snapshot-semantics.ts @@ -1,6 +1,6 @@ import { isRectVisibleInViewport, resolveViewportRect } from './rect-visibility.ts'; import { inferVerticalScrollIndicatorDirections } from './scroll-indicator.ts'; -import type { Rect, SnapshotNode } from './snapshot.ts'; +import type { HiddenContentHint, Rect, SnapshotNode } from './snapshot.ts'; import { buildSnapshotNodeMap, displayNodeLabel } from './snapshot-tree.ts'; import { isScrollableNodeLike } from './scrollable.ts'; @@ -17,19 +17,8 @@ export function buildMobileSnapshotPresentation(nodes: SnapshotNode[]): MobileSn return { nodes, hiddenCount: 0, summaryLines: [] }; } - const byIndex = buildSnapshotNodeMap(nodes); - const visibleNodeIndexes = new Set(); - const offscreenNodes: SnapshotNode[] = []; - - for (const node of nodes) { - if (isNodeVisibleInEffectiveViewport(node, nodes, byIndex)) { - markNodeAndAncestorsVisible(node, visibleNodeIndexes, byIndex); - continue; - } - offscreenNodes.push(node); - } - - const hintedContainers = deriveContainerHints(nodes, offscreenNodes, visibleNodeIndexes, byIndex); + const { byIndex, visibleNodeIndexes, offscreenNodes, hintedContainers } = + analyzeMobileSnapshotVisibility(nodes); const presentedNodes = visibleNodeIndexes.size === 0 ? nodes @@ -52,6 +41,42 @@ export function buildMobileSnapshotPresentation(nodes: SnapshotNode[]): MobileSn }; } +export function deriveMobileSnapshotHiddenContentHints( + nodes: SnapshotNode[], +): Map { + if (nodes.length === 0) { + return new Map(); + } + + const { hintedContainers } = analyzeMobileSnapshotVisibility(nodes); + return toHiddenContentHints(hintedContainers.directionsByContainer); +} + +function analyzeMobileSnapshotVisibility(nodes: SnapshotNode[]): { + byIndex: Map; + visibleNodeIndexes: Set; + offscreenNodes: SnapshotNode[]; + hintedContainers: { + directionsByContainer: Map>; + coveredNodeIndexes: Set; + }; +} { + const byIndex = buildSnapshotNodeMap(nodes); + const visibleNodeIndexes = new Set(); + const offscreenNodes: SnapshotNode[] = []; + + for (const node of nodes) { + if (isNodeVisibleInEffectiveViewport(node, nodes, byIndex)) { + markNodeAndAncestorsVisible(node, visibleNodeIndexes, byIndex); + continue; + } + offscreenNodes.push(node); + } + + const hintedContainers = deriveContainerHints(nodes, offscreenNodes, visibleNodeIndexes, byIndex); + return { byIndex, visibleNodeIndexes, offscreenNodes, hintedContainers }; +} + export function isNodeVisibleInEffectiveViewport( node: Pick, nodes: SnapshotNode[], @@ -114,6 +139,25 @@ function deriveContainerHints( return { directionsByContainer, coveredNodeIndexes }; } +function toHiddenContentHints( + directionsByContainer: Map>, +): Map { + const hints = new Map(); + for (const [index, directions] of directionsByContainer) { + const hint: HiddenContentHint = {}; + if (directions.has('above')) { + hint.hiddenContentAbove = true; + } + if (directions.has('below')) { + hint.hiddenContentBelow = true; + } + if (hint.hiddenContentAbove || hint.hiddenContentBelow) { + hints.set(index, hint); + } + } + return hints; +} + function applyDerivedHiddenContentHints( node: SnapshotNode, directionsByContainer: Map>, diff --git a/src/utils/snapshot.ts b/src/utils/snapshot.ts index f1cde15bcd..4d7c474a93 100644 --- a/src/utils/snapshot.ts +++ b/src/utils/snapshot.ts @@ -41,6 +41,11 @@ export type RawSnapshotNode = { hiddenContentBelow?: boolean; }; +export type HiddenContentHint = { + hiddenContentAbove?: true; + hiddenContentBelow?: true; +}; + export type SnapshotNode = RawSnapshotNode & { ref: string; };