Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/platforms/android/__tests__/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,14 @@ test('snapshotAndroid derives hidden content hints for interactive snapshots fro
const scrollArea = result.nodes.find((node) => node.type === 'android.widget.ScrollView');

assert.ok(scrollArea);
assert.equal(
result.nodes.some((node) => node.type === 'android.view.ViewGroup'),
false,
);
assert.equal(
result.nodes.some((node) => node.label === 'Offscreen message'),
false,
);
assert.equal(scrollArea?.hiddenContentAbove, undefined);
assert.equal(scrollArea?.hiddenContentBelow, true);
});
Expand Down
35 changes: 30 additions & 5 deletions src/platforms/android/scroll-hints.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { RawSnapshotNode, Rect } from '../../utils/snapshot.ts';
import type { HiddenContentHint, RawSnapshotNode, Rect } from '../../utils/snapshot.ts';
import { isScrollableType } from '../../utils/scrollable.ts';

type ViewNode = {
Expand All @@ -17,16 +17,36 @@ export function annotateAndroidScrollableContentHints(
nodes: RawSnapshotNode[],
activityTopDump: string,
): void {
const hintsByIndex = deriveAndroidScrollableContentHints(nodes, activityTopDump);
for (const node of nodes) {
const hint = hintsByIndex.get(node.index);
if (!hint) {
continue;
}
if (hint.hiddenContentAbove) {
node.hiddenContentAbove = true;
}
if (hint.hiddenContentBelow) {
node.hiddenContentBelow = true;
}
}
}

export function deriveAndroidScrollableContentHints(
nodes: RawSnapshotNode[],
activityTopDump: string,
): Map<number, HiddenContentHint> {
const viewTree = parseActivityTopViewTree(activityTopDump);
if (!viewTree) {
return;
return new Map();
}

const nativeScrollViews = collectNativeScrollViews(viewTree);
if (nativeScrollViews.length === 0) {
return;
return new Map();
}

const hintsByIndex = new Map<number, HiddenContentHint>();
for (const node of nodes) {
if (!node.rect || !isScrollableType(node.type)) {
continue;
Expand All @@ -44,13 +64,18 @@ export function annotateAndroidScrollableContentHints(
if (!hiddenContent) {
continue;
}
const hint: HiddenContentHint = {};
if (hiddenContent.above) {
node.hiddenContentAbove = true;
hint.hiddenContentAbove = true;
}
if (hiddenContent.below) {
node.hiddenContentBelow = true;
hint.hiddenContentBelow = true;
}
if (hint.hiddenContentAbove || hint.hiddenContentBelow) {
hintsByIndex.set(node.index, hint);
}
}
return hintsByIndex;
}

type NativeScrollView = {
Expand Down
112 changes: 66 additions & 46 deletions src/platforms/android/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,24 @@ import { runCmd } from '../../utils/exec.ts';
import { withRetry } from '../../utils/retry.ts';
import { AppError } from '../../utils/errors.ts';
import type { DeviceInfo } from '../../utils/device.ts';
import { attachRefs, type RawSnapshotNode, type SnapshotOptions } from '../../utils/snapshot.ts';
import {
attachRefs,
type HiddenContentHint,
type RawSnapshotNode,
type SnapshotOptions,
} from '../../utils/snapshot.ts';
import { isScrollableType } from '../../utils/scrollable.ts';
import { buildMobileSnapshotPresentation } from '../../utils/mobile-snapshot-semantics.ts';
import { deriveMobileSnapshotHiddenContentHints } from '../../utils/mobile-snapshot-semantics.ts';
import {
buildUiHierarchySnapshot,
parseUiHierarchy,
parseUiHierarchyTree,
type AndroidBuiltSnapshot,
type AndroidSnapshotAnalysis,
type AndroidUiHierarchy,
} from './ui-hierarchy.ts';
import { adbArgs } from './adb.ts';
import { annotateAndroidScrollableContentHints } from './scroll-hints.ts';
import { deriveAndroidScrollableContentHints } from './scroll-hints.ts';

export async function snapshotAndroid(
device: DeviceInfo,
Expand All @@ -25,34 +32,38 @@ export async function snapshotAndroid(
const xml = await dumpUiHierarchy(device);
if (!options.interactiveOnly) {
const parsed = parseUiHierarchy(xml, 800, options);
await annotateScrollableContentHintsIfNeeded(device, parsed.nodes);
const nativeHints = await deriveScrollableContentHintsIfNeeded(device, parsed.nodes);
applyHiddenContentHintsToNodes(nativeHints, parsed.nodes);
return parsed;
}

const tree = parseUiHierarchyTree(xml);
const parsed = buildUiHierarchySnapshot(tree, 800, { ...options, interactiveOnly: false });
await annotateScrollableContentHintsIfNeeded(device, parsed.nodes);
applyDerivedPresentationHiddenContentHints(parsed.nodes);
applyHiddenContentHintsToSourceNodes(parsed);
const { sourceNodes: _sourceNodes, ...interactiveSnapshot } = buildUiHierarchySnapshot(
tree,
800,
options,
);
return interactiveSnapshot;
const fullSnapshot = buildUiHierarchySnapshot(tree, 800, { ...options, interactiveOnly: false });
const interactiveSnapshot = buildUiHierarchySnapshot(tree, 800, options);
const nativeHints = await deriveScrollableContentHintsIfNeeded(device, fullSnapshot.nodes);
applyHiddenContentHintsToInteractiveNodes(nativeHints, fullSnapshot, interactiveSnapshot);
if (nativeHints.size === 0) {
const presentationHints = deriveMobileSnapshotHiddenContentHints(
attachRefs(fullSnapshot.nodes),
);
applyHiddenContentHintsToInteractiveNodes(presentationHints, fullSnapshot, interactiveSnapshot);
}
const { sourceNodes: _sourceNodes, ...snapshot } = interactiveSnapshot;
return snapshot;
}

async function annotateScrollableContentHintsIfNeeded(
async function deriveScrollableContentHintsIfNeeded(
device: DeviceInfo,
nodes: RawSnapshotNode[],
): Promise<void> {
): Promise<Map<number, HiddenContentHint>> {
if (!nodes.some((node) => isScrollableType(node.type))) {
return;
return new Map();
}
const activityTopDump = await dumpActivityTop(device);
if (activityTopDump) {
annotateAndroidScrollableContentHints(nodes, activityTopDump);
if (!activityTopDump) {
return new Map();
}
return deriveAndroidScrollableContentHints(nodes, activityTopDump);
}

export async function dumpUiHierarchy(device: DeviceInfo): Promise<string> {
Expand Down Expand Up @@ -137,41 +148,50 @@ async function dumpActivityTop(device: DeviceInfo): Promise<string | null> {
}
}

function applyHiddenContentHintsToSourceNodes(
parsed: ReturnType<typeof buildUiHierarchySnapshot>,
function applyHiddenContentHintsToInteractiveNodes(
hintsByFullNodeIndex: ReadonlyMap<number, HiddenContentHint>,
fullSnapshot: AndroidBuiltSnapshot,
interactiveSnapshot: AndroidBuiltSnapshot,
): void {
// `tree` is parsed fresh for each snapshot call, so mutating the paired source nodes here
// is scoped to this invocation and feeds the interactive rebuild below.
for (const [index, sourceNode] of parsed.sourceNodes.entries()) {
const snapshotNode = parsed.nodes[index];
if (!snapshotNode) {
if (hintsByFullNodeIndex.size === 0) {
return;
}

// Both snapshots come from one parsed hierarchy, so source node identity is the stable bridge
// between full geometry context and the pruned interactive output.
const interactiveNodesBySource = new Map<AndroidUiHierarchy, RawSnapshotNode>();
for (const [index, sourceNode] of interactiveSnapshot.sourceNodes.entries()) {
const node = interactiveSnapshot.nodes[index];
if (node) {
interactiveNodesBySource.set(sourceNode, node);
}
}

for (const [fullIndex, hint] of hintsByFullNodeIndex) {
const sourceNode = fullSnapshot.sourceNodes[fullIndex];
if (!sourceNode) {
continue;
}
if (snapshotNode.hiddenContentAbove) {
sourceNode.hiddenContentAbove = true;
const interactiveNode = interactiveNodesBySource.get(sourceNode);
if (!interactiveNode) {
continue;
}
if (snapshotNode.hiddenContentBelow) {
sourceNode.hiddenContentBelow = true;
if (hint.hiddenContentAbove) {
interactiveNode.hiddenContentAbove = true;
}
if (hint.hiddenContentBelow) {
interactiveNode.hiddenContentBelow = true;
}
}
}

function applyDerivedPresentationHiddenContentHints(nodes: RawSnapshotNode[]): void {
if (
nodes.length === 0 ||
nodes.some((node) => node.hiddenContentAbove || node.hiddenContentBelow)
) {
return;
}
const presentation = buildMobileSnapshotPresentation(attachRefs(nodes));
const hintsByIndex = new Map(
presentation.nodes
.filter((node) => node.hiddenContentAbove || node.hiddenContentBelow)
.map((node) => [node.index, node] as const),
);
for (const node of nodes) {
const hint = hintsByIndex.get(node.index);
if (!hint) {
function applyHiddenContentHintsToNodes(
hintsByIndex: ReadonlyMap<number, HiddenContentHint>,
nodes: RawSnapshotNode[],
): void {
for (const [index, hint] of hintsByIndex) {
const node = nodes[index];
if (!node) {
continue;
}
if (hint.hiddenContentAbove) {
Expand Down
72 changes: 58 additions & 14 deletions src/utils/mobile-snapshot-semantics.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { isRectVisibleInViewport, resolveViewportRect } from './rect-visibility.ts';
import { inferVerticalScrollIndicatorDirections } from './scroll-indicator.ts';
import type { Rect, SnapshotNode } from './snapshot.ts';
import type { HiddenContentHint, Rect, SnapshotNode } from './snapshot.ts';
import { buildSnapshotNodeMap, displayNodeLabel } from './snapshot-tree.ts';
import { isScrollableNodeLike } from './scrollable.ts';

Expand All @@ -17,19 +17,8 @@ export function buildMobileSnapshotPresentation(nodes: SnapshotNode[]): MobileSn
return { nodes, hiddenCount: 0, summaryLines: [] };
}

const byIndex = buildSnapshotNodeMap(nodes);
const visibleNodeIndexes = new Set<number>();
const offscreenNodes: SnapshotNode[] = [];

for (const node of nodes) {
if (isNodeVisibleInEffectiveViewport(node, nodes, byIndex)) {
markNodeAndAncestorsVisible(node, visibleNodeIndexes, byIndex);
continue;
}
offscreenNodes.push(node);
}

const hintedContainers = deriveContainerHints(nodes, offscreenNodes, visibleNodeIndexes, byIndex);
const { byIndex, visibleNodeIndexes, offscreenNodes, hintedContainers } =
analyzeMobileSnapshotVisibility(nodes);
const presentedNodes =
visibleNodeIndexes.size === 0
? nodes
Expand All @@ -52,6 +41,42 @@ export function buildMobileSnapshotPresentation(nodes: SnapshotNode[]): MobileSn
};
}

export function deriveMobileSnapshotHiddenContentHints(
nodes: SnapshotNode[],
): Map<number, HiddenContentHint> {
if (nodes.length === 0) {
return new Map();
}

const { hintedContainers } = analyzeMobileSnapshotVisibility(nodes);
return toHiddenContentHints(hintedContainers.directionsByContainer);
}

function analyzeMobileSnapshotVisibility(nodes: SnapshotNode[]): {
byIndex: Map<number, SnapshotNode>;
visibleNodeIndexes: Set<number>;
offscreenNodes: SnapshotNode[];
hintedContainers: {
directionsByContainer: Map<number, Set<Direction>>;
coveredNodeIndexes: Set<number>;
};
} {
const byIndex = buildSnapshotNodeMap(nodes);
const visibleNodeIndexes = new Set<number>();
const offscreenNodes: SnapshotNode[] = [];

for (const node of nodes) {
if (isNodeVisibleInEffectiveViewport(node, nodes, byIndex)) {
markNodeAndAncestorsVisible(node, visibleNodeIndexes, byIndex);
continue;
}
offscreenNodes.push(node);
}

const hintedContainers = deriveContainerHints(nodes, offscreenNodes, visibleNodeIndexes, byIndex);
return { byIndex, visibleNodeIndexes, offscreenNodes, hintedContainers };
}

export function isNodeVisibleInEffectiveViewport(
node: Pick<SnapshotNode, 'rect' | 'index' | 'parentIndex' | 'type' | 'role' | 'subrole'>,
nodes: SnapshotNode[],
Expand Down Expand Up @@ -114,6 +139,25 @@ function deriveContainerHints(
return { directionsByContainer, coveredNodeIndexes };
}

function toHiddenContentHints(
directionsByContainer: Map<number, Set<Direction>>,
): Map<number, HiddenContentHint> {
const hints = new Map<number, HiddenContentHint>();
for (const [index, directions] of directionsByContainer) {
const hint: HiddenContentHint = {};
if (directions.has('above')) {
hint.hiddenContentAbove = true;
}
if (directions.has('below')) {
hint.hiddenContentBelow = true;
}
if (hint.hiddenContentAbove || hint.hiddenContentBelow) {
hints.set(index, hint);
}
}
return hints;
}

function applyDerivedHiddenContentHints(
node: SnapshotNode,
directionsByContainer: Map<number, Set<Direction>>,
Expand Down
5 changes: 5 additions & 0 deletions src/utils/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ export type RawSnapshotNode = {
hiddenContentBelow?: boolean;
};

export type HiddenContentHint = {
hiddenContentAbove?: true;
hiddenContentBelow?: true;
};

export type SnapshotNode = RawSnapshotNode & {
ref: string;
};
Expand Down
Loading