diff --git a/src/compat/maestro/__tests__/runtime-assertions.test.ts b/src/compat/maestro/__tests__/runtime-assertions.test.ts index c83fd1f613..ded03fa15e 100644 --- a/src/compat/maestro/__tests__/runtime-assertions.test.ts +++ b/src/compat/maestro/__tests__/runtime-assertions.test.ts @@ -555,6 +555,39 @@ test('invokeMaestroAssertVisible does not use Android raw fallback for generated ); }); +test('invokeMaestroAssertVisible bounds Android verification retries after native wait succeeds', async () => { + vi.useFakeTimers(); + + const calls: Array<[string, string[] | undefined]> = []; + const responsePromise = invokeMaestroAssertVisible({ + baseReq: { + token: 't', + session: 's', + flags: { platform: 'android' }, + }, + positionals: ['label="Input" || text="Input" || id="Input"', '60000'], + invoke: async (req): Promise => { + calls.push([req.command, req.positionals]); + if (req.command === 'snapshot') { + return { ok: true, data: snapshot([node('Loading')]) }; + } + if (req.command === 'wait') return { ok: true, data: { matches: 1 } }; + return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; + }, + }); + + await vi.advanceTimersByTimeAsync(6500); + const response = await responsePromise; + + assert.equal(response.ok, false); + assert.deepEqual(calls.slice(0, 3), [ + ['wait', ['Input', '60000']], + ['snapshot', []], + ['snapshot', []], + ]); + assert.ok(calls.filter(([command]) => command === 'snapshot').length < 40); +}); + test('invokeMaestroAssertVisible writes terminal snapshot artifacts for failed attempts', async () => { const artifactsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-assert-artifacts-')); try { diff --git a/src/compat/maestro/runtime-assertions.ts b/src/compat/maestro/runtime-assertions.ts index 3348900bbe..700b9797d8 100644 --- a/src/compat/maestro/runtime-assertions.ts +++ b/src/compat/maestro/runtime-assertions.ts @@ -99,7 +99,7 @@ async function invokeNativeMaestroVisibleWaitWithSnapshotFallback( nativeStartedAt, ); if (failedSample.kind === 'return') return failedSample.response; - return await invokeSnapshotMaestroAssertVisible(params, args); + return await invokeSnapshotMaestroAssertVisible(params, visibleAssertionRetryArgs(args)); } } rememberMaestroVisibleContext(params.scope, args.selector); @@ -277,7 +277,7 @@ async function confirmVisibleAfterAndroidRecovery( ): Promise { const retryArgs = { ...args, - timeoutMs: Math.min(args.timeoutMs, MAESTRO_ASSERTION_POLICY.assertVisibleRetryTimeoutMs), + timeoutMs: visibleAssertionRetryTimeoutMs(args.timeoutMs), }; const nativeWaitQuery = readNativeVisibleWaitQuery(params.baseReq, retryArgs.selector); if (!nativeWaitQuery) return await invokeSnapshotMaestroAssertVisible(params, retryArgs); @@ -431,6 +431,19 @@ function readVisibleAssertionDeadlineAction(params: { : 'finish'; } +function visibleAssertionRetryArgs( + args: MaestroVisibilityAssertionArgs, +): MaestroVisibilityAssertionArgs { + return { + ...args, + timeoutMs: visibleAssertionRetryTimeoutMs(args.timeoutMs), + }; +} + +function visibleAssertionRetryTimeoutMs(timeoutMs: number): number { + return Math.min(timeoutMs, MAESTRO_ASSERTION_POLICY.assertVisibleRetryTimeoutMs); +} + function isReactNativeOverlayBlockingAssertion(response: DaemonResponse): boolean { return ( !response.ok && diff --git a/src/compat/maestro/runtime-click.ts b/src/compat/maestro/runtime-click.ts new file mode 100644 index 0000000000..fced71eb29 --- /dev/null +++ b/src/compat/maestro/runtime-click.ts @@ -0,0 +1,19 @@ +import type { DaemonResponse } from '../../daemon/types.ts'; +import type { Point } from '../../kernel/snapshot.ts'; +import type { MaestroRuntimeInvoke, ReplayBaseRequest } from './runtime-support.ts'; + +export async function invokeMaestroClickPoint(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + point: Point; +}): Promise { + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [String(params.point.x), String(params.point.y)], + flags: { + ...params.baseReq.flags, + postGestureStabilization: true, + }, + }); +} diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts index 2def284c73..1c73e16cde 100644 --- a/src/compat/maestro/runtime-interactions.ts +++ b/src/compat/maestro/runtime-interactions.ts @@ -8,8 +8,10 @@ import { type ScrollDirection, } from '../../core/scroll-gesture.ts'; import type { ReplayVarScope } from '../../replay/vars.ts'; +import type { SnapshotState } from '../../kernel/snapshot.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { sleep } from '../../utils/timeouts.ts'; +import { invokeMaestroClickPoint } from './runtime-click.ts'; import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts'; import { captureMaestroSnapshot, @@ -140,15 +142,7 @@ export async function invokeMaestroTapPointPercent(params: { } const point = pointFromPercent(frame, xPercent, yPercent); - const response = await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [String(point.x), String(point.y)], - flags: { - ...params.baseReq.flags, - postGestureStabilization: true, - }, - }); + const response = await invokeMaestroClickPoint({ ...params, point }); if (response.ok) clearMaestroRecoverableInteraction(params.scope); return response; } @@ -520,15 +514,7 @@ async function clickMaestroResolvedTarget( point, }, }); - const response = await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [String(point.x), String(point.y)], - flags: { - ...params.baseReq.flags, - postGestureStabilization: true, - }, - }); + const response = await invokeMaestroClickPoint({ ...params, point }); if (response.ok) { clearMaestroVisibleContext(params.scope); rememberMaestroRecoverableInteraction(params.scope, { @@ -586,7 +572,8 @@ async function resolveMaestroInteractionTarget( commandLabel: string, resolutionOptions: { promoteTapTarget: boolean }, ): Promise< - { ok: true; target: ResolvedMaestroInteractionTarget } | { ok: false; response: DaemonResponse } + | { ok: true; target: ResolvedMaestroInteractionTarget; snapshot: SnapshotState } + | { ok: false; response: DaemonResponse } > { const snapshotResponse = await captureMaestroSnapshot({ ...params, raw: true }); return resolveMaestroInteractionTargetFromResponse( @@ -610,7 +597,7 @@ function resolveMaestroInteractionTargetFromResponse( resolutionOptions: { promoteTapTarget: boolean }, snapshotResponse: DaemonResponse, ): - | { ok: true; target: ResolvedMaestroInteractionTarget } + | { ok: true; target: ResolvedMaestroInteractionTarget; snapshot: SnapshotState } | { ok: false; response: DaemonResponse } { if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse }; const snapshot = readSnapshotState(snapshotResponse.data); @@ -650,6 +637,7 @@ function resolveMaestroInteractionTargetFromResponse( rect: fuzzyResolution.rect, frame, }, + snapshot, }; } } @@ -671,6 +659,7 @@ function resolveMaestroInteractionTargetFromResponse( rect: resolution.rect, frame, }, + snapshot, }; } diff --git a/src/platforms/android/input-ownership.ts b/src/core/android-input-ownership.ts similarity index 100% rename from src/platforms/android/input-ownership.ts rename to src/core/android-input-ownership.ts diff --git a/src/daemon/handlers/__tests__/snapshot-capture.test.ts b/src/daemon/handlers/__tests__/snapshot-capture.test.ts index 2f8a3ae0a9..f8aac6eb06 100644 --- a/src/daemon/handlers/__tests__/snapshot-capture.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-capture.test.ts @@ -139,6 +139,98 @@ test('buildSnapshotState marks content covered by floating overlays as visible b expect(state.nodes.some((node) => node.type === 'TabBar')).toBe(true); }); +test('buildSnapshotState marks Android app content covered by IME overlays as blocked', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'android.widget.FrameLayout', + bundleId: 'org.example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Push Article', + bundleId: 'org.example', + rect: { x: 40, y: 600, width: 180, height: 56 }, + hittable: true, + }, + { + index: 2, + depth: 1, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: 0, y: 400, width: 390, height: 444 }, + }, + ], + backend: 'android', + }, + undefined, + ); + + expect(state.nodes.find((node) => node.label === 'Push Article')).toMatchObject({ + hittable: false, + interactionBlocked: 'covered', + presentationHints: ['covered'], + }); +}); + +test('buildSnapshotState treats large Android IME subtrees as one overlay root', () => { + const imeChildren = Array.from({ length: 2000 }, (_, offset) => ({ + index: offset + 3, + depth: 2, + parentIndex: 2, + type: 'android.widget.TextView', + label: `Keyboard suggestion ${offset}`, + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: offset % 300, y: 500 + (offset % 200), width: 80, height: 32 }, + })); + + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'android.widget.FrameLayout', + bundleId: 'org.example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Covered action', + bundleId: 'org.example', + rect: { x: 40, y: 620, width: 180, height: 56 }, + hittable: true, + }, + { + index: 2, + depth: 1, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: 0, y: 400, width: 390, height: 444 }, + }, + ...imeChildren, + ], + backend: 'android', + }, + undefined, + ); + + expect(state.nodes.find((node) => node.label === 'Covered action')).toMatchObject({ + hittable: false, + interactionBlocked: 'covered', + }); +}); + test('buildSnapshotState does not treat later generic hittable containers as covers', () => { const state = buildSnapshotState( { diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 7cfff64ea3..8cccd61444 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -16,6 +16,7 @@ import { import { annotateCoveredSnapshotNodes } from '../../snapshot/snapshot-occlusion.ts'; import { normalizeSnapshotTree } from '../../snapshot/snapshot-tree.ts'; export { buildSnapshotVisibility } from '../../snapshot/snapshot-visibility.ts'; +import { isAndroidInputMethodSnapshotNode } from '../../snapshot/android-input-method-overlays.ts'; import type { SessionState } from '../types.ts'; import { ANDROID_FRESHNESS_RETRY_DEADLINE_MS, @@ -402,7 +403,12 @@ export function buildSnapshotState( ? presentIosInteractiveSnapshot(scopedNodes) : scopedNodes; const nodes = attachRefs( - snapshotRaw ? presentableNodes : annotateCoveredSnapshotNodes(presentableNodes), + snapshotRaw + ? presentableNodes + : annotateCoveredSnapshotNodes(presentableNodes, { + isAdditionalOverlayNode: + data?.backend === 'android' ? isAndroidInputMethodSnapshotNode : undefined, + }), ); const snapshotQuality = snapshotCaptureAnnotationsFrom(data).quality; return { diff --git a/src/platforms/android/__tests__/input-ownership.test.ts b/src/platforms/android/__tests__/input-ownership.test.ts index 814f827051..f9ffacddfb 100644 --- a/src/platforms/android/__tests__/input-ownership.test.ts +++ b/src/platforms/android/__tests__/input-ownership.test.ts @@ -4,7 +4,7 @@ import { classifyAndroidInputOwnership, parseAndroidInputMethodPackage, readAndroidActiveInputMethodPackage, -} from '../input-ownership.ts'; +} from '../../../core/android-input-ownership.ts'; test('classifies active input method package as IME-owned', () => { assert.deepEqual( diff --git a/src/platforms/android/__tests__/snapshot-content-recovery.test.ts b/src/platforms/android/__tests__/snapshot-content-recovery.test.ts new file mode 100644 index 0000000000..468c3219dc --- /dev/null +++ b/src/platforms/android/__tests__/snapshot-content-recovery.test.ts @@ -0,0 +1,110 @@ +import { test, expect } from 'vitest'; +import { classifyAndroidHelperContentRecovery } from '../snapshot-content-recovery.ts'; + +test('keeps known IME blocking windows instead of falling back to covered app content', () => { + const decision = classifyAndroidHelperContentRecovery( + helperXml([ + node({ + windowType: 1, + packageName: 'org.reactnavigation.playground', + className: 'android.widget.FrameLayout', + }), + node({ + windowType: 2, + packageName: 'com.google.android.inputmethod.latin', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Try out your stylus', + packageName: 'com.google.android.inputmethod.latin', + className: 'android.widget.TextView', + }), + node({ + text: 'Cancel', + resourceId: 'android:id/closeButton', + packageName: 'com.google.android.inputmethod.latin', + className: 'android.widget.Button', + }), + ]), + { + backend: 'android-helper', + nodeCount: 4, + rootPresent: true, + windowCount: 2, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'org.reactnavigation.playground' }, + ); + + expect(decision).toBeUndefined(); +}); + +test('falls back when helper output has only one meaningful IME node', () => { + const decision = classifyAndroidHelperContentRecovery( + helperXml([ + node({ + windowType: 1, + packageName: 'org.reactnavigation.playground', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Try out your stylus', + packageName: 'com.google.android.inputmethod.latin', + className: 'android.widget.TextView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 2, + rootPresent: true, + windowCount: 2, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'org.reactnavigation.playground' }, + ); + + expect(decision?.reason).toBe('content-poor-app-window'); + expect(decision?.diagnostics.helperInputMethodMeaningfulNodeCount).toBe(1); +}); + +test('falls back when helper output has no foreground app or IME content', () => { + const decision = classifyAndroidHelperContentRecovery( + helperXml([ + node({ + windowType: 1, + packageName: 'org.reactnavigation.playground', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Unrelated overlay', + packageName: 'com.example.overlay', + className: 'android.widget.TextView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 2, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'org.reactnavigation.playground' }, + ); + + expect(decision?.reason).toBe('content-poor-app-window'); + expect(decision?.diagnostics.helperInputMethodMeaningfulNodeCount).toBe(0); +}); + +function helperXml(nodes: string[]): string { + return `${nodes.join('')}`; +} + +function node(options: { + text?: string; + resourceId?: string; + packageName: string; + className: string; + windowType?: number; +}): string { + return ``; +} diff --git a/src/platforms/android/device-input-state.ts b/src/platforms/android/device-input-state.ts index 5702be031c..8ab93982d7 100644 --- a/src/platforms/android/device-input-state.ts +++ b/src/platforms/android/device-input-state.ts @@ -13,7 +13,7 @@ import { isFallbackAndroidInputMethodResource, readAndroidActiveInputMethodPackage, type AndroidInputOwner, -} from './input-ownership.ts'; +} from '../../core/android-input-ownership.ts'; const ANDROID_INPUT_TYPE_CLASS_MASK = 0x0000000f; const ANDROID_INPUT_TYPE_CLASS_TEXT = 0x00000001; diff --git a/src/platforms/android/fill-verification.ts b/src/platforms/android/fill-verification.ts index f4461c4d3d..7a165e434d 100644 --- a/src/platforms/android/fill-verification.ts +++ b/src/platforms/android/fill-verification.ts @@ -10,7 +10,7 @@ import { } from '../fill-diagnostics.ts'; import { sleep } from './adb.ts'; import { getAndroidKeyboardState } from './device-input-state.ts'; -import { isAndroidInputMethodOwnedNode } from './input-ownership.ts'; +import { isAndroidInputMethodOwnedNode } from '../../core/android-input-ownership.ts'; import { captureAndroidUiHierarchyXml } from './snapshot.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; diff --git a/src/platforms/android/snapshot-content-recovery.ts b/src/platforms/android/snapshot-content-recovery.ts index a9ae3417fc..cdc81cd976 100644 --- a/src/platforms/android/snapshot-content-recovery.ts +++ b/src/platforms/android/snapshot-content-recovery.ts @@ -1,10 +1,14 @@ import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; +import { isAndroidInputMethodOwnedNode } from '../../core/android-input-ownership.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; const ANDROID_WINDOW_TYPE_APPLICATION = 1; const MAX_REPORTED_WINDOW_TYPES = 8; const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2; +const MIN_INPUT_METHOD_MEANINGFUL_NODES = 2; const ANDROID_SYSTEM_PACKAGES = new Set(['android', 'com.android.systemui']); +const INSUFFICIENT_APP_CONTENT_REASON = + 'Android snapshot helper returned insufficient application window content'; export type AndroidHelperContentRecoveryDecision = { reason: 'empty-helper-output' | 'system-window-only' | 'content-poor-app-window'; @@ -16,6 +20,7 @@ export type AndroidHelperContentRecoveryDecision = { helperMeaningfulNodeCount: number; helperApplicationMeaningfulNodeCount: number; helperNonSystemMeaningfulNodeCount: number; + helperInputMethodMeaningfulNodeCount: number; helperForegroundAppMeaningfulNodeCount?: number; helperForegroundAppPackage?: string; helperForegroundAppMeaningfulNodeThreshold?: number; @@ -32,7 +37,7 @@ export function classifyAndroidHelperContentRecovery( if (metadata.backend !== 'android-helper') return undefined; const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage); - if (summary.nodeCount === 0 || metadata.nodeCount === 0 || metadata.rootPresent === false) { + if (isEmptyHelperOutput(summary, metadata)) { return buildRecoveryDecision( summary, metadata, @@ -41,13 +46,8 @@ export function classifyAndroidHelperContentRecovery( ); } - const foregroundAppMeaningfulNodeCount = summary.foregroundAppMeaningfulNodeCount; - if ( - foregroundAppMeaningfulNodeCount !== undefined && - (foregroundAppMeaningfulNodeCount === 0 || - (foregroundAppMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES && - summary.meaningfulNodeCount > foregroundAppMeaningfulNodeCount)) - ) { + if (isForegroundAppContentHiddenByInputMethod(summary)) return undefined; + if (isForegroundAppContentPoor(summary)) { return buildRecoveryDecision( summary, metadata, @@ -56,34 +56,25 @@ export function classifyAndroidHelperContentRecovery( ); } - if ( - foregroundAppMeaningfulNodeCount === undefined && - summary.applicationWindowRootCount > 0 && - summary.applicationMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES - ) { + if (isApplicationWindowContentPoor(summary)) { return buildRecoveryDecision( summary, metadata, 'content-poor-app-window', - 'Android snapshot helper returned insufficient application window content', + INSUFFICIENT_APP_CONTENT_REASON, ); } - if ( - foregroundAppMeaningfulNodeCount === undefined && - summary.windowRootCount === 0 && - (metadata.windowCount ?? 0) > 1 && - summary.nonSystemMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES - ) { + if (isWindowlessMultiWindowContentPoor(summary, metadata)) { return buildRecoveryDecision( summary, metadata, 'content-poor-app-window', - 'Android snapshot helper returned insufficient application window content', + INSUFFICIENT_APP_CONTENT_REASON, ); } - if (summary.windowRootCount > 0 && summary.applicationWindowRootCount === 0) { + if (isSystemWindowOnly(summary)) { return buildRecoveryDecision( summary, metadata, @@ -95,6 +86,54 @@ export function classifyAndroidHelperContentRecovery( return undefined; } +function isEmptyHelperOutput( + summary: AndroidHelperXmlSummary, + metadata: AndroidSnapshotBackendMetadata, +): boolean { + return summary.nodeCount === 0 || metadata.nodeCount === 0 || metadata.rootPresent === false; +} + +function isForegroundAppContentHiddenByInputMethod(summary: AndroidHelperXmlSummary): boolean { + return ( + isForegroundAppContentPoor(summary) && + summary.inputMethodMeaningfulNodeCount >= MIN_INPUT_METHOD_MEANINGFUL_NODES + ); +} + +function isForegroundAppContentPoor(summary: AndroidHelperXmlSummary): boolean { + const foregroundCount = summary.foregroundAppMeaningfulNodeCount; + if (foregroundCount === undefined) return false; + if (foregroundCount === 0) return true; + return ( + foregroundCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES && + summary.meaningfulNodeCount > foregroundCount + ); +} + +function isApplicationWindowContentPoor(summary: AndroidHelperXmlSummary): boolean { + return ( + summary.foregroundAppMeaningfulNodeCount === undefined && + summary.applicationWindowRootCount > 0 && + summary.applicationMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES + ); +} + +function isWindowlessMultiWindowContentPoor( + summary: AndroidHelperXmlSummary, + metadata: AndroidSnapshotBackendMetadata, +): boolean { + return ( + summary.foregroundAppMeaningfulNodeCount === undefined && + summary.windowRootCount === 0 && + (metadata.windowCount ?? 0) > 1 && + summary.nonSystemMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES + ); +} + +function isSystemWindowOnly(summary: AndroidHelperXmlSummary): boolean { + return summary.windowRootCount > 0 && summary.applicationWindowRootCount === 0; +} + function buildRecoveryDecision( summary: AndroidHelperXmlSummary, metadata: AndroidSnapshotBackendMetadata, @@ -115,6 +154,7 @@ type AndroidHelperXmlSummary = { meaningfulNodeCount: number; applicationMeaningfulNodeCount: number; nonSystemMeaningfulNodeCount: number; + inputMethodMeaningfulNodeCount: number; foregroundAppPackage?: string; foregroundAppMeaningfulNodeCount?: number; windowTypes: number[]; @@ -148,6 +188,7 @@ function createAndroidHelperXmlSummaryState( meaningfulNodeCount: 0, applicationMeaningfulNodeCount: 0, nonSystemMeaningfulNodeCount: 0, + inputMethodMeaningfulNodeCount: 0, ...(foregroundAppPackage !== undefined ? { foregroundAppPackage, foregroundAppMeaningfulNodeCount: 0 } : {}), @@ -191,6 +232,14 @@ function recordAndroidHelperMeaningfulNode( if (!isAndroidSystemPackage(node.packageName)) { summary.nonSystemMeaningfulNodeCount += 1; } + if ( + isAndroidInputMethodOwnedNode({ + packageName: node.packageName, + resourceId: node.resourceId, + }) + ) { + summary.inputMethodMeaningfulNodeCount += 1; + } if ( summary.foregroundAppPackage !== undefined && node.packageName === summary.foregroundAppPackage @@ -209,6 +258,7 @@ function finalizeAndroidHelperXmlSummary( meaningfulNodeCount: summary.meaningfulNodeCount, applicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount, nonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount, + inputMethodMeaningfulNodeCount: summary.inputMethodMeaningfulNodeCount, ...(summary.foregroundAppPackage !== undefined ? { foregroundAppPackage: summary.foregroundAppPackage, @@ -243,6 +293,7 @@ function buildRecoveryDiagnostics( helperMeaningfulNodeCount: summary.meaningfulNodeCount, helperApplicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount, helperNonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount, + helperInputMethodMeaningfulNodeCount: summary.inputMethodMeaningfulNodeCount, ...(summary.foregroundAppPackage !== undefined ? { helperForegroundAppPackage: summary.foregroundAppPackage, diff --git a/src/snapshot/android-input-method-overlays.ts b/src/snapshot/android-input-method-overlays.ts new file mode 100644 index 0000000000..2bd29a3bf9 --- /dev/null +++ b/src/snapshot/android-input-method-overlays.ts @@ -0,0 +1,12 @@ +import { classifyAndroidInputOwnership } from '../core/android-input-ownership.ts'; +import type { RawSnapshotNode } from '../kernel/snapshot.ts'; + +export function isAndroidInputMethodSnapshotNode( + node: Pick | undefined, +): boolean { + if (!node) return false; + return classifyAndroidInputOwnership({ + packageName: node.bundleId, + resourceId: node.identifier, + }).inputMethodOwned; +} diff --git a/src/snapshot/snapshot-occlusion.ts b/src/snapshot/snapshot-occlusion.ts index b097e93f18..522907eceb 100644 --- a/src/snapshot/snapshot-occlusion.ts +++ b/src/snapshot/snapshot-occlusion.ts @@ -34,20 +34,32 @@ const SEMANTIC_TOUCH_KIND_FRAGMENTS = [ type OcclusionScan = { nodes: RawSnapshotNode[]; byIndex: Map; + overlayPositions: number[]; }; -export function annotateCoveredSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { +export type SnapshotOcclusionOptions = { + isAdditionalOverlayNode?: (node: RawSnapshotNode) => boolean; +}; + +export function annotateCoveredSnapshotNodes( + nodes: RawSnapshotNode[], + options: SnapshotOcclusionOptions = {}, +): RawSnapshotNode[] { if (nodes.length < 2) return nodes; const annotated = [...nodes]; + const byIndex = new Map(annotated.map((node) => [node.index, node])); const scan: OcclusionScan = { nodes: annotated, - byIndex: new Map(annotated.map((node) => [node.index, node])), + byIndex, + overlayPositions: annotated.flatMap((node, position) => + isOverlayLikeNode(node, byIndex, options) ? [position] : [], + ), }; let changed = false; for (const [position, node] of annotated.entries()) { if (!isCandidateTouchNode(node)) continue; - const cover = findCoveringNode(scan, position, node); + const cover = findCoveringNode(scan, position, node, options); if (!cover) continue; changed = true; const coveredNode = { @@ -73,14 +85,18 @@ function findCoveringNode( scan: OcclusionScan, targetPosition: number, target: RawSnapshotNode, + options: SnapshotOcclusionOptions, ): RawSnapshotNode | null { const targetRect = positiveRect(target.rect); if (!targetRect) return null; const center = centerOfRect(targetRect); - for (let position = targetPosition + 1; position < scan.nodes.length; position += 1) { + for (const position of scan.overlayPositions) { + if (position <= targetPosition) continue; const candidate = scan.nodes[position]; - if (candidate && canCoverPoint(scan, position, target, targetRect, center)) return candidate; + if (candidate && canCoverPoint(scan, position, target, targetRect, center, options)) { + return candidate; + } } return null; @@ -92,10 +108,11 @@ function canCoverPoint( target: RawSnapshotNode, targetRect: Rect, point: { x: number; y: number }, + options: SnapshotOcclusionOptions, ): boolean { const candidate = scan.nodes[candidatePosition]; if (!candidate) return false; - const coverRect = visibleCoverRect(scan, candidatePosition, target, targetRect); + const coverRect = visibleCoverRect(scan, candidatePosition, target, targetRect, options); return Boolean(coverRect && containsPoint(coverRect, point.x, point.y)); } @@ -104,13 +121,14 @@ function visibleCoverRect( candidatePosition: number, target: RawSnapshotNode, targetRect: Rect, + options: SnapshotOcclusionOptions, ): Rect | null { const candidate = scan.nodes[candidatePosition]; - if (!candidate || !isOverlayLikeNode(candidate)) return null; + if (!candidate || !isOverlayLikeNode(candidate, scan.byIndex, options)) return null; if (areRelatedSnapshotNodes(target, candidate, scan.byIndex)) return null; const candidateRect = positiveRect(candidate.rect); if (!candidateRect || areRectsApproximatelyEqual(targetRect, candidateRect)) return null; - if (findCoveringNode(scan, candidatePosition, candidate)) return null; + if (findCoveringNode(scan, candidatePosition, candidate, options)) return null; return candidateRect; } @@ -121,12 +139,55 @@ function isCandidateTouchNode(node: RawSnapshotNode): boolean { return Boolean(node.label?.trim() || node.value?.trim() || node.identifier?.trim()); } -function isOverlayLikeNode(node: RawSnapshotNode): boolean { +function isOverlayLikeNode( + node: RawSnapshotNode, + byIndex: Map, + options: SnapshotOcclusionOptions, +): boolean { if (!positiveRect(node.rect)) return false; if (isViewportRoot(node)) return false; // This is a presentation-order heuristic: only known floating UI chrome should cover // later targets. Generic hittable containers can appear later without being visually on top. - return nodeKindIncludesAny(node, OVERLAY_KIND_FRAGMENTS); + return ( + nodeKindIncludesAny(node, OVERLAY_KIND_FRAGMENTS) || + isAdditionalOverlayRootNode(node, byIndex, options) + ); +} + +function isAdditionalOverlayRootNode( + node: RawSnapshotNode, + byIndex: Map, + options: SnapshotOcclusionOptions, +): boolean { + if (options.isAdditionalOverlayNode?.(node) !== true) return false; + return !hasRenderableAdditionalOverlayAncestor(node, byIndex, options); +} + +function hasRenderableAdditionalOverlayAncestor( + node: RawSnapshotNode, + byIndex: Map, + options: SnapshotOcclusionOptions, +): boolean { + let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; + const visited = new Set(); + while (current && !visited.has(current.index)) { + if (isRenderableAdditionalOverlayNode(current, options)) return true; + visited.add(current.index); + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + return false; +} + +function isRenderableAdditionalOverlayNode( + node: RawSnapshotNode, + options: SnapshotOcclusionOptions, +): boolean { + return ( + options.isAdditionalOverlayNode?.(node) === true && + positiveRect(node.rect) !== null && + !isViewportRoot(node) + ); } function isSemanticTouchNode(node: RawSnapshotNode): boolean {