Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions docs/adr/0012-interactive-replay.md
Original file line number Diff line number Diff line change
Expand Up @@ -481,10 +481,22 @@ number of suggestions available at default/full) so a caller knows whether a re-
has material; default and full carry at most **20** screen refs and **5** suggestions ranked per
decision 1's total order. These counts are absolute, including error payloads. Individual
labels, ids, selectors, source paths, mismatch values, cause messages, and hints are UTF-8 truncated to
**256 bytes**; an action summary has no positional array, and fill text, expanded variables, and arbitrary
nested cause details are never serialized. All rendered strings and any overflow artifact pass through the
central diagnostics redactor before truncation. The report sets truncation/redaction markers for every
omission.
**256 bytes**; an action summary has no positional array, and arbitrary nested cause details are never
serialized. Maestro failure provenance renders resolved diagnostic identifiers, including targets and
`runFlow` paths, so the report names what the runtime actually attempted instead of emitting an unresolved
`${VAR}` or synthetic `<var:VAR>` token. Text-entry payloads remain semantic secrets: `inputText` progress,
failure messages, suggestions, and overflow artifacts never serialize the entered text. Injected replay
values are not registered as global sensitive literals: a short ordinary value such as `2` or `on` would
otherwise corrupt unrelated timestamps, paths, and typed error fields throughout the request log.
Text-entry values are registered at the actual dispatch boundary before platform work, independently of
the user-facing failure projection. Users must not place secrets in selectors, links, filenames, or other
diagnostic identifiers that are expected to appear in failure output.

Native `.ad` replay retains its categorical `<var:NAME>` replacement in human-readable divergence
messages, hints, and bounded diagnostic fields. That existing fail-closed policy is intentionally
separate from Maestro compatibility output; semantically masked positionals and daemon-owned
machine-readable fields and paths are never substring-rewritten. The report sets truncation/redaction
markers for every omission.

When the bounded form would omit material, the daemon writes the same redacted, bounded-per-field detail
to a session-scoped divergence artifact and returns its path plus `overflow: { omittedBytes, artifactPath
Expand Down
41 changes: 41 additions & 0 deletions src/compat/maestro/__tests__/daemon-runtime-port.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,53 @@ import path from 'node:path';
import { expect, test, vi } from 'vitest';
import type { DaemonInvokeFn, DaemonRequest } from '../../../daemon/types.ts';
import { PNG } from '../../../utils/png.ts';
import {
emitDiagnostic,
flushDiagnosticsToSessionFile,
withDiagnosticsScope,
} from '../../../utils/diagnostics.ts';
import { createDaemonMaestroRuntimePort } from '../daemon-runtime-port.ts';
import { MAESTRO_OBSERVATION_POLL_MS } from '../daemon-runtime-port-observation.ts';
import { parseMaestroProgram } from '../program-ir-parser.ts';
import { executeMaestroProgram } from './runtime-port-fixtures.ts';
import { makeBaseRequest, makeDependencies, makeSnapshot } from './daemon-runtime-port-fixtures.ts';

test('registers Maestro inputText as sensitive before nested platform work', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-input-diagnostics-'));
const logPath = path.join(root, 'request.ndjson');
const text = 'opaque-maestro-input';
const port = createDaemonMaestroRuntimePort({
baseReq: makeBaseRequest({ flags: { platform: 'android', replayBackend: 'maestro' } }),
invoke: async (request) => {
if (request.command === 'type') {
emitDiagnostic({
phase: 'platform_echo',
data: { message: `Backend echoed ${request.positionals?.[0]}` },
});
}
return request.command === 'snapshot'
? { ok: true, data: { nodes: [], createdAt: 0 } }
: { ok: true, data: {} };
},
dependencies: makeDependencies(),
platform: 'android',
});

await withDiagnosticsScope({ command: 'replay', logPath }, async () => {
await port.execute({
command: { kind: 'inputText', source: { line: 2 }, text },
generation: 0,
env: {},
invalidateObservation() {},
});
flushDiagnosticsToSessionFile({ force: true });
});

const diagnostics = fs.readFileSync(logPath, 'utf8');
expect(diagnostics).not.toContain(text);
expect(diagnostics).toContain('Backend echoed [REDACTED]');
});

test('delegates lifecycle and coordinate gestures through public daemon commands', async () => {
const requests: DaemonRequest[] = [];
const invoke: DaemonInvokeFn = async (request) => {
Expand Down
26 changes: 20 additions & 6 deletions src/compat/maestro/__tests__/engine-context.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { expect, test, vi } from 'vitest';
import { maestroTestFailure } from '../compatibility-errors.ts';
import { createMaestroExecutionContext } from '../engine-context.ts';
import type { MaestroRuntimePort } from '../engine-types.ts';
import { parseMaestroProgram } from '../program-ir-parser.ts';
import { executeMaestroProgram } from './runtime-port-fixtures.ts';

test('resolves transitive scoped variables to their final value', () => {
test('resolves transitive scoped variables', () => {
const context = createMaestroExecutionContext();
const leave = context.enter({
TARGET: '${NEXT}',
Expand All @@ -13,22 +14,35 @@ test('resolves transitive scoped variables to their final value', () => {
});

expect(context.resolve('${TARGET}')).toBe('Done');
expect(context.expandedVariables).toEqual({ TARGET: 'Done' });
leave();
});

test('retains expanded values after nested scopes unwind', () => {
test('keeps nested scopes valid until they unwind', () => {
const context = createMaestroExecutionContext();
const rootLeave = context.enter({ SECRET: 'nested-scope-secret' });
const nestedLeave = context.enter({ TARGET: '${SECRET}' });

expect(context.resolve('${TARGET}')).toBe('nested-scope-secret');
nestedLeave();
rootLeave();
});

expect(context.expandedVariables).toEqual({
TARGET: 'nested-scope-secret',
});
test('renders resolved target variables in optional-step warnings', async () => {
const target = 'Missing checkout button';
const program = parseMaestroProgram(
['---', '- tapOn:', ' text: ${TARGET}', ' optional: true'].join('\n'),
{ sourcePath: '/flows/optional.yaml' },
);
const port: MaestroRuntimePort = {
execute: vi.fn(async () => {
throw maestroTestFailure(`Missing ${target}`);
}),
observe: vi.fn(async ({ generation }) => ({ generation, matched: true })),
};

const result = await executeMaestroProgram(program, port, { env: { TARGET: target } });

expect(result.warnings).toEqual([expect.stringContaining(target)]);
});

test('rejects cyclic references instead of recursing indefinitely', () => {
Expand Down
3 changes: 2 additions & 1 deletion src/compat/maestro/daemon-runtime-port.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { AppError, asAppError } from '../../kernel/errors.ts';
import type { Rect } from '../../kernel/snapshot.ts';
import { emitDiagnostic } from '../../utils/diagnostics.ts';
import { emitDiagnostic, registerDiagnosticSensitiveValue } from '../../utils/diagnostics.ts';
import { stripUndefined } from '../../utils/parsing.ts';
import { executeRunScriptFile } from './run-script-execution.ts';
import {
Expand Down Expand Up @@ -96,6 +96,7 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper
text: string,
context: MaestroRuntimeOperationContext,
): Promise<void> => {
registerDiagnosticSensitiveValue(text);
await invokeMutation({ kind: 'typeText', text }, context);
const stable = await waitForTypedSnapshotStability({
timeoutMs: MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS,
Expand Down
16 changes: 2 additions & 14 deletions src/compat/maestro/engine-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ export function createMaestroExecutionContext(
// Flow config and runFlow env values are stack-scoped; script output variables persist.
let persistentValues = stringifyValues(defaults);
const scopes: Record<string, string>[] = [];
const expandedValues = new Map<string, string>();
let cachedValues: Readonly<Record<string, string>> | undefined;
let generation = 0;
let observation: MaestroObservation | undefined;
Expand All @@ -26,9 +25,6 @@ export function createMaestroExecutionContext(
get observation(): MaestroObservation | undefined {
return observation?.generation === generation ? observation : undefined;
},
get expandedVariables(): Readonly<Record<string, string>> {
return Object.fromEntries(expandedValues);
},
enter(scopedValues: Record<string, string | number | boolean> = {}): () => void {
const resolved = resolveScopedValues(scopedValues);
scopes.push(resolved);
Expand Down Expand Up @@ -62,10 +58,10 @@ export function createMaestroExecutionContext(
observation = undefined;
},
resolve(value: string): string {
return resolveValue(value, currentValues(), recordExpandedValue);
return resolveValue(value, currentValues());
},
resolveDeferred(value: string): string {
return resolveValue(value, currentValues(), undefined, new Set(), false);
return resolveValue(value, currentValues(), new Set(), false);
},
};

Expand All @@ -92,17 +88,12 @@ export function createMaestroExecutionContext(
...resolved,
...overrides,
},
undefined,
new Set(),
false,
);
}
return resolved;
}

function recordExpandedValue(name: string, value: string): void {
expandedValues.set(name, value);
}
}

function stringifyValues(
Expand All @@ -114,7 +105,6 @@ function stringifyValues(
function resolveValue(
value: string,
values: Readonly<Record<string, string>>,
onExpanded?: (name: string, value: string) => void,
resolving = new Set<string>(),
failOnUnresolved = true,
): string {
Expand All @@ -130,11 +120,9 @@ function resolveValue(
const resolved = resolveValue(
values[key]!,
values,
onExpanded,
new Set([...resolving, key]),
failOnUnresolved,
);
onExpanded?.(key, resolved);
return resolved;
});
if (failOnUnresolved) assertNoUnsupportedInterpolation(resolved);
Expand Down
1 change: 0 additions & 1 deletion src/compat/maestro/engine-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,6 @@ export type MaestroEngineObserver = {
runtimeMetrics?: MaestroRuntimeMetrics;
error: unknown;
artifactPaths: readonly string[];
expandedVariables: Readonly<Record<string, string>>;
},
): void;
};
Expand Down
1 change: 0 additions & 1 deletion src/compat/maestro/replay-plan-execution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ async function executeObservedStep(
...runtimeMetricsDelta(metricsBefore, state.port.readMetrics?.()),
error: failure.error,
artifactPaths: [...state.artifacts],
expandedVariables: state.context.expandedVariables,
}),
);
throw failure;
Expand Down
29 changes: 20 additions & 9 deletions src/compat/maestro/replay-plan-step-execution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,18 +75,21 @@ async function executeStep(
}

async function executeOptionalCommand(
command: MaestroRuntimeCommand,
rawCommand: MaestroRuntimeCommand,
appId: string | undefined,
state: MaestroReplayPlanExecutionState,
): Promise<MaestroRuntimeResult | undefined> {
const command = resolveCommand(rawCommand, state.context);
try {
return await executeCommand(command, appId, state);
return await executeResolvedCommand(command, appId, state);
} catch (error) {
checkpointMaestroCancellation(state.options.signal);
if (!isOptionalCommand(command) || !isMaestroTestFailure(error)) throw error;
state.warnings.push(formatOptionalWarning(command, error));
state.skipped += 1;
return undefined;
if (isOptionalCommand(command) && isMaestroTestFailure(error)) {
state.warnings.push(formatOptionalWarning(command, error));
state.skipped += 1;
return undefined;
}
throw commandFailure(error, command);
}
}

Expand All @@ -100,12 +103,11 @@ function isOptionalCommand(command: MaestroRuntimeCommand): boolean {
return 'optional' in command && command.optional === true;
}

async function executeCommand(
rawCommand: MaestroRuntimeCommand,
async function executeResolvedCommand(
command: MaestroRuntimeCommand,
appId: string | undefined,
state: MaestroReplayPlanExecutionState,
): Promise<MaestroRuntimeResult | undefined> {
const command = resolveCommand(rawCommand, state.context);
switch (command.kind) {
case 'assertVisible':
await requireObservation(
Expand Down Expand Up @@ -303,6 +305,15 @@ export function asMaestroReplayPlanStepFailure(
};
}

function commandFailure(error: unknown, command: MaestroRuntimeCommand): PlanStepFailure {
return {
kind: 'maestroPlanStepFailure',
error: withSource(error, command),
source: command.source,
command,
};
}

function isPlanStepFailure(value: unknown): value is PlanStepFailure {
return Boolean(
value &&
Expand Down
1 change: 1 addition & 0 deletions src/compat/maestro/support-matrix.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export const MAESTRO_COMPAT_LIMITATIONS = [
'Runtime: iOS and Android only; launchApp.clearState supports Android and iOS simulators, launch arguments are Apple-only, and standalone device utility/state commands are unsupported.',
'Expressions: when.true supports boolean literals and maestro.platform comparisons; repeat.while, evalScript, and broader JavaScript expressions are unsupported.',
'Environment: flow env is the default, AD_VAR_* overrides it, and CLI -e KEY=VALUE wins over both.',
'Failure diagnostics: resolved targets and runFlow paths are rendered, while inputText payloads remain hidden; do not place secrets in diagnostic identifiers.',
'Trust: runScript executes trusted scripts, may make http.post network requests, and is not a security sandbox; output keys cannot contain a dot.',
'Errors and tracking: unsupported commands and fields fail with source context when available; open a focused issue only when implementation work is planned.',
] as const;
Expand Down
70 changes: 70 additions & 0 deletions src/daemon/__tests__/request-router-replay-env.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { expect, test } from 'vitest';
import { makeSessionStore } from '../../__tests__/test-utils/index.ts';
import { LeaseRegistry } from '../lease-registry.ts';
import { createRequestHandler } from '../request-router.ts';

function createHarness() {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-router-replay-env-'));
return {
root,
handler: createRequestHandler({
logPath: path.join(root, 'daemon.log'),
stateDir: root,
token: 'test-token',
sessionStore: makeSessionStore('agent-device-router-replay-env-store-'),
leaseRegistry: new LeaseRegistry(),
trackDownloadableArtifact: () => 'artifact-id',
}),
};
}

test('malformed replay env returns a normalized INVALID_ARGS response', async () => {
const { root, handler } = createHarness();
const flowPath = path.join(root, 'flow.ad');
fs.writeFileSync(flowPath, 'wait 1\n');

await expect(
handler({
token: 'test-token',
session: 'default',
command: 'replay',
positionals: [flowPath],
flags: { replayEnv: ['NOEQUAL'] },
meta: { requestId: 'req-invalid-replay-env' },
}),
).resolves.toMatchObject({
ok: false,
error: {
code: 'INVALID_ARGS',
message: expect.stringContaining('expected KEY=VALUE'),
},
});
});

test('ordinary replay env values do not globally corrupt request diagnostics', async () => {
const { root, handler } = createHarness();
const missingPath = path.join(root, '2-missing.ad');

const response = await handler({
token: 'test-token',
session: 'default',
command: 'replay',
positionals: [missingPath],
flags: { replayEnv: ['RETRIES=2', 'USER=demo'] },
meta: { requestId: 'req-ordinary-replay-env' },
});

expect(response.ok).toBe(false);
if (response.ok) return;
expect(response.error.logPath).toBeTruthy();
const diagnostics = fs.readFileSync(response.error.logPath!, 'utf8');
expect(diagnostics).toContain(missingPath);
expect(diagnostics).not.toContain('[REDACTED]-missing.ad');
for (const line of diagnostics.trim().split('\n')) {
const event = JSON.parse(line) as { ts: string };
expect(event.ts).not.toContain('[REDACTED]');
}
});
Loading
Loading