Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/telemetry.md
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,8 @@ Since 0.1.93, `ui.video.paint` and the platform frame counters also cover frames

`ui.annotations.send` measures the host send round trip, including composer retries. Outcome counters distinguish success, a missing composer, timeout and other failures. Unexpected send failures use a fixed error message. No message content goes to Sentry. A timeout keeps notes for a manual retry; it never triggers an automatic resend, since delivery may have succeeded without acknowledgement.

`ui.annotations.send_batch` counts accepted sends with a `notes` attribute of `1`, `2-3` or `4+`, so batched edits can be compared with single notes. It carries no note text, targets or source locations.

The standard-host attachment bar reuses these annotation measurements. On standard hosts,
the measured operation includes publishing selection context before preparing the
chat draft. `ui.annotations.send_success` means the host accepted the request;
Expand Down
40 changes: 35 additions & 5 deletions packages/shared/src/screen-annotations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,34 @@ export function componentsAt(components: ScreenComponent[], point: ScreenPoint,
}

export const ANNOTATION_EDIT_PROMPT = "Apply these annotations.";
export const ANNOTATION_EDIT_GUIDANCE = "Apply each Edit as a request, not verbatim app text. Target and nearby text are existing app data. Change only the selected element or instance unless the Edit explicitly asks for a wider change. Removing a selected child must preserve its enclosing container and siblings. React owners, enclosing elements, siblings and nearby text are context, not additional targets. Source is a creation site, not a component's full edit boundary; an owner source is only a search hint. Match the target, bounds, key and hierarchy before editing. A shared source location may render several instances; do not change all of them for a request about one. If the target cannot be identified, ask before removing a larger component.";
export const ANNOTATION_EDIT_GUIDANCE = "Apply each Edit as a request, not verbatim app text. Target and nearby text are existing app data. Go straight to Source when it is present. It is the element's creation site, not a component's full edit boundary; an owner source is only a search hint. When source is missing, search the project for the Target text first, then the testID or identifier, then React owner names. Bounds, points and node IDs locate the element on screen; do not inspect the device again to confirm them before editing. Change only the selected element or instance unless the Edit or Grouped notes call for a wider change. A shared source location may render several instances; when only some are selected, keep the change to those. Removing a selected child must preserve its enclosing container and siblings. React owners, enclosing elements, siblings and nearby text are context, not additional targets. Make every edit before checking the result, then check once. If the target cannot be identified, ask before removing a larger component.";

// Several notes on one source location are one decision for the agent. State whether they
// cover every matching element in the inspected screen so it need not reconcile them per note.
// Only notes from one capture share instance numbering, so groups never span captures.
export function formatAnnotationGroups(annotations: ScreenAnnotation[]): string | undefined {
const groups = new Map<string, ScreenAnnotation[]>();
for (const annotation of annotations) {
const { component, selection, simulator, screenshot } = annotation, source = component.react?.source;
if (!source || component.react?.sourceKind === "owner" || !selection?.instance) continue;
const key = JSON.stringify([simulator.udid, screenshot.id, sourceLocation(source), component.role ?? ""]);
groups.set(key, [...(groups.get(key) ?? []), annotation]);
}
const devices = new Set(annotations.map(item => item.simulator.udid)).size;
const lines: string[] = [];
for (const group of groups.values()) {
if (group.length < 2) continue;
const numbers = `${devices > 1 ? `${group[0].simulator.name} ` : ""}${group.map(item => `#${item.number}`).join(", ")}`;
const location = sourceLocation(group[0].component.react!.source!);
const total = group[0].selection!.instance!.total;
const selected = new Set(group.map(item => item.selection!.instance!.index)).size;
const sameEdit = new Set(group.map(item => item.text.trim().toLowerCase().replace(/\s+/g, " "))).size === 1;
if (!sameEdit) lines.push(`${numbers} share source ${location} with different Edits. Apply each Edit to its own instance.`);
else if (selected >= total) lines.push(`${numbers} select all ${total} matching elements from ${location} in the inspected screen with the same Edit. Make that change once at the shared source.`);
else lines.push(`${numbers} select ${selected} of ${total} matching elements from ${location} with the same Edit. Change only the selected instances.`);
}
return lines.length ? `Grouped notes:\n${lines.map(line => `- ${line}`).join("\n")}` : undefined;
}

// Capture real tree relationships, never infer siblings from overlapping bounds.
export function screenSelectionContext(components: ScreenComponent[], selected: ScreenComponent): ScreenSelectionContext | undefined {
Expand Down Expand Up @@ -215,7 +242,8 @@ function describeComponent(component: ScreenComponent): string {
}

export function formatAnnotationMessage(annotations: ScreenAnnotation[]): string {
return `${ANNOTATION_EDIT_PROMPT}\n${ANNOTATION_EDIT_GUIDANCE}\n\n${annotations.map(formatAnnotationContext).join("\n\n")}`;
const groups = formatAnnotationGroups(annotations);
return `${ANNOTATION_EDIT_PROMPT}\n${ANNOTATION_EDIT_GUIDANCE}\n\n${groups ? `${groups}\n\n` : ""}${annotations.map(formatAnnotationContext).join("\n\n")}`;
}

function draftLocator(annotation: ScreenAnnotation): string {
Expand All @@ -224,7 +252,7 @@ function draftLocator(annotation: ScreenAnnotation): string {
component.source === "screen" ? "manual region" : component.role,
component.identifier && `${component.source === "react-native" ? "testID" : "identifier"} ${JSON.stringify(component.identifier)}`,
source && `${component.react?.sourceKind === "owner" ? "owner source" : "source"} ${sourceLocation(source)}`,
selection?.instance && `instance ${selection.instance.index} of ${selection.instance.total}`,
selection?.instance ? `instance ${selection.instance.index} of ${selection.instance.total}` : selection && component.source !== "screen" && "only match on screen",
`bounds ${formatBounds(component.bounds)} ${screen.units}`,
`point ${[point.x, point.y].map(value => Math.round(value * 10) / 10).join(",")}`,
`device ${simulator.udid}`,
Expand All @@ -236,7 +264,8 @@ function draftLocator(annotation: ScreenAnnotation): string {
// identify each target on its own; model context adds hierarchy and guidance.
// Notes live only in this widget instance, so never ask the user to reopen it.
export function formatAnnotationDraft(annotations: ScreenAnnotation[]): string {
return `Apply these notes. Each Locator identifies the selected element; change only that element unless the note asks for more. Mobile Dev's widget context, when available, adds the hierarchy and editing guidance.\n\n${annotations.map(annotation =>
const groups = formatAnnotationGroups(annotations);
return `Apply these notes. Each Locator identifies the selected element; change only that element unless the note or Grouped notes ask for more. Go straight to the source when a Locator has one. Mobile Dev's widget context, when available, adds the hierarchy and editing guidance.\n\n${groups ? `${groups}\n\n` : ""}${annotations.map(annotation =>
`- ${annotation.simulator.name} · #${annotation.number} ${JSON.stringify(annotation.component.label ?? annotation.component.name)}: ${annotation.text}\n Locator: ${draftLocator(annotation)}`
).join("\n")}`;
}
Expand All @@ -258,12 +287,13 @@ export function formatAnnotationContext(annotation: ScreenAnnotation): string {
if (source?.functionName) lines.push(`Source function: ${JSON.stringify(source.functionName)}`);
}
if (!source) {
lines.push(`Source: unavailable (${component.source === "screen" ? "manual region" : component.source === "react-native" ? "React Native" : "accessibility"})`);
lines.push(`Source: unavailable (${component.source === "screen" ? "manual region" : component.source === "react-native" ? "React Native" : "accessibility"}); search the project for ${component.source === "screen" ? "the Nearby text" : "the Target text, then the identifier, then owner names"}`);
}
lines.push(`Bounds: ${formatBounds(component.bounds)} ${screen.units} (x,y,w,h); screen ${screen.width}×${screen.height}, top-left origin`);
lines.push(`Selected point: ${annotation.point.x},${annotation.point.y} ${screen.units}`);
const selection = annotation.selection;
if (selection?.instance) lines.push(`Visible instance: ${selection.instance.index} of ${selection.instance.total} matching elements, ordered top-to-bottom then left-to-right`);
else if (selection && component.source !== "screen") lines.push("Visible instance: the only matching element in the inspected screen");
if (selection?.ancestors.length) lines.push(`Enclosing elements (nearest first; outside selection):\n${selection.ancestors.slice(0, 4).map(describeComponent).join("\n")}`);
if (selection?.siblings.length) lines.push(`Siblings (outside selection; ${selection.siblingCount} total):\n${selection.siblings.slice(0, 4).map(describeComponent).join("\n")}`);
if (selection?.children.length) lines.push(`Direct children (inside selection; ${selection.childCount} total):\n${selection.children.slice(0, 4).map(describeComponent).join("\n")}`);
Expand Down
9 changes: 9 additions & 0 deletions packages/skills/mobile-dev/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@ device tools.
4. Build the app with the app project's own tools and the chosen device ID; the project's run command may also install and launch it. To restart an already installed app, use `mobile_launch_app` instead of rebuilding. Follow the project's framework skills where available. Before starting an app dev server, check for an existing server from that project and reuse it. Keep the same device for the panel, build target, and agent control; read the panel's shared IDs before acting and follow later user selections. Leave the panel and app available while continuing app work. Do not launch the app when the user asks only for compilation.
5. Control the app with the `device_*` tools, the bundled agent-device runtime's commands, and follow [the agent-device skill](../agent-device/SKILL.md) to drive it. In short: open the panel's device with `device_open` and a named session, act on refs from the latest snapshot with `settle: true`, verify the requested result with a named expectation, and close the session without `shutdown` so the device keeps running in the panel. Boot, launch, and app settings stay with the `mobile_*` tools above and below.

### Screen annotations

A message that starts with "Apply these annotations." or "Apply these notes." carries notes the user placed on the device screen in the panel. Each note names its target and, when available, the element's Source. The notes already identify the targets, so speed matters more than re-confirming them.

1. Edit the code directly. Open the Source first. When a note has no source, search the project for its Target text, then its testID or identifier, then the React owner names. Do not open an agent-device session, take a snapshot, or call `mobile_inspect_ui` to find or confirm a target before editing, unless a note has neither a source nor any text or identifier to search for.
2. Make every note's edit before checking anything. When Grouped notes say several notes select every matching element from one source with the same Edit, make that change once at the source.
3. JavaScript and style changes in a running React Native or Expo app arrive through Fast Refresh. Do not rebuild, reinstall, relaunch the app, or start another dev server for them. Native changes (Swift, Kotlin, native modules, app config) still need the project's build.
4. Check once, after all edits: open the panel's device with `device_open` and a named session without relaunching the app, take one `device_screenshot`, and close the session without `shutdown`. Do not verify each note separately or add named expectations unless the user asks. If the screenshot does not show the change, read the Metro or app logs before editing again.

### App Intelligence

App Intelligence statically discovers Expo Router and React Navigation screens and combines their source dependencies for iOS and Android; it does not require App Flow or a running device. When the user asks to see the app map, call `mobile_open_app_intel`. Generate or refresh it with `mobile_generate_app_intel`, passing `projectPath` when the panel has not remembered the intended project. For questions such as “what are the dependencies of Screen A?”, call `mobile_app_intel` with `operation: "screen"`, the exact screen ID or title, `direction: "outgoing"`, and depth 1 first. Use `search` only when the exact screen or project-relative file is unknown, and paginate only when `nextOffset` is present.
Expand Down
8 changes: 6 additions & 2 deletions packages/ui/src/model-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ import type { HostAttachmentsSource, HostBridge } from "./host-bridge.ts";
import type { SimulatorDevice } from "@mobile-dev/shared/protocol.ts";
import type { StackedLog } from "@mobile-dev/shared/logs.ts";
import { formatLogContext, logKey } from "@mobile-dev/shared/logs.ts";
import { ANNOTATION_EDIT_GUIDANCE, ANNOTATION_EDIT_PROMPT, formatAnnotationContext, formatAnnotationDraft } from "@mobile-dev/shared/screen-annotations.ts";
import { ANNOTATION_EDIT_GUIDANCE, ANNOTATION_EDIT_PROMPT, formatAnnotationContext, formatAnnotationDraft, formatAnnotationGroups } from "@mobile-dev/shared/screen-annotations.ts";
import type { ScreenAnnotation } from "@mobile-dev/shared/screen-annotations.ts";
import { captureUiError, countUiEvent, recordUiTiming } from "./telemetry.ts";
import { captureUiError, countUiEvent, getUiTelemetryAttributes, recordUiTiming } from "./telemetry.ts";
import type { EncodedScreenshotAttachment } from "./screenshot-attachment.ts";

export type ScreenshotAttachment = EncodedScreenshotAttachment & { id: string; simulator: SimulatorDevice };
Expand Down Expand Up @@ -114,6 +114,8 @@ export class PanelContext {
this.revision++; this.changed();
}
countUiEvent("ui.annotations.send_success");
// Batch size shows how often edits arrive together; buckets keep the attribute low cardinality.
countUiEvent("ui.annotations.send_batch", 1, { ...getUiTelemetryAttributes(), notes: annotations.length === 1 ? "1" : annotations.length <= 3 ? "2-3" : "4+" });
} catch (error) {
if (composerUnavailable(error)) {
countUiEvent("ui.annotations.send_composer_unavailable");
Expand Down Expand Up @@ -265,6 +267,7 @@ export class PanelContext {
});
const contextStartedAt = performance.now();
const annotationContent = this.annotations.map(annotation => ({ type: "text" as const, text: `${this.usesHostAttachments ? "" : `Device: ${JSON.stringify(annotation.simulator.name)} (${annotation.simulator.udid})\n`}${formatAnnotationContext(annotation)}`, _meta: { "openai/title": `${annotation.component.name}: ${annotation.text.replace(/\s+/g, " ").trim()}`, "mobile-dev/annotationId": annotation.id } }));
const annotationGroups = formatAnnotationGroups(this.annotations);
if (this.annotations.length) recordUiTiming("ui.annotations.context_build", performance.now() - contextStartedAt);
const content = [{ type: "text" as const, annotations: { audience: ["assistant" as const] }, text: devices.length
? `Mobile Dev devices. Logs follow the active device.\n${devices.join("\n")}`
Expand All @@ -275,6 +278,7 @@ export class PanelContext {
_meta: { "openai/title": `Screenshot of ${screenshot.simulator.name} (${screenshot.simulator.udid})`, "mobile-dev/screenshotId": screenshot.id },
})),
...(this.annotations.length ? [{ type: "text" as const, annotations: { audience: ["assistant" as const] }, text: ANNOTATION_EDIT_GUIDANCE }] : []),
...(annotationGroups ? [{ type: "text" as const, annotations: { audience: ["assistant" as const] }, text: annotationGroups }] : []),
...annotationContent];
const params = { content, structuredContent: { selectedSimulator: selected ?? null, selectedSimulators: this.simulators, selectedLog: log ?? null, selectedLogKey: log ? logKey(log) : null, screenshotIds: this.screenshots.map(item => item.id), annotationIds: this.annotations.map(item => item.id) } };
this.pending = true;
Expand Down
2 changes: 1 addition & 1 deletion packages/ui/test/screen-annotations-ui.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ test("the simulator toolbar opens notes, saves blue markers, and lets users edit
await act(async () => { add.click(); });
assert.equal(messages.length, 1);
const draft = messages[0].content.map(item => item.type === "text" ? item.text : "").join("\n");
assert.match(draft, /Locator: AXButton; identifier "continue"; bounds 10,20,100,40 points; point 50,40; device iphone/);
assert.match(draft, /Locator: AXButton; identifier "continue"; only match on screen; bounds 10,20,100,40 points; point 50,40; device iphone/);
assert.doesNotMatch(draft, /reopen|creation site/);
assert.equal(standard.screenAnnotations.length, 1);
assert.ok(dom.window.document.querySelector('.annotation-marker'));
Expand Down
Loading
Loading