From dd3554e9d120661cbc2d8d95b18ff4bc0c5ca50e Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Wed, 18 Nov 2026 17:10:37 +0100 Subject: [PATCH 1/2] perf(annotations): apply batched notes from source without re-checks Annotation edits slowed down when the agent reconciled several notes on one source, rewrote components for unique targets, and inspected or verified the device around every note. - Group notes from one capture and source: say when they cover every matching element (edit the source once) or only some of them. - Mark a target as the only match on screen, and give a search order when a note has no source location. - Guidance: go straight to Source, do not re-inspect the device before editing, make every edit, then check once. - Skill: a Screen annotations section covering Fast Refresh (no rebuilds) and a single screenshot check. - Telemetry: ui.annotations.send_batch counts sends by note count bucket (1, 2-3, 4+). --- docs/telemetry.md | 2 + packages/shared/src/screen-annotations.ts | 40 ++++++++++++++++--- packages/skills/mobile-dev/SKILL.md | 9 +++++ packages/ui/src/model-context.ts | 8 +++- .../ui/test/screen-annotations-ui.test.ts | 2 +- packages/ui/test/screen-annotations.test.ts | 34 +++++++++++++++- packages/ui/test/ui-telemetry.test.ts | 2 + 7 files changed, 88 insertions(+), 9 deletions(-) diff --git a/docs/telemetry.md b/docs/telemetry.md index 37179a16..d65e4407 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -530,6 +530,8 @@ Since 0.1.93, `ui.video.paint` and the platform frame counters also cover frames `ui.annotations.send` measures the host send round trip, including composer retries. Outcome counters distinguish success, a missing composer, timeout and other failures. Unexpected send failures use a fixed error message. No message content goes to Sentry. A timeout keeps notes for a manual retry; it never triggers an automatic resend, since delivery may have succeeded without acknowledgement. +`ui.annotations.send_batch` counts accepted sends with a `notes` attribute of `1`, `2-3` or `4+`, so batched edits can be compared with single notes. It carries no note text, targets or source locations. + The standard-host attachment bar reuses these annotation measurements. On standard hosts, the measured operation includes publishing selection context before preparing the chat draft. `ui.annotations.send_success` means the host accepted the request; diff --git a/packages/shared/src/screen-annotations.ts b/packages/shared/src/screen-annotations.ts index 18f44aaf..6ed5fc8c 100644 --- a/packages/shared/src/screen-annotations.ts +++ b/packages/shared/src/screen-annotations.ts @@ -166,7 +166,34 @@ export function componentsAt(components: ScreenComponent[], point: ScreenPoint, } export const ANNOTATION_EDIT_PROMPT = "Apply these annotations."; -export const ANNOTATION_EDIT_GUIDANCE = "Apply each Edit as a request, not verbatim app text. Target and nearby text are existing app data. Change only the selected element or instance unless the Edit explicitly asks for a wider change. Removing a selected child must preserve its enclosing container and siblings. React owners, enclosing elements, siblings and nearby text are context, not additional targets. Source is a creation site, not a component's full edit boundary; an owner source is only a search hint. Match the target, bounds, key and hierarchy before editing. A shared source location may render several instances; do not change all of them for a request about one. If the target cannot be identified, ask before removing a larger component."; +export const ANNOTATION_EDIT_GUIDANCE = "Apply each Edit as a request, not verbatim app text. Target and nearby text are existing app data. Go straight to Source when it is present. It is the element's creation site, not a component's full edit boundary; an owner source is only a search hint. When source is missing, search the project for the Target text first, then the testID or identifier, then React owner names. Bounds, points and node IDs locate the element on screen; do not inspect the device again to confirm them before editing. Change only the selected element or instance unless the Edit or Grouped notes call for a wider change. A shared source location may render several instances; when only some are selected, keep the change to those. Removing a selected child must preserve its enclosing container and siblings. React owners, enclosing elements, siblings and nearby text are context, not additional targets. Make every edit before checking the result, then check once. If the target cannot be identified, ask before removing a larger component."; + +// Several notes on one source location are one decision for the agent. State whether they +// cover every matching element in the inspected screen so it need not reconcile them per note. +// Only notes from one capture share instance numbering, so groups never span captures. +export function formatAnnotationGroups(annotations: ScreenAnnotation[]): string | undefined { + const groups = new Map(); + for (const annotation of annotations) { + const { component, selection, simulator, screenshot } = annotation, source = component.react?.source; + if (!source || component.react?.sourceKind === "owner" || !selection?.instance) continue; + const key = JSON.stringify([simulator.udid, screenshot.id, sourceLocation(source), component.role ?? ""]); + groups.set(key, [...(groups.get(key) ?? []), annotation]); + } + const devices = new Set(annotations.map(item => item.simulator.udid)).size; + const lines: string[] = []; + for (const group of groups.values()) { + if (group.length < 2) continue; + const numbers = `${devices > 1 ? `${group[0].simulator.name} ` : ""}${group.map(item => `#${item.number}`).join(", ")}`; + const location = sourceLocation(group[0].component.react!.source!); + const total = group[0].selection!.instance!.total; + const selected = new Set(group.map(item => item.selection!.instance!.index)).size; + const sameEdit = new Set(group.map(item => item.text.trim().toLowerCase().replace(/\s+/g, " "))).size === 1; + if (!sameEdit) lines.push(`${numbers} share source ${location} with different Edits. Apply each Edit to its own instance.`); + else if (selected >= total) lines.push(`${numbers} select all ${total} matching elements from ${location} in the inspected screen with the same Edit. Make that change once at the shared source.`); + else lines.push(`${numbers} select ${selected} of ${total} matching elements from ${location} with the same Edit. Change only the selected instances.`); + } + return lines.length ? `Grouped notes:\n${lines.map(line => `- ${line}`).join("\n")}` : undefined; +} // Capture real tree relationships, never infer siblings from overlapping bounds. export function screenSelectionContext(components: ScreenComponent[], selected: ScreenComponent): ScreenSelectionContext | undefined { @@ -215,7 +242,8 @@ function describeComponent(component: ScreenComponent): string { } export function formatAnnotationMessage(annotations: ScreenAnnotation[]): string { - return `${ANNOTATION_EDIT_PROMPT}\n${ANNOTATION_EDIT_GUIDANCE}\n\n${annotations.map(formatAnnotationContext).join("\n\n")}`; + const groups = formatAnnotationGroups(annotations); + return `${ANNOTATION_EDIT_PROMPT}\n${ANNOTATION_EDIT_GUIDANCE}\n\n${groups ? `${groups}\n\n` : ""}${annotations.map(formatAnnotationContext).join("\n\n")}`; } function draftLocator(annotation: ScreenAnnotation): string { @@ -224,7 +252,7 @@ function draftLocator(annotation: ScreenAnnotation): string { component.source === "screen" ? "manual region" : component.role, component.identifier && `${component.source === "react-native" ? "testID" : "identifier"} ${JSON.stringify(component.identifier)}`, source && `${component.react?.sourceKind === "owner" ? "owner source" : "source"} ${sourceLocation(source)}`, - selection?.instance && `instance ${selection.instance.index} of ${selection.instance.total}`, + selection?.instance ? `instance ${selection.instance.index} of ${selection.instance.total}` : selection && component.source !== "screen" && "only match on screen", `bounds ${formatBounds(component.bounds)} ${screen.units}`, `point ${[point.x, point.y].map(value => Math.round(value * 10) / 10).join(",")}`, `device ${simulator.udid}`, @@ -236,7 +264,8 @@ function draftLocator(annotation: ScreenAnnotation): string { // identify each target on its own; model context adds hierarchy and guidance. // Notes live only in this widget instance, so never ask the user to reopen it. export function formatAnnotationDraft(annotations: ScreenAnnotation[]): string { - return `Apply these notes. Each Locator identifies the selected element; change only that element unless the note asks for more. Mobile Dev's widget context, when available, adds the hierarchy and editing guidance.\n\n${annotations.map(annotation => + const groups = formatAnnotationGroups(annotations); + return `Apply these notes. Each Locator identifies the selected element; change only that element unless the note or Grouped notes ask for more. Go straight to the source when a Locator has one. Mobile Dev's widget context, when available, adds the hierarchy and editing guidance.\n\n${groups ? `${groups}\n\n` : ""}${annotations.map(annotation => `- ${annotation.simulator.name} · #${annotation.number} ${JSON.stringify(annotation.component.label ?? annotation.component.name)}: ${annotation.text}\n Locator: ${draftLocator(annotation)}` ).join("\n")}`; } @@ -258,12 +287,13 @@ export function formatAnnotationContext(annotation: ScreenAnnotation): string { if (source?.functionName) lines.push(`Source function: ${JSON.stringify(source.functionName)}`); } if (!source) { - lines.push(`Source: unavailable (${component.source === "screen" ? "manual region" : component.source === "react-native" ? "React Native" : "accessibility"})`); + lines.push(`Source: unavailable (${component.source === "screen" ? "manual region" : component.source === "react-native" ? "React Native" : "accessibility"}); search the project for ${component.source === "screen" ? "the Nearby text" : "the Target text, then the identifier, then owner names"}`); } lines.push(`Bounds: ${formatBounds(component.bounds)} ${screen.units} (x,y,w,h); screen ${screen.width}×${screen.height}, top-left origin`); lines.push(`Selected point: ${annotation.point.x},${annotation.point.y} ${screen.units}`); const selection = annotation.selection; if (selection?.instance) lines.push(`Visible instance: ${selection.instance.index} of ${selection.instance.total} matching elements, ordered top-to-bottom then left-to-right`); + else if (selection && component.source !== "screen") lines.push("Visible instance: the only matching element in the inspected screen"); if (selection?.ancestors.length) lines.push(`Enclosing elements (nearest first; outside selection):\n${selection.ancestors.slice(0, 4).map(describeComponent).join("\n")}`); if (selection?.siblings.length) lines.push(`Siblings (outside selection; ${selection.siblingCount} total):\n${selection.siblings.slice(0, 4).map(describeComponent).join("\n")}`); if (selection?.children.length) lines.push(`Direct children (inside selection; ${selection.childCount} total):\n${selection.children.slice(0, 4).map(describeComponent).join("\n")}`); diff --git a/packages/skills/mobile-dev/SKILL.md b/packages/skills/mobile-dev/SKILL.md index 4b7f70b1..6f088794 100644 --- a/packages/skills/mobile-dev/SKILL.md +++ b/packages/skills/mobile-dev/SKILL.md @@ -36,6 +36,15 @@ device tools. 4. Build the app with the app project's own tools and the chosen device ID; the project's run command may also install and launch it. To restart an already installed app, use `mobile_launch_app` instead of rebuilding. Follow the project's framework skills where available. Before starting an app dev server, check for an existing server from that project and reuse it. Keep the same device for the panel, build target, and agent control; read the panel's shared IDs before acting and follow later user selections. Leave the panel and app available while continuing app work. Do not launch the app when the user asks only for compilation. 5. Control the app with the `device_*` tools, the bundled agent-device runtime's commands, and follow [the agent-device skill](../agent-device/SKILL.md) to drive it. In short: open the panel's device with `device_open` and a named session, act on refs from the latest snapshot with `settle: true`, verify the requested result with a named expectation, and close the session without `shutdown` so the device keeps running in the panel. Boot, launch, and app settings stay with the `mobile_*` tools above and below. +### Screen annotations + +A message that starts with "Apply these annotations." or "Apply these notes." carries notes the user placed on the device screen in the panel. Each note names its target and, when available, the element's Source. The notes already identify the targets, so speed matters more than re-confirming them. + +1. Edit the code directly. Open the Source first. When a note has no source, search the project for its Target text, then its testID or identifier, then the React owner names. Do not open an agent-device session, take a snapshot, or call `mobile_inspect_ui` to find or confirm a target before editing, unless a note has neither a source nor any text or identifier to search for. +2. Make every note's edit before checking anything. When Grouped notes say several notes select every matching element from one source with the same Edit, make that change once at the source. +3. JavaScript and style changes in a running React Native or Expo app arrive through Fast Refresh. Do not rebuild, reinstall, relaunch the app, or start another dev server for them. Native changes (Swift, Kotlin, native modules, app config) still need the project's build. +4. Check once, after all edits: open the panel's device with `device_open` and a named session without relaunching the app, take one `device_screenshot`, and close the session without `shutdown`. Do not verify each note separately or add named expectations unless the user asks. If the screenshot does not show the change, read the Metro or app logs before editing again. + ### App Intelligence App Intelligence statically discovers Expo Router and React Navigation screens and combines their source dependencies for iOS and Android; it does not require App Flow or a running device. When the user asks to see the app map, call `mobile_open_app_intel`. Generate or refresh it with `mobile_generate_app_intel`, passing `projectPath` when the panel has not remembered the intended project. For questions such as “what are the dependencies of Screen A?”, call `mobile_app_intel` with `operation: "screen"`, the exact screen ID or title, `direction: "outgoing"`, and depth 1 first. Use `search` only when the exact screen or project-relative file is unknown, and paginate only when `nextOffset` is present. diff --git a/packages/ui/src/model-context.ts b/packages/ui/src/model-context.ts index 3f3983d5..05ea99fa 100644 --- a/packages/ui/src/model-context.ts +++ b/packages/ui/src/model-context.ts @@ -3,9 +3,9 @@ import type { HostAttachmentsSource, HostBridge } from "./host-bridge.ts"; import type { SimulatorDevice } from "@mobile-dev/shared/protocol.ts"; import type { StackedLog } from "@mobile-dev/shared/logs.ts"; import { formatLogContext, logKey } from "@mobile-dev/shared/logs.ts"; -import { ANNOTATION_EDIT_GUIDANCE, ANNOTATION_EDIT_PROMPT, formatAnnotationContext, formatAnnotationDraft } from "@mobile-dev/shared/screen-annotations.ts"; +import { ANNOTATION_EDIT_GUIDANCE, ANNOTATION_EDIT_PROMPT, formatAnnotationContext, formatAnnotationDraft, formatAnnotationGroups } from "@mobile-dev/shared/screen-annotations.ts"; import type { ScreenAnnotation } from "@mobile-dev/shared/screen-annotations.ts"; -import { captureUiError, countUiEvent, recordUiTiming } from "./telemetry.ts"; +import { captureUiError, countUiEvent, getUiTelemetryAttributes, recordUiTiming } from "./telemetry.ts"; import type { EncodedScreenshotAttachment } from "./screenshot-attachment.ts"; export type ScreenshotAttachment = EncodedScreenshotAttachment & { id: string; simulator: SimulatorDevice }; @@ -114,6 +114,8 @@ export class PanelContext { this.revision++; this.changed(); } countUiEvent("ui.annotations.send_success"); + // Batch size shows how often edits arrive together; buckets keep the attribute low cardinality. + countUiEvent("ui.annotations.send_batch", 1, { ...getUiTelemetryAttributes(), notes: annotations.length === 1 ? "1" : annotations.length <= 3 ? "2-3" : "4+" }); } catch (error) { if (composerUnavailable(error)) { countUiEvent("ui.annotations.send_composer_unavailable"); @@ -265,6 +267,7 @@ export class PanelContext { }); const contextStartedAt = performance.now(); const annotationContent = this.annotations.map(annotation => ({ type: "text" as const, text: `${this.usesHostAttachments ? "" : `Device: ${JSON.stringify(annotation.simulator.name)} (${annotation.simulator.udid})\n`}${formatAnnotationContext(annotation)}`, _meta: { "openai/title": `${annotation.component.name}: ${annotation.text.replace(/\s+/g, " ").trim()}`, "mobile-dev/annotationId": annotation.id } })); + const annotationGroups = formatAnnotationGroups(this.annotations); if (this.annotations.length) recordUiTiming("ui.annotations.context_build", performance.now() - contextStartedAt); const content = [{ type: "text" as const, annotations: { audience: ["assistant" as const] }, text: devices.length ? `Mobile Dev devices. Logs follow the active device.\n${devices.join("\n")}` @@ -275,6 +278,7 @@ export class PanelContext { _meta: { "openai/title": `Screenshot of ${screenshot.simulator.name} (${screenshot.simulator.udid})`, "mobile-dev/screenshotId": screenshot.id }, })), ...(this.annotations.length ? [{ type: "text" as const, annotations: { audience: ["assistant" as const] }, text: ANNOTATION_EDIT_GUIDANCE }] : []), + ...(annotationGroups ? [{ type: "text" as const, annotations: { audience: ["assistant" as const] }, text: annotationGroups }] : []), ...annotationContent]; const params = { content, structuredContent: { selectedSimulator: selected ?? null, selectedSimulators: this.simulators, selectedLog: log ?? null, selectedLogKey: log ? logKey(log) : null, screenshotIds: this.screenshots.map(item => item.id), annotationIds: this.annotations.map(item => item.id) } }; this.pending = true; diff --git a/packages/ui/test/screen-annotations-ui.test.ts b/packages/ui/test/screen-annotations-ui.test.ts index d48d4f00..53de57ca 100644 --- a/packages/ui/test/screen-annotations-ui.test.ts +++ b/packages/ui/test/screen-annotations-ui.test.ts @@ -126,7 +126,7 @@ test("the simulator toolbar opens notes, saves blue markers, and lets users edit await act(async () => { add.click(); }); assert.equal(messages.length, 1); const draft = messages[0].content.map(item => item.type === "text" ? item.text : "").join("\n"); - assert.match(draft, /Locator: AXButton; identifier "continue"; bounds 10,20,100,40 points; point 50,40; device iphone/); + assert.match(draft, /Locator: AXButton; identifier "continue"; only match on screen; bounds 10,20,100,40 points; point 50,40; device iphone/); assert.doesNotMatch(draft, /reopen|creation site/); assert.equal(standard.screenAnnotations.length, 1); assert.ok(dom.window.document.querySelector('.annotation-marker')); diff --git a/packages/ui/test/screen-annotations.test.ts b/packages/ui/test/screen-annotations.test.ts index 061c9313..af6c39b6 100644 --- a/packages/ui/test/screen-annotations.test.ts +++ b/packages/ui/test/screen-annotations.test.ts @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { accessibilityScreen, componentAt, componentsAt, screenComponents, screenSize, formatAnnotationContext, screenSelectionContext, ANNOTATION_EDIT_GUIDANCE } from "../../shared/src/screen-annotations.ts"; +import { accessibilityScreen, componentAt, componentsAt, screenComponents, screenSize, formatAnnotationContext, formatAnnotationDraft, formatAnnotationGroups, formatAnnotationMessage, screenSelectionContext, ANNOTATION_EDIT_GUIDANCE } from "../../shared/src/screen-annotations.ts"; import { ScreenAnnotationsStore } from "../src/screen-annotations.ts"; import type { InspectedTree } from "../src/screen-annotations.ts"; import type { ScreenAnnotation } from "../../shared/src/screen-annotations.ts"; @@ -259,6 +259,38 @@ test("a child deletion identifies one repeated instance and keeps enclosing rows f.store.dispose(); }); +test("notes on one source location say whether they cover every matching instance", () => { + const source = { file: "/project/src/Card.tsx", line: 9, column: 7 }; + const items = screenComponents([0, 1, 2].map(index => ({ ...component, name: `Title ${index}`, label: `Title ${index}`, role: "Text", nodeId: `title-${index}`, parentId: `card-${index}`, source: "react-native", + bounds: { x: 10, y: 20 + index * 80, width: 100, height: 20 }, react: { component: "Text", owners: ["Card"], sourceKind: "element", source } }))); + const notes = items.map((item, index) => ({ ...annotation, id: `note-${index + 1}`, number: index + 1, text: "Make this red", component: item, selection: screenSelectionContext(items, item) })); + assert.match(formatAnnotationGroups(notes)!, /#1, #2, #3 select all 3 matching elements from \/project\/src\/Card\.tsx:9:7 .*Make that change once at the shared source/); + assert.match(formatAnnotationGroups(notes.slice(0, 2))!, /#1, #2 select 2 of 3 matching elements .*Change only the selected instances/); + assert.match(formatAnnotationGroups([notes[0], { ...notes[1], text: "Make this bold" }])!, /share source .* with different Edits/); + assert.equal(formatAnnotationGroups(notes.slice(0, 1)), undefined); + // Instance numbers from another capture describe another screen, so they never complete a group. + const recaptured = notes.map((note, index) => index ? note : { ...note, screenshot: { ...note.screenshot, id: "screen-2" } }); + assert.match(formatAnnotationGroups(recaptured)!, /^Grouped notes:\n- #2, #3 select 2 of 3 matching elements/); + const otherDevice = { ...simulator, udid: "OTHER", name: "Pixel" }; + assert.match(formatAnnotationGroups([...notes, { ...notes[0], id: "note-4", number: 1, simulator: otherDevice }])!, /- iPhone #1, #2, #3 select all 3/); + assert.equal(formatAnnotationGroups(notes.map(note => ({ ...note, component: { ...note.component, react: { ...note.component.react!, sourceKind: "owner" as const } } }))), undefined); + assert.match(formatAnnotationMessage(notes), /Grouped notes:\n- #1, #2, #3/); + assert.match(formatAnnotationDraft(notes), /Grouped notes:\n- #1, #2, #3/); + assert.match(ANNOTATION_EDIT_GUIDANCE, /when only some are selected, keep the change to those/); +}); + +test("notes mark a unique target and give a search order without source", () => { + const items = screenComponents([{ ...component, nodeId: "only" }, { ...component, name: "Other", label: "Other", identifier: "other", nodeId: "other" }]); + const unique = { ...annotation, component: items[0], selection: screenSelectionContext(items, items[0]) }; + assert.match(formatAnnotationContext(unique), /Visible instance: the only matching element in the inspected screen/); + assert.match(formatAnnotationDraft([unique]), /only match on screen/); + assert.match(formatAnnotationContext(unique), /Source: unavailable \(accessibility\); search the project for the Target text, then the identifier, then owner names/); + const manual = { ...annotation, component: { ...component, source: "screen" as const, role: "manual-region" } }; + assert.match(formatAnnotationContext(manual), /search the project for the Nearby text/); + assert.doesNotMatch(formatAnnotationContext(manual), /Visible instance/); + assert.match(ANNOTATION_EDIT_GUIDANCE, /search the project for the Target text first/); +}); + test("flat accessibility records and manual regions never invent a parent or siblings", () => { const items = screenComponents([{ ...component, nodeId: "selected" }, { ...component, name: "Other", nodeId: "other" }]); const selected = screenSelectionContext(items, items[0]); diff --git a/packages/ui/test/ui-telemetry.test.ts b/packages/ui/test/ui-telemetry.test.ts index 69ba8a6d..b142724f 100644 --- a/packages/ui/test/ui-telemetry.test.ts +++ b/packages/ui/test/ui-telemetry.test.ts @@ -475,6 +475,8 @@ test("browser telemetry labels surface measurements, propagates traces, and flus contains(encoded, "ui.annotations.context_build.mean"); contains(encoded, "ui.annotations.send.mean"); contains(encoded, "ui.annotations.send_success"); + contains(encoded, "ui.annotations.send_batch"); + contains(encoded, '"notes":{"value":"1"'); contains(encoded, "ui.annotations.send_timeout"); contains(encoded, "ui.annotations.send_composer_unavailable"); contains(encoded, "ui.annotations.send_failure"); From 063e26a8b5da604aa63475fcfe6f66c9325c731e Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Wed, 18 Nov 2026 17:10:38 +0100 Subject: [PATCH 2/2] test(annotations): add a Codex benchmark for annotation edits tooling/annotation-eval runs Codex on a fixture app with the annotation formatter and skill at a git ref against the working tree, grades each diff, and summarizes time, tool calls and correctness. Device mode drives an Expo app on a simulator with the plugin's MCP server. --- tooling/annotation-eval/README.md | 43 +++++ tooling/annotation-eval/fixture/App.tsx | 10 ++ tooling/annotation-eval/fixture/package.json | 11 ++ .../fixture/src/components/Card.tsx | 19 +++ .../fixture/src/components/CardList.tsx | 23 +++ .../fixture/src/components/Footer.tsx | 16 ++ .../fixture/src/components/Header.tsx | 19 +++ .../fixture/src/components/PrimaryButton.tsx | 17 ++ .../fixture/src/screens/HomeScreen.tsx | 21 +++ .../fixture/src/screens/ProfileScreen.tsx | 17 ++ tooling/annotation-eval/fixture/src/theme.ts | 10 ++ tooling/annotation-eval/grade.ts | 55 ++++++ tooling/annotation-eval/run.ts | 161 ++++++++++++++++++ tooling/annotation-eval/scenarios.ts | 107 ++++++++++++ tooling/annotation-eval/summarize.ts | 38 +++++ 15 files changed, 567 insertions(+) create mode 100644 tooling/annotation-eval/README.md create mode 100644 tooling/annotation-eval/fixture/App.tsx create mode 100644 tooling/annotation-eval/fixture/package.json create mode 100644 tooling/annotation-eval/fixture/src/components/Card.tsx create mode 100644 tooling/annotation-eval/fixture/src/components/CardList.tsx create mode 100644 tooling/annotation-eval/fixture/src/components/Footer.tsx create mode 100644 tooling/annotation-eval/fixture/src/components/Header.tsx create mode 100644 tooling/annotation-eval/fixture/src/components/PrimaryButton.tsx create mode 100644 tooling/annotation-eval/fixture/src/screens/HomeScreen.tsx create mode 100644 tooling/annotation-eval/fixture/src/screens/ProfileScreen.tsx create mode 100644 tooling/annotation-eval/fixture/src/theme.ts create mode 100644 tooling/annotation-eval/grade.ts create mode 100644 tooling/annotation-eval/run.ts create mode 100644 tooling/annotation-eval/scenarios.ts create mode 100644 tooling/annotation-eval/summarize.ts diff --git a/tooling/annotation-eval/README.md b/tooling/annotation-eval/README.md new file mode 100644 index 00000000..6c1a583d --- /dev/null +++ b/tooling/annotation-eval/README.md @@ -0,0 +1,43 @@ +# Annotation edit benchmark + +Measures how fast and how correctly Codex applies screen annotations, comparing the annotation +formatter and Mobile Dev skill at a git ref ("before") with the working tree ("after"). Each run +gets the same message the panel would send, built by the real formatter from a fixture screen. + +```bash +node tooling/annotation-eval/run.ts --before-ref main --runs 3 +node tooling/annotation-eval/summarize.ts "$TMPDIR/mobile-dev-annotation-eval/results.json" +``` + +`--codex-home` defaults to `~/.codex`. Only its auth and top-level settings (model, reasoning +effort, provider) are copied into one CODEX_HOME per version, so your own MCP servers and plugins +do not take part. Skills under `~/.agents/skills` still load and can add time to both versions. + +## Scenarios + +| | Notes | Expected edit | +|---|---|---| +| A | 1 note on a unique button label | that label only | +| B | 3 notes on every card title, same edit | one change at the shared source | +| C | 3 notes on three components | three edits | +| D | 1 note on card 2 of 3 | that card only | +| E | 1 "remove this" on a header badge | badge gone, header kept | +| F | A without a source location | as A | +| G | C without source locations | as C | + +`grade.ts` checks each run's diff. It accepts style edits and call-site props alike and reports +which one the agent chose. + +## Device mode + +`--device --app --plugin ` adds the plugin's MCP server and each +version's skills, and runs one at a time against an app on the simulator. Prepare the app once: + +1. Create an Expo app (`npx create-expo-app@latest --template blank-typescript`), copy + `fixture/App.tsx` and `fixture/src` into it, run `npx expo lint` once so lint is set up, and + commit. The runner refuses a checkout with uncommitted changes, since it restores the app + between runs. +2. Use a simulator without a signed-in Apple Account (for example a new one from + `xcrun simctl create`), install Expo Go and open the app. +3. Start Metro in watch mode. `CI=1` disables Fast Refresh, and `expo start` exits when stdin + closes, so run `tail -f /dev/null | npx expo start`. diff --git a/tooling/annotation-eval/fixture/App.tsx b/tooling/annotation-eval/fixture/App.tsx new file mode 100644 index 00000000..4fdbe95e --- /dev/null +++ b/tooling/annotation-eval/fixture/App.tsx @@ -0,0 +1,10 @@ +import { SafeAreaView } from "react-native"; +import { HomeScreen } from "./src/screens/HomeScreen"; + +export default function App() { + return ( + + + + ); +} diff --git a/tooling/annotation-eval/fixture/package.json b/tooling/annotation-eval/fixture/package.json new file mode 100644 index 00000000..09cec0c5 --- /dev/null +++ b/tooling/annotation-eval/fixture/package.json @@ -0,0 +1,11 @@ +{ + "name": "discover-app", + "version": "1.0.0", + "main": "App.tsx", + "private": true, + "dependencies": { + "expo": "~54.0.0", + "react": "19.1.0", + "react-native": "0.81.4" + } +} diff --git a/tooling/annotation-eval/fixture/src/components/Card.tsx b/tooling/annotation-eval/fixture/src/components/Card.tsx new file mode 100644 index 00000000..8a1dfeac --- /dev/null +++ b/tooling/annotation-eval/fixture/src/components/Card.tsx @@ -0,0 +1,19 @@ +import { StyleSheet, Text, View } from "react-native"; +import { colors, spacing } from "../theme"; + +type Props = { title: string; subtitle: string }; + +export function Card({ title, subtitle }: Props) { + return ( + + {title} + {subtitle} + + ); +} + +const styles = StyleSheet.create({ + card: { padding: spacing.md, borderRadius: 12, borderWidth: 1, borderColor: colors.border, backgroundColor: colors.surface }, + title: { fontSize: 17, fontWeight: "600", color: colors.text }, + subtitle: { fontSize: 14, color: colors.muted, marginTop: spacing.xs }, +}); diff --git a/tooling/annotation-eval/fixture/src/components/CardList.tsx b/tooling/annotation-eval/fixture/src/components/CardList.tsx new file mode 100644 index 00000000..dc1b9623 --- /dev/null +++ b/tooling/annotation-eval/fixture/src/components/CardList.tsx @@ -0,0 +1,23 @@ +import { StyleSheet, View } from "react-native"; +import { Card } from "./Card"; +import { spacing } from "../theme"; + +const ITEMS = [ + { id: "trails", title: "Mountain trails", subtitle: "12 routes nearby" }, + { id: "coffee", title: "Coffee spots", subtitle: "8 open now" }, + { id: "museums", title: "Museums", subtitle: "3 free this week" }, +]; + +export function CardList() { + return ( + + {ITEMS.map(item => ( + + ))} + + ); +} + +const styles = StyleSheet.create({ + list: { gap: spacing.sm }, +}); diff --git a/tooling/annotation-eval/fixture/src/components/Footer.tsx b/tooling/annotation-eval/fixture/src/components/Footer.tsx new file mode 100644 index 00000000..5cdac57c --- /dev/null +++ b/tooling/annotation-eval/fixture/src/components/Footer.tsx @@ -0,0 +1,16 @@ +import { StyleSheet, Text, View } from "react-native"; +import { colors, spacing } from "../theme"; + +export function Footer() { + return ( + + Terms + Privacy + + ); +} + +const styles = StyleSheet.create({ + footer: { flexDirection: "row", justifyContent: "center", gap: spacing.lg, paddingVertical: spacing.lg }, + link: { fontSize: 14, color: colors.muted }, +}); diff --git a/tooling/annotation-eval/fixture/src/components/Header.tsx b/tooling/annotation-eval/fixture/src/components/Header.tsx new file mode 100644 index 00000000..b441cce4 --- /dev/null +++ b/tooling/annotation-eval/fixture/src/components/Header.tsx @@ -0,0 +1,19 @@ +import { StyleSheet, Text, View } from "react-native"; +import { colors, spacing } from "../theme"; + +type Props = { title: string; badge?: string }; + +export function Header({ title, badge }: Props) { + return ( + + {title} + {badge ? {badge} : null} + + ); +} + +const styles = StyleSheet.create({ + row: { flexDirection: "row", alignItems: "center", gap: spacing.sm }, + title: { fontSize: 28, fontWeight: "700", color: colors.text }, + badge: { fontSize: 12, fontWeight: "600", color: colors.surface, backgroundColor: colors.badge, paddingHorizontal: spacing.sm, borderRadius: 8 }, +}); diff --git a/tooling/annotation-eval/fixture/src/components/PrimaryButton.tsx b/tooling/annotation-eval/fixture/src/components/PrimaryButton.tsx new file mode 100644 index 00000000..9a569f55 --- /dev/null +++ b/tooling/annotation-eval/fixture/src/components/PrimaryButton.tsx @@ -0,0 +1,17 @@ +import { Pressable, StyleSheet, Text } from "react-native"; +import { colors, spacing } from "../theme"; + +type Props = { label: string; onPress: () => void }; + +export function PrimaryButton({ label, onPress }: Props) { + return ( + + {label} + + ); +} + +const styles = StyleSheet.create({ + button: { backgroundColor: colors.primary, paddingVertical: spacing.md, borderRadius: 12, alignItems: "center" }, + label: { color: colors.surface, fontSize: 16, fontWeight: "600" }, +}); diff --git a/tooling/annotation-eval/fixture/src/screens/HomeScreen.tsx b/tooling/annotation-eval/fixture/src/screens/HomeScreen.tsx new file mode 100644 index 00000000..4160cfca --- /dev/null +++ b/tooling/annotation-eval/fixture/src/screens/HomeScreen.tsx @@ -0,0 +1,21 @@ +import { ScrollView, StyleSheet } from "react-native"; +import { Header } from "../components/Header"; +import { CardList } from "../components/CardList"; +import { PrimaryButton } from "../components/PrimaryButton"; +import { Footer } from "../components/Footer"; +import { spacing } from "../theme"; + +export function HomeScreen() { + return ( + +
+ + {}} /> +