From c029076eaede48cd3052af73087a29b2b0f3a519 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 26 Oct 2026 15:11:45 +0100 Subject: [PATCH] fix(editor): offer 3D Orbit only with a recorded cursor The moving camera follows the cursor under Auto focus, and a take recorded with the system cursor writes no .cursor.json for the compositor to read, so picking 3D Orbit silently held still (#1397). Not offered under Auto focus unless every take under the selected zoom carries cursor data: a zoom cut at a clip junction is one fragment per clip, and a take without a track would hold the camera still for its part of the pill. Manual focus still offers it, where the focus point poses the camera, and a stored orbit stays listed so the row can show it. --- .../ai-edition/recordedCursorTypes.test.ts | 86 +++++++++++- .../ai-edition/recordedCursorTypes.ts | 82 ++++++++++-- .../ai-edition/v4/FloatingInspector.test.tsx | 122 ++++++++++++++++++ .../ai-edition/v4/FloatingInspector.tsx | 28 +++- 4 files changed, 302 insertions(+), 16 deletions(-) diff --git a/src/components/ai-edition/recordedCursorTypes.test.ts b/src/components/ai-edition/recordedCursorTypes.test.ts index c20d805961..f30f815815 100644 --- a/src/components/ai-edition/recordedCursorTypes.test.ts +++ b/src/components/ai-edition/recordedCursorTypes.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it } from "vitest"; -import { cursorStateChanges, cursorStatesBetween } from "./recordedCursorTypes"; +import { + type AxcutDocument, + type AxcutZoomRegion, + assetSchema, + clipSchema, + createEmptyDocument, + zoomRegionSchema, +} from "@/lib/ai-edition/schema"; +import { cursorStateChanges, cursorStatesBetween, zoomRecordingPaths } from "./recordedCursorTypes"; describe("cursorStateChanges", () => { it("keeps the changes only, and reads a sample with no state as the arrow", () => { @@ -36,3 +44,79 @@ describe("cursorStatesBetween", () => { expect(cursorStatesBetween(changes, 3500, 9000)).toEqual(new Set(["resize-ew"])); }); }); + +describe("zoomRecordingPaths", () => { + /** One clip per take, ten seconds each, laid end to end. */ + function project(takes: string[]): AxcutDocument { + const assets = takes.map((take) => + assetSchema.parse({ + id: `a_${take}`, + label: "take", + originalPath: `/recordings/${take}.mp4`, + }), + ); + const clips = assets.map((asset, i) => + clipSchema.parse({ + id: `c${i}`, + assetId: asset.id, + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: i * 10, + timelineEndSec: i * 10 + 10, + origin: "user", + }), + ); + const document = createEmptyDocument({ projectId: "p", title: "t" }); + return { ...document, assets, timeline: { ...document.timeline, clips } }; + } + + const zoom = (id: string, clipId: string, startMs: number, endMs: number): AxcutZoomRegion => + zoomRegionSchema.parse({ + id, + clipId, + startMs, + endMs, + depth: 3, + focus: { cx: 0.5, cy: 0.5 }, + focusMode: "auto", + }); + + it("names the recording of the clip the zoom is anchored to", () => { + const document = project(["a", "b"]); + const regions = [zoom("z", "c0", 0, 5000)]; + expect(zoomRecordingPaths(regions, "z", document)).toEqual(["/recordings/a.mp4"]); + }); + + // The same zoom, cut at the clip junction: one fragment per clip, one pill on the ruler. + it("names every recording of a zoom cut at a clip junction", () => { + const document = project(["a", "b"]); + const pill = [zoom("z1", "c0", 5000, 10000), zoom("z2", "c1", 10000, 15000)]; + expect(zoomRecordingPaths(pill, "z1", document)).toEqual([ + "/recordings/a.mp4", + "/recordings/b.mp4", + ]); + }); + + // A document migrated before clip anchors only knows the region's raw span. + it("falls back to the clips an unanchored zoom overlaps", () => { + const document = project(["a", "b"]); + const legacy = zoomRegionSchema.parse({ + id: "z", + startMs: 8000, + endMs: 12000, + depth: 3, + focus: { cx: 0.5, cy: 0.5 }, + focusMode: "auto", + }); + expect(zoomRecordingPaths([legacy], "z", document)).toEqual([ + "/recordings/a.mp4", + "/recordings/b.mp4", + ]); + }); + + it("answers nothing for an unknown zoom, or without a document", () => { + const document = project(["a"]); + expect(zoomRecordingPaths([], "z", document)).toEqual([]); + expect(zoomRecordingPaths([zoom("z", "c0", 0, 5000)], "z", null)).toEqual([]); + }); +}); diff --git a/src/components/ai-edition/recordedCursorTypes.ts b/src/components/ai-edition/recordedCursorTypes.ts index dc6a28c2a6..9786a0544f 100644 --- a/src/components/ai-edition/recordedCursorTypes.ts +++ b/src/components/ai-edition/recordedCursorTypes.ts @@ -1,7 +1,8 @@ import { useEffect, useMemo, useReducer, useState } from "react"; -import type { AxcutDocument } from "@/lib/ai-edition/schema"; +import type { AxcutDocument, AxcutZoomRegion } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { resolveClipSourceEndSec } from "@/lib/ai-edition/timeline/clipDuration"; +import { resolveRegionIds } from "@/lib/ai-edition/timeline/timelineMap"; import { nativeBridgeClient } from "@/native"; interface CursorStateChange { @@ -91,15 +92,29 @@ function cursorDataIn(paths: readonly string[]): boolean | null { } /** - * Whether any recording the timeline plays has cursor data, or `null` while they are read. A take - * recorded with the system cursor has it baked into the pixels and no cursor file, so every cursor - * setting would change nothing: the editor leaves them out. + * Whether EVERY recording in `paths` has cursor data, or `null` while some are still unread. + * One take without a cursor file is enough to answer no: a zoom over it has no track to follow. */ -export function useHasRecordedCursor(): boolean | null { - // A string, so the recordings are read again when one is swapped, not on every other edit. - const pathsJson = useProjectStore((s) => - JSON.stringify([...new Set(clipSourceRanges(s.document).map(([path]) => path))]), - ); +export function useRecordingsHaveCursor(paths: readonly string[]): boolean | null { + // Sorted, so the same set of recordings is one read whatever order the clips name them in. + const pathsJson = JSON.stringify([...new Set(paths)].sort()); + const read = useRecordingReads(pathsJson); + return everyCursorDataIn(read); +} + +/** Whether every one of `paths` has cursor data: `false` as soon as one answered without it, + * `null` while one is unread. An empty list names no recording, and nothing could follow. */ +function everyCursorDataIn(paths: readonly string[]): boolean | null { + if (paths.length === 0) return false; + if (paths.some((path) => hasCursorByPath.get(path) === false)) return false; + return paths.every((path) => hasCursorByPath.get(path) === true) ? true : null; +} + +/** + * Reads every recording in `paths` once per session and asks for a re-render as each answer + * lands. Returns the deduplicated paths it read, which is what the caller's answer is about. + */ +function useRecordingReads(pathsJson: string): string[] { const paths = useMemo(() => JSON.parse(pathsJson) as string[], [pathsJson]); // The answer is read off the cache on every render, so it always belongs to the current // recordings; each read finishing only asks for another render. @@ -115,9 +130,58 @@ export function useHasRecordedCursor(): boolean | null { cancelled = true; }; }, [paths]); + return paths; +} + +/** + * Whether any recording the timeline plays has cursor data, or `null` while they are read. A take + * recorded with the system cursor has it baked into the pixels and no cursor file, so every cursor + * setting would change nothing: the editor leaves them out. + */ +export function useHasRecordedCursor(): boolean | null { + // A string, so the recordings are read again when one is swapped, not on every other edit. + const pathsJson = useProjectStore((s) => + JSON.stringify([...new Set(clipSourceRanges(s.document).map(([path]) => path))]), + ); + const paths = useRecordingReads(pathsJson); return cursorDataIn(paths); } +/** + * The recordings a zoom region plays over, one path per distinct video file. + * + * A region cut at a clip junction is stored as one fragment per clip, so a zoom that runs from + * one take onto another names both. A region migrated without a clip anchor only knows its raw + * span, and the clips it overlaps are its recordings instead. Empty when nothing can be resolved + * (never saved without clips; a fragment whose clip is gone is not rendered either). + */ +export function zoomRecordingPaths( + regions: AxcutZoomRegion[], + id: string, + document: AxcutDocument | null, +): string[] { + if (!document) return []; + const fragments = new Set(resolveRegionIds(regions, id, 0.001)); + const clips = new Map(document.timeline.clips.map((clip) => [clip.id, clip])); + const paths = new Set(); + for (const region of regions) { + if (!fragments.has(region.id)) continue; + const covered = region.clipId + ? [clips.get(region.clipId)] + : document.timeline.clips.filter( + (clip) => + clip.timelineStartSec * 1000 < region.endMs && + clip.timelineEndSec * 1000 > region.startMs, + ); + for (const clip of covered) { + if (!clip) continue; + const asset = document.assets.find((candidate) => candidate.id === clip.assetId); + if (asset?.kind === "video" && asset.originalPath) paths.add(asset.originalPath); + } + } + return [...paths]; +} + /** * Every cursor state the timeline's clips put on screen, or `null` while their recordings are * read. The cursor pane offers to redraw these and nothing else: a choice about a cursor the diff --git a/src/components/ai-edition/v4/FloatingInspector.test.tsx b/src/components/ai-edition/v4/FloatingInspector.test.tsx index 090f87fbc7..dc48c48bd8 100644 --- a/src/components/ai-edition/v4/FloatingInspector.test.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.test.tsx @@ -367,7 +367,58 @@ describe("FloatingInspector", () => { "button", ); + /** A project whose takes recorded `samples` cursor samples each (one clip per take). */ + function openTakes(samples: number[]) { + const run = crypto.randomUUID(); + const assets = samples.map((_, i) => + assetSchema.parse({ + id: `a${i}`, + label: "take", + originalPath: `/recordings/${run}-${i}.mp4`, + }), + ); + const clips = assets.map((asset, i) => + clipSchema.parse({ + id: `c${i}`, + assetId: asset.id, + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: i * 10, + timelineEndSec: i * 10 + 10, + origin: "user", + }), + ); + const samplesOf = new Map(assets.map((asset, i) => [asset.originalPath, samples[i]])); + const read = vi + .spyOn(nativeBridgeClient.cursor, "getRecordingData") + .mockImplementation(async (videoPath) => ({ + version: 2, + provider: "native", + assets: [], + samples: Array.from({ length: samplesOf.get(videoPath ?? "") ?? 0 }, (_, i) => ({ + timeMs: i * 100, + cx: 0.5, + cy: 0.5, + })), + })); + const document = createEmptyDocument({ projectId: "p", title: "t" }); + useProjectStore.setState({ + projectId: "p", + document: { ...document, assets, timeline: { ...document.timeline, clips } }, + }); + return read; + } + + /** Lets the takes' cursor files be read, so an absence is an answer and not a wait. */ + async function readTakes(read: ReturnType) { + await waitFor(() => expect(read).toHaveBeenCalled()); + await act(async () => { + await Promise.all(read.mock.results.map((result) => result.value)); + }); + } + afterEach(() => { + vi.restoreAllMocks(); useProjectStore.setState({ document: null }); }); @@ -459,6 +510,77 @@ describe("FloatingInspector", () => { editorSettings.autoFocusAll = false; } }); + + // Issue #1397. A take recorded with the system cursor has no cursor file, so Auto focus + // has nothing to read and 3D Orbit would hold still — exactly the camera the reporter + // picked and then watched do nothing. + it("leaves the orbit out under Auto focus when the take has no recorded cursor", async () => { + const read = openTakes([0]); + const { tl } = zoomTl({ focusMode: "auto", clipId: "c0" }); + render(); + await readTakes(read); + expect( + screen.queryByRole("button", { name: "settings.zoom.camera.preset.orbit" }), + ).toBeNull(); + // The fixed angles need no cursor, so they stay. + expect( + screen.getByRole("button", { name: "settings.zoom.camera.preset.left" }), + ).toBeInTheDocument(); + }); + + it("offers the orbit under Auto focus once the take has a recorded cursor", async () => { + const read = openTakes([3]); + const { tl } = zoomTl({ focusMode: "auto", clipId: "c0" }); + render(); + await readTakes(read); + expect( + await screen.findByRole("button", { name: "settings.zoom.camera.preset.orbit" }), + ).toBeInTheDocument(); + }); + + // The recordings under the zoom decide, not the project: a zoom that stays on the recorded + // take still follows the cursor, next to a system-cursor take on the same timeline. + it("offers the orbit under Auto focus over a take with a cursor beside one without", async () => { + const read = openTakes([3, 0]); + const { tl } = zoomTl({ focusMode: "auto", clipId: "c0" }); + render(); + await readTakes(read); + expect( + await screen.findByRole("button", { name: "settings.zoom.camera.preset.orbit" }), + ).toBeInTheDocument(); + }); + + // ...and the opposite: a zoom on the system-cursor take is left out even though another + // recording on the timeline has a cursor. + it("leaves the orbit out under Auto focus over the take without a cursor", async () => { + const read = openTakes([3, 0]); + const { tl } = zoomTl({ focusMode: "auto", clipId: "c1" }); + render(); + await readTakes(read); + expect( + screen.queryByRole("button", { name: "settings.zoom.camera.preset.orbit" }), + ).toBeNull(); + }); + + // A zoom cut at a clip junction is one pill over both takes: the camera would follow on the + // recorded one and hold still on the other, so Auto focus must not offer it at all. + it("leaves the orbit out under Auto focus when the zoom runs onto a take without a cursor", async () => { + const read = openTakes([3, 0]); + const pill = [ + { id: "z1", clipId: "c0", startMs: 5000, endMs: 10000, depth: 3, focusMode: "auto" }, + { id: "z2", clipId: "c1", startMs: 10000, endMs: 15000, depth: 3, focusMode: "auto" }, + ]; + const tl = { + ...defaultProps.tl, + selection: { kind: "zoom", id: "z1" }, + zoomRegions: pill, + } as unknown as React.ComponentProps["tl"]; + render(); + await readTakes(read); + expect( + screen.queryByRole("button", { name: "settings.zoom.camera.preset.orbit" }), + ).toBeNull(); + }); }); describe("cursor facet", () => { diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index b82a0e9cd8..08c2180f51 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -87,7 +87,11 @@ import { TranscriptPane, VideoEffectsPane, } from "../RightPanes"; -import { useHasRecordedCursor } from "../recordedCursorTypes"; +import { + useHasRecordedCursor, + useRecordingsHaveCursor, + zoomRecordingPaths, +} from "../recordedCursorTypes"; import { TextColorField } from "../TextColorField"; import styles from "./EditorShellV4.module.css"; @@ -759,6 +763,17 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void } () => (doc && zoomId ? zoomScaleLimit(doc, zoomId) : MAX_ZOOM_SCALE), [doc, zoomId], ); + // The recordings under the selected zoom, one per clip it covers (a region cut at a clip + // junction is one fragment per clip), empty for a non-zoom selection. + const zoomPaths = useMemo( + () => + tl.selection?.kind === "zoom" ? zoomRecordingPaths(tl.zoomRegions, tl.selection.id, doc) : [], + [doc, tl.selection, tl.zoomRegions], + ); + // Every take under the zoom must carry a cursor track: the 3D Orbit follows the cursor of + // whichever take plays, so one without a track would hold the camera still for its part of the + // pill, and the camera is not offered there under Auto focus. + const zoomHasCursor = useRecordingsHaveCursor(zoomPaths) === true; // Mise à jour en direct regroupée à une par frame. `updateAnnotationLive` remplace le document // dans le store, donc chaque appel fait reconstruire et re-sérialiser toute la scène avant de // la pousser au natif : c'est le juste prix une fois par image, mais un `` @@ -852,11 +867,12 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void } tiles options={[ "off" as const, - // The orbit follows the cursor under auto focus, and a hidden cursor drives - // nothing: not offered then, rather than a camera that silently holds still. - // Under manual focus the focus point poses it, cursor or not. Still listed - // once picked, so the row can show it. - ...(settings.cursorShow || + // The orbit follows the cursor under auto focus, so it needs one the compositor + // can read: a recording made with the system cursor writes no cursor file, and + // a hidden cursor drives nothing. Not offered then, rather than a camera that + // silently holds still. Under manual focus the focus point poses it, cursor or + // not. Still listed once picked, so the row can show it. + ...((zoomHasCursor && settings.cursorShow) || (!autoFocusAll && (region.focusMode ?? "manual") === "manual") || region.rotationPreset === "orbit" ? MOVING_ROTATION_3D_PRESETS