From dec150d1ef5682071ef57fa45fc4b343fec531de Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Thu, 17 Sep 2026 21:14:46 -0400 Subject: [PATCH 1/2] Recentre the other stereo camera on the same object when synchronised panning settles --- client/dive-common/components/Viewer.vue | 42 +++++++++++ .../use/stereo/useStereoOnnxTransfer.ts | 18 +++++ .../frontend/components/ViewerLoader.vue | 34 ++++++++- .../platform/web-girder/useStereoOnnxWeb.ts | 6 ++ .../web-girder/views/ViewerLoader.vue | 4 +- .../annotators/mediaControllerType.ts | 9 +++ .../annotators/useMediaController.ts | 29 ++++++++ .../components/annotators/viewLink.spec.ts | 73 +++++++++++++++++++ client/src/components/annotators/viewLink.ts | 56 ++++++++++++++ docs/Multicamera-data.md | 2 +- 10 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 client/src/components/annotators/viewLink.spec.ts create mode 100644 client/src/components/annotators/viewLink.ts diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 75b3710e4..6fcf47dca 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -100,6 +100,13 @@ import AlignedViewToggle from './AlignedViewToggle.vue'; import PrimaryAttributeTrackFilter from './PrimaryAttributeTrackFilter.vue'; import UserSettingsDialog from './UserSettingsDialog.vue'; +export interface StereoViewLinkParams { + camera: string; + frameNum: number; + point: [number, number]; +} +export type StereoViewLinkFunc = (params: StereoViewLinkParams) => Promise<[number, number] | null>; + export interface ImageDataItem { url: string; filename: string; @@ -185,6 +192,15 @@ export default defineComponent({ type: Number as PropType, default: undefined, }, + /** + * Where a point on one stereo camera lands on the other, using the loaded + * stereo matcher; null when it cannot be found. Lets synchronised panning + * follow the same object on both cameras. + */ + stereoViewLink: { + type: Function as PropType, + default: undefined, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -801,6 +817,32 @@ export default defineComponent({ provideAutoRegisterJob(autoRegisterJob); onBeforeUnmount(() => autoRegisterJob.dispose()); + // Linked panning: with camera controls synchronised and auto-compute on, + // the other pane recentres on where this pane's centre is on its camera. + const stereoViewLinkResolver = async (camera: string, point: [number, number]) => { + if (!props.stereoViewLink) return null; + let frameNum: number; + try { + frameNum = aggregateController.value.getController(camera).frame.value; + } catch { + return null; + } + return props.stereoViewLink({ camera, frameNum, point }); + }; + watch( + [ + () => clientSettings.stereoSettings.autoComputeOtherCamera, + () => props.stereoViewLink, + () => multiCamList.value.length, + ], + ([autoCompute, link, cameras]) => { + aggregateController.value.setViewLinkResolver( + autoCompute && link && cameras === 2 ? stereoViewLinkResolver : null, + ); + }, + { immediate: true }, + ); + // Provides wrappers for actions to integrate with settings const { linkingTrack, diff --git a/client/dive-common/use/stereo/useStereoOnnxTransfer.ts b/client/dive-common/use/stereo/useStereoOnnxTransfer.ts index cb4382eeb..be1e05e1e 100644 --- a/client/dive-common/use/stereo/useStereoOnnxTransfer.ts +++ b/client/dive-common/use/stereo/useStereoOnnxTransfer.ts @@ -568,10 +568,28 @@ export default function useStereoOnnxTransfer(config: StereoOnnxTransferConfig) return counts; } + /** + * Where one point on `sourceCamera` lands on the other camera at `frameNum`, + * or null when the match is rejected or the stereo setup is incomplete. + * Used for linked panning, so it never reports errors to the user. + */ + async function warpPoint(point: Point, sourceCamera: string, frameNum: number): Promise { + const otherCamera = getMultiCamList().find((c) => c !== sourceCamera); + if (!otherCamera || getMultiCamList().length !== 2) return null; + try { + const [result] = await warp([point], sourceCamera, otherCamera, frameNum); + return result?.accepted && Number.isFinite(result.x) && Number.isFinite(result.y) + ? [result.x, result.y] : null; + } catch { + return null; + } + } + return { handleStereoAnnotationComplete, handleStereoTrackLinked, warpAllFromCamera, + warpPoint, measureAtFrame, precomputeFrame, }; diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index 4f41204fb..cfb39d307 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -9,7 +9,7 @@ import { import { ANNOTATION_SOURCE_QUERY } from 'dive-common/scoring/viewerNavigation'; import { parseViewerFocus } from 'dive-common/review/viewerNavigation'; import { useRoute, useRouter } from 'vue-router/composables'; -import Viewer from 'dive-common/components/Viewer.vue'; +import Viewer, { StereoViewLinkParams } from 'dive-common/components/Viewer.vue'; import RunPipelineMenu from 'dive-common/components/RunPipelineMenu.vue'; import ImportAnnotations from 'dive-common//components/ImportAnnotations.vue'; import CalibrationMenu from 'dive-common/components/CalibrationMenu.vue'; @@ -1508,6 +1508,36 @@ export default defineComponent({ * result status instead — used by bulk import "Warp to All" so failures * can be aggregated rather than cleared by the next job. */ + /** + * Where `point` on `camera` lands on the other stereo camera, for linked + * panning. Uses whatever matcher the stereo service loaded; null when the + * service is off or the match is rejected. + */ + async function stereoViewLink(params: StereoViewLinkParams): Promise<[number, number] | null> { + if (!stereoEnabled.value) return null; + const cameras = Object.keys(stereoImagePathGetters.value); + if (cameras.length !== 2 || !cameras.includes(params.camera)) return null; + if (!(await ensureStereoFrame(params.frameNum))) return null; + const fps = stereoCameraFps.value[cameras[0]] || stereoDatasetFps || Object.values(stereoCameraFps.value)[0]; + try { + const response = await stereoTransferPoints({ + points: [params.point], + strict: true, + sourceCamera: params.camera === cameras[0] ? 'left' : 'right', + leftImagePath: stereoImagePathGetters.value[cameras[0]](params.frameNum), + rightImagePath: stereoImagePathGetters.value[cameras[1]](params.frameNum), + frameTime: fps ? params.frameNum / fps : undefined, + }); + const point = response.transferredPoints?.[0]; + if (!response.success || response.validMatches?.[0] !== true || !point?.every(Number.isFinite)) { + return null; + } + return [point[0], point[1]]; + } catch { + return null; + } + } + async function handleStereoAnnotationComplete( params: StereoAnnotationCompleteParams, forceAutoCompute = false, @@ -2188,6 +2218,7 @@ export default defineComponent({ stereoLengthMessage, closeStereoLoadingDialog, handleStereoAnnotationComplete, + stereoViewLink, handleStereoWarpImported, handleStereoAnnotationReset, handleStereoSegmentationFinalize, @@ -2215,6 +2246,7 @@ export default defineComponent({ :initial-track-id="viewerFocus.trackId" :text-query-enabled="true" :text-query-available="textQueryAvailable" + :stereo-view-link="stereoViewLink" @return-to-current-annotations="returnToCurrentAnnotations" @change-camera="changeCamera" @large-image-warning="largeImageWarning()" diff --git a/client/platform/web-girder/useStereoOnnxWeb.ts b/client/platform/web-girder/useStereoOnnxWeb.ts index 2732f37d9..c85919430 100644 --- a/client/platform/web-girder/useStereoOnnxWeb.ts +++ b/client/platform/web-girder/useStereoOnnxWeb.ts @@ -424,10 +424,16 @@ export default function useStereoOnnxWeb(opts: StereoOnnxWebOptions) { return getTransfer()?.warpAllFromCamera(cameraName); } + async function stereoViewLink(params: { camera: string; frameNum: number; point: [number, number] }) { + if (!clientSettings.stereoSettings.autoComputeOtherCamera) return null; + return (await getTransfer()?.warpPoint(params.point, params.camera, params.frameNum)) ?? null; + } + return { handleStereoAnnotationComplete, handleStereoTrackLinked, warpAllFromCamera, + stereoViewLink, precomputeCurrentFrame, invalidateCalibration, }; diff --git a/client/platform/web-girder/views/ViewerLoader.vue b/client/platform/web-girder/views/ViewerLoader.vue index 8c9737094..eedb65de1 100644 --- a/client/platform/web-girder/views/ViewerLoader.vue +++ b/client/platform/web-girder/views/ViewerLoader.vue @@ -177,7 +177,7 @@ export default defineComponent({ const { handleStereoAnnotationComplete, handleStereoTrackLinked, warpAllFromCamera, - invalidateCalibration, + invalidateCalibration, stereoViewLink, } = useStereoOnnxWeb({ getViewer: () => viewerRef.value, getDatasetId: () => parentDatasetId(props.id), @@ -518,6 +518,7 @@ export default defineComponent({ exportDatasetIds, handleStereoAnnotationComplete, handleStereoTrackLinked, + stereoViewLink, stereoBusyMessage, stereoDownloadProgress, stereoDownloadPercent, @@ -550,6 +551,7 @@ export default defineComponent({ :annotation-source-returnable="annotationSourceReturnable" :initial-frame="viewerFocus.frame" :initial-track-id="viewerFocus.trackId" + :stereo-view-link="stereoViewLink" @return-to-current-annotations="returnToCurrentAnnotations" @large-image-warning="largeImageWarning()" @update:set="routeSet" diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index 95c9bd6de..5b3cd4ba0 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -75,6 +75,15 @@ export interface AggregateMediaController { setSpeed: (speed: number) => void; getController: (cameraName: string) => MediaController; toggleSynchronizeCameras: (sync: boolean) => void; + /** + * While cameras are synchronised, recentre the other panes on where the + * source pane's centre lands on their camera (null to disable). The + * resolver returns null when the point cannot be found there. + */ + setViewLinkResolver: ( + resolver: ((sourceCamera: string, point: [number, number]) => + Promise<[number, number] | null>) | null, + ) => void; /** * Seeks so that `camera` lands on its own local frame `localFrame` (e.g. * jumping to a track's stored begin/end, which is in local-frame units). diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 3f7f47336..29110e43d 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -7,6 +7,7 @@ import Vue, { ref, shallowRef, reactive, provide, toRef, Ref, UnwrapRef, computed, watch, } from 'vue'; import { map, over } from 'lodash'; +import createViewLink from './viewLink'; import { use } from '../../provides'; import type { @@ -113,6 +114,23 @@ export function useMediaController() { let state: Record> = {}; let cameraControllerSymbols: Record = {}; const synchronizeCameras: Ref = ref(false); + + // Installed by the viewer while stereo auto-compute is on; otherwise + // synchronised panes only copy each other's screen motion. + const viewLink = createViewLink({ + center: (key) => geoViewers[key]?.value?.center(), + cameraName: (key) => state[key]?.cameraName, + synced: () => synchronizeCameras.value, + recenter: (sourceKey, point) => { + allowCameraTrigger = false; + Object.entries(geoViewers).forEach(([camera, geoViewer]) => { + if (geoViewer.value && camera !== sourceKey) { + geoViewer.value.center({ x: point[0], y: point[1] }); + } + }); + allowCameraTrigger = true; + }, + }); const resizeTrigger: Ref = ref(0); // Raised only while onResize applies its programmatic resetZoom, so the // linked-viewer navigation ignores the resulting pan/zoom events (see @@ -150,6 +168,7 @@ export function useMediaController() { currentTime: emptyControllerCurrentTime, getController, toggleSynchronizeCameras, + setViewLinkResolver, cameraSync: synchronizeCameras, resizeTrigger, resizing, @@ -287,6 +306,12 @@ export function useMediaController() { synchronizeCameras.value = val; } + function setViewLinkResolver( + resolver: Parameters[0], + ) { + viewLink.setResolver(resolver); + } + /** * Optional replacement for the aggregate "reset pan and zoom" behavior, * installed by the aligned-view navigation link (useAlignedNavigation). @@ -320,6 +345,7 @@ export function useMediaController() { } }); allowCameraTrigger = true; + viewLink.schedule(camEvent.camera); } }); @@ -333,6 +359,7 @@ export function useMediaController() { } }); allowCameraTrigger = true; + viewLink.schedule(camEvent.camera); } }); /** @@ -654,6 +681,7 @@ export function useMediaController() { getController, resetMapDimensions, toggleSynchronizeCameras, + setViewLinkResolver, cameraSync: synchronizeCameras, resizeTrigger, resizing, @@ -820,6 +848,7 @@ export function useMediaController() { currentTime: defaultController.currentTime, getController, toggleSynchronizeCameras, + setViewLinkResolver, cameraSync: synchronizeCameras, resizeTrigger, resizing, diff --git a/client/src/components/annotators/viewLink.spec.ts b/client/src/components/annotators/viewLink.spec.ts new file mode 100644 index 000000000..5841b26c0 --- /dev/null +++ b/client/src/components/annotators/viewLink.spec.ts @@ -0,0 +1,73 @@ +import createViewLink from './viewLink'; + +type Resolver = (camera: string, point: [number, number]) => Promise<[number, number] | null>; + +function harness(resolver: Resolver = vi.fn(async () => [30, 40] as [number, number])) { + const recenter = vi.fn(); + let synced = true; + const link = createViewLink({ + center: (key) => (key === 'A' ? { x: 10, y: 20 } : undefined), + cameraName: (key) => ({ A: 'left', B: 'right' } as Record)[key], + recenter, + synced: () => synced, + delayMs: 100, + }); + link.setResolver(resolver); + return { + link, recenter, resolver, setSynced: (v: boolean) => { synced = v; }, + }; +} + +const settle = async () => { await vi.runAllTimersAsync(); }; + +beforeEach(() => vi.useFakeTimers()); +afterEach(() => vi.useRealTimers()); + +it('recentres the other pane on where the source centre lands', async () => { + const h = harness(); + h.link.schedule('A'); + await settle(); + expect(h.resolver).toHaveBeenCalledWith('left', [10, 20]); + expect(h.recenter).toHaveBeenCalledWith('A', [30, 40]); +}); + +it('issues one lookup per settled drag', async () => { + const h = harness(); + h.link.schedule('A'); vi.advanceTimersByTime(50); + h.link.schedule('A'); vi.advanceTimersByTime(50); + h.link.schedule('A'); + await settle(); + expect(h.resolver).toHaveBeenCalledTimes(1); +}); + +it('leaves the plain sync alone when the point is not found or the lookup throws', async () => { + const miss = harness(vi.fn(async () => null)); + miss.link.schedule('A'); await settle(); + expect(miss.recenter).not.toHaveBeenCalled(); + const boom = harness(vi.fn(async () => { throw new Error('no model'); })); + boom.link.schedule('A'); await settle(); + expect(boom.recenter).not.toHaveBeenCalled(); +}); + +it('drops a lookup overtaken by a newer one or finished after sync was turned off', async () => { + let release: (p: [number, number]) => void = () => undefined; + const slow: Resolver = () => new Promise<[number, number]>((resolve) => { release = resolve; }); + const h = harness(slow); + h.link.schedule('A'); vi.advanceTimersByTime(100); + const firstRelease = release; + h.link.schedule('A'); vi.advanceTimersByTime(100); + firstRelease([1, 1]); await Promise.resolve(); await Promise.resolve(); + expect(h.recenter).not.toHaveBeenCalled(); + h.setSynced(false); + release([2, 2]); await Promise.resolve(); await Promise.resolve(); + expect(h.recenter).not.toHaveBeenCalled(); +}); + +it('does nothing without a resolver or for a pane without a map', async () => { + const h = harness(); + h.link.schedule('B'); await settle(); + expect(h.resolver).not.toHaveBeenCalled(); + h.link.setResolver(null); + h.link.schedule('A'); await settle(); + expect(h.resolver).not.toHaveBeenCalled(); +}); diff --git a/client/src/components/annotators/viewLink.ts b/client/src/components/annotators/viewLink.ts new file mode 100644 index 000000000..0aedf9f70 --- /dev/null +++ b/client/src/components/annotators/viewLink.ts @@ -0,0 +1,56 @@ +export type ViewLinkResolver = (sourceCamera: string, point: [number, number]) => + Promise<[number, number] | null>; + +export interface ViewLinkDeps { + /** Centre of a pane in image coordinates; undefined when it has no map. */ + center: (paneKey: string) => { x: number; y: number } | undefined; + cameraName: (paneKey: string) => string | undefined; + /** Recentre every pane but the source on `point`. */ + recenter: (sourceKey: string, point: [number, number]) => void; + synced: () => boolean; + delayMs?: number; +} + +/** + * Keeps synchronised panes on the same object across a stereo rig. Plain + * screen-delta sync stays in charge of the motion; once it settles, the + * source pane's centre is looked up on the other camera and the other panes + * recentre on it. A lookup that fails, is overtaken by a newer one, or lands + * after sync was turned off leaves the plain sync as is. + */ +export default function createViewLink(deps: ViewLinkDeps) { + const delay = deps.delayMs ?? 150; + let resolver: ViewLinkResolver | null = null; + let timer: ReturnType | undefined; + let request = 0; + + function setResolver(next: ViewLinkResolver | null) { + resolver = next; + clearTimeout(timer); + } + + async function link(sourceKey: string) { + const resolve = resolver; + const center = deps.center(sourceKey); + const sourceName = deps.cameraName(sourceKey); + if (!resolve || !center || !sourceName || !deps.synced()) return; + request += 1; + const current = request; + let target: [number, number] | null = null; + try { + target = await resolve(sourceName, [center.x, center.y]); + } catch { + target = null; + } + if (!target || current !== request || !deps.synced()) return; + deps.recenter(sourceKey, target); + } + + function schedule(sourceKey: string) { + if (!resolver) return; + clearTimeout(timer); + timer = setTimeout(() => { link(sourceKey); }, delay); + } + + return { setResolver, schedule, enabled: () => resolver !== null }; +} diff --git a/docs/Multicamera-data.md b/docs/Multicamera-data.md index 5cdac87eb..e67f3a80a 100644 --- a/docs/Multicamera-data.md +++ b/docs/Multicamera-data.md @@ -252,7 +252,7 @@ Open the ==:material-cog:== creation settings menu in the [Track List](UI-Track- | Setting | What it does | |---------|--------------| | **Update lengths when modified** | Recomputes stereo length measurements when you edit a head/tail line on a detection linked across both cameras. | -| **Auto-compute location on other camera** | Warps a new annotation drawn on one camera to the other camera when no detection exists there yet. | +| **Auto-compute location on other camera** | Warps a new annotation drawn on one camera to the other camera when no detection exists there yet. With **Synchronize camera controls** also on, panning or zooming one camera recentres the other on the same object: the point at the centre of the moved view is matched on the other camera with the loaded stereo method, and the other view is recentred there once the motion settles. Where no match is found the views keep moving together as before. | Enabling either option loads the interactive stereo service (shared with [interactive segmentation](Interactive-Annotation.md)). Warped head/tail lines become normal editable line annotations; manual edits are preserved and not overwritten by later auto-warping. From 0dac9660974342b1c309c8c281898bfaa0d87176 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Sun, 20 Sep 2026 16:21:58 -0400 Subject: [PATCH 2/2] Invalidate outstanding stereo pan lookups when settings or views change --- client/src/components/annotators/viewLink.spec.ts | 14 ++++++++++++++ client/src/components/annotators/viewLink.ts | 2 ++ 2 files changed, 16 insertions(+) diff --git a/client/src/components/annotators/viewLink.spec.ts b/client/src/components/annotators/viewLink.spec.ts index 5841b26c0..7a8a5af20 100644 --- a/client/src/components/annotators/viewLink.spec.ts +++ b/client/src/components/annotators/viewLink.spec.ts @@ -71,3 +71,17 @@ it('does nothing without a resolver or for a pane without a map', async () => { h.link.schedule('A'); await settle(); expect(h.resolver).not.toHaveBeenCalled(); }); + +it.each(['disable', 'replace', 'pan'] as const)('drops an active lookup immediately on %s', async (action) => { + let release!: (point: [number, number]) => void; + const h = harness(() => new Promise((resolve) => { release = resolve; })); + h.link.schedule('A'); + vi.advanceTimersByTime(100); + if (action === 'disable') h.link.setResolver(null); + if (action === 'replace') h.link.setResolver(async () => [50, 60]); + if (action === 'pan') h.link.schedule('A'); + // The old result must be discarded even before the next debounce expires. + release([1, 2]); + await Promise.resolve(); + expect(h.recenter).not.toHaveBeenCalled(); +}); diff --git a/client/src/components/annotators/viewLink.ts b/client/src/components/annotators/viewLink.ts index 0aedf9f70..5dfbcc88c 100644 --- a/client/src/components/annotators/viewLink.ts +++ b/client/src/components/annotators/viewLink.ts @@ -25,6 +25,7 @@ export default function createViewLink(deps: ViewLinkDeps) { let request = 0; function setResolver(next: ViewLinkResolver | null) { + request += 1; resolver = next; clearTimeout(timer); } @@ -47,6 +48,7 @@ export default function createViewLink(deps: ViewLinkDeps) { } function schedule(sourceKey: string) { + request += 1; if (!resolver) return; clearTimeout(timer); timer = setTimeout(() => { link(sourceKey); }, delay);