Recentre the other stereo camera on the same object when synchronised panning settles

client/dive-common/components/Viewer.vue

@@ -100,6 +100,13 @@ import AlignedViewToggle from './AlignedViewToggle.vue';
 import PrimaryAttributeTrackFilter from './PrimaryAttributeTrackFilter.vue';
 import UserSettingsDialog from './UserSettingsDialog.vue';
 
+export interface StereoViewLinkParams {
+  camera: string;
+  frameNum: number;
+  point: [number, number];
+}
+export type StereoViewLinkFunc = (params: StereoViewLinkParams) => Promise<[number, number] | null>;
+
 export interface ImageDataItem {
   url: string;
   filename: string;
@@ -185,6 +192,15 @@ export default defineComponent({
       type: Number as PropType<number | undefined>,
       default: undefined,
     },
+    /**
+     * Where a point on one stereo camera lands on the other, using the loaded
+     * stereo matcher; null when it cannot be found. Lets synchronised panning
+     * follow the same object on both cameras.
+     */
+    stereoViewLink: {
+      type: Function as PropType<StereoViewLinkFunc | undefined>,
+      default: undefined,
+    },
   },
   setup(props, { emit }) {
     const { prompt, visible } = usePrompt();
@@ -801,6 +817,32 @@ export default defineComponent({
     provideAutoRegisterJob(autoRegisterJob);
     onBeforeUnmount(() => autoRegisterJob.dispose());
 
+    // Linked panning: with camera controls synchronised and auto-compute on,
+    // the other pane recentres on where this pane's centre is on its camera.
+    const stereoViewLinkResolver = async (camera: string, point: [number, number]) => {
+      if (!props.stereoViewLink) return null;
+      let frameNum: number;
+      try {
+        frameNum = aggregateController.value.getController(camera).frame.value;
+      } catch {
+        return null;
+      }
+      return props.stereoViewLink({ camera, frameNum, point });
+    };
+    watch(
+      [
+        () => clientSettings.stereoSettings.autoComputeOtherCamera,
+        () => props.stereoViewLink,
+        () => multiCamList.value.length,
+      ],
+      ([autoCompute, link, cameras]) => {
+        aggregateController.value.setViewLinkResolver(
+          autoCompute && link && cameras === 2 ? stereoViewLinkResolver : null,
+        );
+      },
+      { immediate: true },
+    );
+
     // Provides wrappers for actions to integrate with settings
     const {
       linkingTrack,

client/dive-common/use/stereo/useStereoOnnxTransfer.ts

@@ -568,10 +568,28 @@ export default function useStereoOnnxTransfer(config: StereoOnnxTransferConfig)
     return counts;
   }
 
+  /**
+   * Where one point on `sourceCamera` lands on the other camera at `frameNum`,
+   * or null when the match is rejected or the stereo setup is incomplete.
+   * Used for linked panning, so it never reports errors to the user.
+   */
+  async function warpPoint(point: Point, sourceCamera: string, frameNum: number): Promise<Point | null> {
+    const otherCamera = getMultiCamList().find((c) => c !== sourceCamera);
+    if (!otherCamera || getMultiCamList().length !== 2) return null;
+    try {
+      const [result] = await warp([point], sourceCamera, otherCamera, frameNum);
+      return result?.accepted && Number.isFinite(result.x) && Number.isFinite(result.y)
+        ? [result.x, result.y] : null;
+    } catch {
+      return null;
+    }
+  }
+
   return {
     handleStereoAnnotationComplete,
     handleStereoTrackLinked,
     warpAllFromCamera,
+    warpPoint,
     measureAtFrame,
     precomputeFrame,
   };

client/platform/desktop/frontend/components/ViewerLoader.vue

@@ -9,7 +9,7 @@ import {
 import { ANNOTATION_SOURCE_QUERY } from 'dive-common/scoring/viewerNavigation';
 import { parseViewerFocus } from 'dive-common/review/viewerNavigation';
 import { useRoute, useRouter } from 'vue-router/composables';
-import Viewer from 'dive-common/components/Viewer.vue';
+import Viewer, { StereoViewLinkParams } from 'dive-common/components/Viewer.vue';
 import RunPipelineMenu from 'dive-common/components/RunPipelineMenu.vue';
 import ImportAnnotations from 'dive-common//components/ImportAnnotations.vue';
 import CalibrationMenu from 'dive-common/components/CalibrationMenu.vue';
@@ -1508,6 +1508,36 @@ export default defineComponent({
      *   result status instead — used by bulk import "Warp to All" so failures
      *   can be aggregated rather than cleared by the next job.
      */
+    /**
+     * Where `point` on `camera` lands on the other stereo camera, for linked
+     * panning. Uses whatever matcher the stereo service loaded; null when the
+     * service is off or the match is rejected.
+     */
+    async function stereoViewLink(params: StereoViewLinkParams): Promise<[number, number] | null> {
+      if (!stereoEnabled.value) return null;
+      const cameras = Object.keys(stereoImagePathGetters.value);
+      if (cameras.length !== 2 || !cameras.includes(params.camera)) return null;
+      if (!(await ensureStereoFrame(params.frameNum))) return null;
+      const fps = stereoCameraFps.value[cameras[0]] || stereoDatasetFps || Object.values(stereoCameraFps.value)[0];
+      try {
+        const response = await stereoTransferPoints({
+          points: [params.point],
+          strict: true,
+          sourceCamera: params.camera === cameras[0] ? 'left' : 'right',
+          leftImagePath: stereoImagePathGetters.value[cameras[0]](params.frameNum),
+          rightImagePath: stereoImagePathGetters.value[cameras[1]](params.frameNum),
+          frameTime: fps ? params.frameNum / fps : undefined,
+        });
+        const point = response.transferredPoints?.[0];
+        if (!response.success || response.validMatches?.[0] !== true || !point?.every(Number.isFinite)) {
+          return null;
+        }
+        return [point[0], point[1]];
+      } catch {
+        return null;
+      }
+    }
+
     async function handleStereoAnnotationComplete(
       params: StereoAnnotationCompleteParams,
       forceAutoCompute = false,
@@ -2188,6 +2218,7 @@ export default defineComponent({
       stereoLengthMessage,
       closeStereoLoadingDialog,
       handleStereoAnnotationComplete,
+      stereoViewLink,
       handleStereoWarpImported,
       handleStereoAnnotationReset,
       handleStereoSegmentationFinalize,
@@ -2215,6 +2246,7 @@ export default defineComponent({
       :initial-track-id="viewerFocus.trackId"
       :text-query-enabled="true"
       :text-query-available="textQueryAvailable"
+      :stereo-view-link="stereoViewLink"
       @return-to-current-annotations="returnToCurrentAnnotations"
       @change-camera="changeCamera"
       @large-image-warning="largeImageWarning()"

client/platform/web-girder/useStereoOnnxWeb.ts

@@ -424,10 +424,16 @@ export default function useStereoOnnxWeb(opts: StereoOnnxWebOptions) {
     return getTransfer()?.warpAllFromCamera(cameraName);
   }
 
+  async function stereoViewLink(params: { camera: string; frameNum: number; point: [number, number] }) {
+    if (!clientSettings.stereoSettings.autoComputeOtherCamera) return null;
+    return (await getTransfer()?.warpPoint(params.point, params.camera, params.frameNum)) ?? null;
+  }
+
   return {
     handleStereoAnnotationComplete,
     handleStereoTrackLinked,
     warpAllFromCamera,
+    stereoViewLink,
     precomputeCurrentFrame,
     invalidateCalibration,
   };

client/platform/web-girder/views/ViewerLoader.vue

@@ -177,7 +177,7 @@ export default defineComponent({
 
     const {
       handleStereoAnnotationComplete, handleStereoTrackLinked, warpAllFromCamera,
-      invalidateCalibration,
+      invalidateCalibration, stereoViewLink,
     } = useStereoOnnxWeb({
       getViewer: () => viewerRef.value,
       getDatasetId: () => parentDatasetId(props.id),
@@ -518,6 +518,7 @@ export default defineComponent({
       exportDatasetIds,
       handleStereoAnnotationComplete,
       handleStereoTrackLinked,
+      stereoViewLink,
       stereoBusyMessage,
       stereoDownloadProgress,
       stereoDownloadPercent,
@@ -550,6 +551,7 @@ export default defineComponent({
       :annotation-source-returnable="annotationSourceReturnable"
       :initial-frame="viewerFocus.frame"
       :initial-track-id="viewerFocus.trackId"
+      :stereo-view-link="stereoViewLink"
       @return-to-current-annotations="returnToCurrentAnnotations"
       @large-image-warning="largeImageWarning()"
       @update:set="routeSet"

client/src/components/annotators/mediaControllerType.ts

@@ -75,6 +75,15 @@ export interface AggregateMediaController {
   setSpeed: (speed: number) => void;
   getController: (cameraName: string) => MediaController;
   toggleSynchronizeCameras: (sync: boolean) => void;
+  /**
+   * While cameras are synchronised, recentre the other panes on where the
+   * source pane's centre lands on their camera (null to disable). The
+   * resolver returns null when the point cannot be found there.
+   */
+  setViewLinkResolver: (
+    resolver: ((sourceCamera: string, point: [number, number]) =>
+      Promise<[number, number] | null>) | null,
+  ) => void;
   /**
    * Seeks so that `camera` lands on its own local frame `localFrame` (e.g.
    * jumping to a track's stored begin/end, which is in local-frame units).

client/src/components/annotators/useMediaController.ts

@@ -7,6 +7,7 @@ import Vue, {
   ref, shallowRef, reactive, provide, toRef, Ref, UnwrapRef, computed, watch,
 } from 'vue';
 import { map, over } from 'lodash';
+import createViewLink from './viewLink';
 
 import { use } from '../../provides';
 import type {
@@ -113,6 +114,23 @@ export function useMediaController() {
   let state: Record<string, UnwrapRef<MediaControllerReactiveData>> = {};
   let cameraControllerSymbols: Record<string, symbol> = {};
   const synchronizeCameras: Ref<boolean> = ref(false);
+
+  // Installed by the viewer while stereo auto-compute is on; otherwise
+  // synchronised panes only copy each other's screen motion.
+  const viewLink = createViewLink({
+    center: (key) => geoViewers[key]?.value?.center(),
+    cameraName: (key) => state[key]?.cameraName,
+    synced: () => synchronizeCameras.value,
+    recenter: (sourceKey, point) => {
+      allowCameraTrigger = false;
+      Object.entries(geoViewers).forEach(([camera, geoViewer]) => {
+        if (geoViewer.value && camera !== sourceKey) {
+          geoViewer.value.center({ x: point[0], y: point[1] });
+        }
+      });
+      allowCameraTrigger = true;
+    },
+  });
   const resizeTrigger: Ref<number> = ref(0);
   // Raised only while onResize applies its programmatic resetZoom, so the
   // linked-viewer navigation ignores the resulting pan/zoom events (see
@@ -150,6 +168,7 @@ export function useMediaController() {
     currentTime: emptyControllerCurrentTime,
     getController,
     toggleSynchronizeCameras,
+    setViewLinkResolver,
     cameraSync: synchronizeCameras,
     resizeTrigger,
     resizing,
@@ -287,6 +306,12 @@ export function useMediaController() {
     synchronizeCameras.value = val;
   }
 
+  function setViewLinkResolver(
+    resolver: Parameters<typeof viewLink.setResolver>[0],
+  ) {
+    viewLink.setResolver(resolver);
+  }
+
   /**
    * Optional replacement for the aggregate "reset pan and zoom" behavior,
    * installed by the aligned-view navigation link (useAlignedNavigation).
@@ -320,6 +345,7 @@ export function useMediaController() {
         }
       });
       allowCameraTrigger = true;
+      viewLink.schedule(camEvent.camera);
     }
   });
 
@@ -333,6 +359,7 @@ export function useMediaController() {
         }
       });
       allowCameraTrigger = true;
+      viewLink.schedule(camEvent.camera);
     }
   });
   /**
@@ -654,6 +681,7 @@ export function useMediaController() {
       getController,
       resetMapDimensions,
       toggleSynchronizeCameras,
+      setViewLinkResolver,
       cameraSync: synchronizeCameras,
       resizeTrigger,
       resizing,
@@ -820,6 +848,7 @@ export function useMediaController() {
       currentTime: defaultController.currentTime,
       getController,
       toggleSynchronizeCameras,
+      setViewLinkResolver,
       cameraSync: synchronizeCameras,
       resizeTrigger,
       resizing,

client/src/components/annotators/viewLink.spec.ts

@@ -0,0 +1,87 @@
+import createViewLink from './viewLink';
+
+type Resolver = (camera: string, point: [number, number]) => Promise<[number, number] | null>;
+
+function harness(resolver: Resolver = vi.fn(async () => [30, 40] as [number, number])) {
+  const recenter = vi.fn();
+  let synced = true;
+  const link = createViewLink({
+    center: (key) => (key === 'A' ? { x: 10, y: 20 } : undefined),
+    cameraName: (key) => ({ A: 'left', B: 'right' } as Record<string, string>)[key],
+    recenter,
+    synced: () => synced,
+    delayMs: 100,
+  });
+  link.setResolver(resolver);
+  return {
+    link, recenter, resolver, setSynced: (v: boolean) => { synced = v; },
+  };
+}
+
+const settle = async () => { await vi.runAllTimersAsync(); };
+
+beforeEach(() => vi.useFakeTimers());
+afterEach(() => vi.useRealTimers());
+
+it('recentres the other pane on where the source centre lands', async () => {
+  const h = harness();
+  h.link.schedule('A');
+  await settle();
+  expect(h.resolver).toHaveBeenCalledWith('left', [10, 20]);
+  expect(h.recenter).toHaveBeenCalledWith('A', [30, 40]);
+});
+
+it('issues one lookup per settled drag', async () => {
+  const h = harness();
+  h.link.schedule('A'); vi.advanceTimersByTime(50);
+  h.link.schedule('A'); vi.advanceTimersByTime(50);
+  h.link.schedule('A');
+  await settle();
+  expect(h.resolver).toHaveBeenCalledTimes(1);
+});
+
+it('leaves the plain sync alone when the point is not found or the lookup throws', async () => {
+  const miss = harness(vi.fn(async () => null));
+  miss.link.schedule('A'); await settle();
+  expect(miss.recenter).not.toHaveBeenCalled();
+  const boom = harness(vi.fn(async () => { throw new Error('no model'); }));
+  boom.link.schedule('A'); await settle();
+  expect(boom.recenter).not.toHaveBeenCalled();
+});
+
+it('drops a lookup overtaken by a newer one or finished after sync was turned off', async () => {
+  let release: (p: [number, number]) => void = () => undefined;
+  const slow: Resolver = () => new Promise<[number, number]>((resolve) => { release = resolve; });
+  const h = harness(slow);
+  h.link.schedule('A'); vi.advanceTimersByTime(100);
+  const firstRelease = release;
+  h.link.schedule('A'); vi.advanceTimersByTime(100);
+  firstRelease([1, 1]); await Promise.resolve(); await Promise.resolve();
+  expect(h.recenter).not.toHaveBeenCalled();
+  h.setSynced(false);
+  release([2, 2]); await Promise.resolve(); await Promise.resolve();
+  expect(h.recenter).not.toHaveBeenCalled();
+});
+
+it('does nothing without a resolver or for a pane without a map', async () => {
+  const h = harness();
+  h.link.schedule('B'); await settle();
+  expect(h.resolver).not.toHaveBeenCalled();
+  h.link.setResolver(null);
+  h.link.schedule('A'); await settle();
+  expect(h.resolver).not.toHaveBeenCalled();
+});
+
+it.each(['disable', 'replace', 'pan'] as const)('drops an active lookup immediately on %s', async (action) => {
+  let release!: (point: [number, number]) => void;
+  const h = harness(() => new Promise((resolve) => { release = resolve; }));
+  h.link.schedule('A');
+  vi.advanceTimersByTime(100);
+  if (action === 'disable') h.link.setResolver(null);
+  if (action === 'replace') h.link.setResolver(async () => [50, 60]);
+  if (action === 'pan') h.link.schedule('A');
+  // The old result must be discarded even before the next debounce expires.
+  release([1, 2]);
+  await Promise.resolve();
+  expect(h.recenter).not.toHaveBeenCalled();
+});