diff --git a/doc/changelog.d/124.added.md b/doc/changelog.d/124.added.md new file mode 100644 index 00000000..20edf4d6 --- /dev/null +++ b/doc/changelog.d/124.added.md @@ -0,0 +1 @@ +[Remote rendering 3.2c] attribute camera changes to the input that caused them diff --git a/src/ansys/visor/visor-client/src/jest-tests/CameraGestureTracker.test.js b/src/ansys/visor/visor-client/src/jest-tests/CameraGestureTracker.test.js new file mode 100644 index 00000000..5d777c56 --- /dev/null +++ b/src/ansys/visor/visor-client/src/jest-tests/CameraGestureTracker.test.js @@ -0,0 +1,302 @@ +/** + * Camera settle debounce and its origin capture. + * + * The tracker is tested directly rather than through VtkScene: VtkScene's + * construction awaits four wasm proxies and starts a requestAnimationFrame + * loop, so a fake large enough to build one would be what the tests measured. + * + * The debounce is 300 ms. That literal is written out in every assertion here + * and the production constant is deliberately not imported, so that changing + * it fails these tests instead of silently redefining what they assert. + */ +import CameraGestureTracker from '../wasm/CameraGestureTracker.js'; + +describe('CameraGestureTracker', () => { + /**@type{HTMLElement}*/ + let canvasDiv; + /**@type{HTMLElement}*/ + let canvas; + /**@type{CameraGestureTracker}*/ + let tracker; + /**@type{jest.Mock}*/ + let onSettled; + + beforeEach(() => { + jest.useFakeTimers(); + canvasDiv = document.createElement('div'); + canvas = document.createElement('canvas'); + canvasDiv.appendChild(canvas); + document.body.appendChild(canvasDiv); + tracker = new CameraGestureTracker(canvasDiv, canvas); + onSettled = jest.fn(); + tracker.addSettledListener(onSettled); + }); + + afterEach(() => { + tracker.dispose(); + document.body.removeChild(canvasDiv); + jest.useRealTimers(); + }); + + const mouseDown = (button) => + canvasDiv.dispatchEvent(new MouseEvent('mousedown', { button, bubbles: true })); + const mouseUp = (button) => + canvasDiv.dispatchEvent(new MouseEvent('mouseup', { button, bubbles: true })); + const mouseOut = () => canvasDiv.dispatchEvent(new MouseEvent('mouseout', { bubbles: true })); + const mouseMove = () => canvasDiv.dispatchEvent(new MouseEvent('mousemove', { bubbles: true })); + const wheel = () => canvas.dispatchEvent(new Event('wheel')); + const keyUp = (key) => window.dispatchEvent(new KeyboardEvent('keyup', { key })); + const blur = () => window.dispatchEvent(new Event('blur')); + + // ---- the debounce ------------------------------------------------------ + + test('no report at 299 ms and exactly one report at 300 ms', () => { + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(299); + expect(onSettled).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(1); + expect(onSettled).toHaveBeenCalledTimes(1); + }); + + test('a second event before the deadline restarts the debounce', () => { + tracker.noteCameraEvent(); + jest.advanceTimersByTime(200); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(299); + expect(onSettled).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(1); + expect(onSettled).toHaveBeenCalledTimes(1); + }); + + // ---- origin, captured at event time ------------------------------------ + + test('an event with no input reports programmatic', () => { + tracker.noteCameraEvent(); + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledTimes(1); + expect(onSettled).toHaveBeenCalledWith('programmatic'); + }); + + test('an event while a button is held reports gesture even though the button is released before the deadline', () => { + mouseDown(0); + tracker.noteCameraEvent(); + mouseUp(0); + + jest.advanceTimersByTime(300); + + // An origin computed when the timer fires would say 'programmatic' + // here, because by then the button is up. + expect(onSettled).toHaveBeenCalledTimes(1); + expect(onSettled).toHaveBeenCalledWith('gesture'); + }); + + test('one gesture event makes the whole window gesture', () => { + mouseDown(0); + tracker.noteCameraEvent(); + mouseUp(0); + jest.advanceTimersByTime(100); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledTimes(1); + expect(onSettled).toHaveBeenCalledWith('gesture'); + }); + + // ---- what counts as input ---------------------------------------------- + + test('pointer movement with no button held is not input', () => { + mouseMove(); + tracker.noteCameraEvent(); + mouseMove(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledWith('programmatic'); + }); + + test('a mouseout ends the hold', () => { + mouseDown(2); + mouseOut(); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledWith('programmatic'); + }); + + test('a window blur ends the hold', () => { + mouseDown(1); + blur(); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledWith('programmatic'); + }); + + test('an event within 300 ms of a wheel reports gesture', () => { + wheel(); + jest.advanceTimersByTime(299); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledWith('gesture'); + }); + + test('an event 300 ms after a wheel reports programmatic', () => { + wheel(); + jest.advanceTimersByTime(300); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledWith('programmatic'); + }); + + test('an event within 300 ms of a z keyup reports gesture', () => { + keyUp('z'); + jest.advanceTimersByTime(299); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledWith('gesture'); + }); + + test('an event within 300 ms of an r keyup reports gesture', () => { + keyUp('r'); + jest.advanceTimersByTime(299); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledWith('gesture'); + }); + + test('a keyup that is not z or r does not arm input', () => { + keyUp('a'); + tracker.noteCameraEvent(); + + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledWith('programmatic'); + }); + + // ---- listener management and teardown ---------------------------------- + + test('the remover returned by addSettledListener stops reports', () => { + const second = jest.fn(); + const remove = tracker.addSettledListener(second); + remove(); + + tracker.noteCameraEvent(); + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledTimes(1); + expect(second).not.toHaveBeenCalled(); + }); + + test('a settle armed before teardown produces no report', () => { + tracker.noteCameraEvent(); + tracker.dispose(); + + jest.advanceTimersByTime(300); + + expect(onSettled).not.toHaveBeenCalled(); + }); + + test('after teardown a further event produces no report', () => { + tracker.dispose(); + + tracker.noteCameraEvent(); + jest.advanceTimersByTime(300); + + expect(onSettled).not.toHaveBeenCalled(); + }); +}); + +/** + * The wasm canvas's own wheel handler and VtkScene's window keyup handler are + * registered before the tracker is constructed, so the camera event raised by a + * single wheel notch or a single z/r press reaches noteCameraEvent() before the + * tracker's own handler has marked input active. + * + * Each test here reproduces that order exactly: a stand-in listener is + * registered on the same target *before* the tracker exists, and calls + * noteCameraEvent() synchronously from inside its own handler. + */ +describe('CameraGestureTracker, when the camera event precedes the tracker handler', () => { + /**@type{HTMLElement}*/ + let canvasDiv; + /**@type{HTMLElement}*/ + let canvas; + /**@type{CameraGestureTracker|null}*/ + let tracker; + /**@type{jest.Mock}*/ + let onSettled; + /**@type{Array<()=>void>}*/ + let standIns; + + beforeEach(() => { + jest.useFakeTimers(); + canvasDiv = document.createElement('div'); + canvas = document.createElement('canvas'); + canvasDiv.appendChild(canvas); + document.body.appendChild(canvasDiv); + tracker = null; + onSettled = jest.fn(); + standIns = []; + }); + + afterEach(() => { + for (const remove of standIns) { + remove(); + } + if (tracker != null) { + tracker.dispose(); + } + document.body.removeChild(canvasDiv); + jest.useRealTimers(); + }); + + /** + * Registers a handler that raises one camera event, before the tracker is + * constructed, so that it runs first. + */ + const standInBefore = (target, type) => { + const handler = () => tracker.noteCameraEvent(); + target.addEventListener(type, handler); + standIns.push(() => target.removeEventListener(type, handler)); + }; + + test('a single wheel event reports gesture', () => { + standInBefore(canvas, 'wheel'); + tracker = new CameraGestureTracker(canvasDiv, canvas); + tracker.addSettledListener(onSettled); + + canvas.dispatchEvent(new Event('wheel')); + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledTimes(1); + expect(onSettled).toHaveBeenCalledWith('gesture'); + }); + + test('a single r keyup reports gesture', () => { + standInBefore(window, 'keyup'); + tracker = new CameraGestureTracker(canvasDiv, canvas); + tracker.addSettledListener(onSettled); + + window.dispatchEvent(new KeyboardEvent('keyup', { key: 'r' })); + jest.advanceTimersByTime(300); + + expect(onSettled).toHaveBeenCalledTimes(1); + expect(onSettled).toHaveBeenCalledWith('gesture'); + }); +}); diff --git a/src/ansys/visor/visor-client/src/renderer/IRenderer.ts b/src/ansys/visor/visor-client/src/renderer/IRenderer.ts index d490b024..6cb94001 100644 --- a/src/ansys/visor/visor-client/src/renderer/IRenderer.ts +++ b/src/ansys/visor/visor-client/src/renderer/IRenderer.ts @@ -36,6 +36,13 @@ export type AppliedCameraState = Readonly<{ parallelScale: number; }>; +/** + * Where a settled camera change came from: `gesture` if user input was + * active during the change, `programmatic` otherwise. + * See `wasm/CameraGestureTracker.js`. + */ +export type CameraOrigin = 'gesture' | 'programmatic'; + /** Descriptor consumed by setColorVariableAsync. */ export type ColorVariableDescriptor = Readonly<{ spectrumId: string; @@ -119,6 +126,14 @@ export interface IRenderer { // ---- Subscriptions (return unsubscribe closures) ------------------------ /** Fires whenever the camera changes. Callback receives a full snapshot. */ addCameraChangedListener(callback: (state: VisorCameraState) => void): () => void; + /** + * Fires once per gesture, after the last camera change has stopped changing for + * CAMERA_SETTLE_MS, with the origin of that change. The callback receives + * the origin only; read the camera with getCameraStateAsync if needed. + * A `gesture` report is the user's own camera; a `programmatic` one is a + * camera the application or the server applied. + */ + addCameraSettledListener(callback: (origin: CameraOrigin) => void): () => void; /** Fires each frame with the current FPS. */ addFrameRenderedListener(callback: (fps: number) => void): () => void; /** diff --git a/src/ansys/visor/visor-client/src/renderer/NullRenderer.ts b/src/ansys/visor/visor-client/src/renderer/NullRenderer.ts index 9955a1d2..78c88f4d 100644 --- a/src/ansys/visor/visor-client/src/renderer/NullRenderer.ts +++ b/src/ansys/visor/visor-client/src/renderer/NullRenderer.ts @@ -1,5 +1,6 @@ import { AppliedCameraState, + CameraOrigin, ColorVariableDescriptor, GeometryPickMode, IRenderer, @@ -94,6 +95,12 @@ export class NullRenderer implements IRenderer { }; } + addCameraSettledListener(_callback: (origin: CameraOrigin) => void): () => void { + return () => { + // no-op unsubscribe + }; + } + addFrameRenderedListener(_callback: (fps: number) => void): () => void { return () => { // no-op unsubscribe diff --git a/src/ansys/visor/visor-client/src/renderer/WasmRenderer.ts b/src/ansys/visor/visor-client/src/renderer/WasmRenderer.ts index d9d75804..d4471d8b 100644 --- a/src/ansys/visor/visor-client/src/renderer/WasmRenderer.ts +++ b/src/ansys/visor/visor-client/src/renderer/WasmRenderer.ts @@ -1,5 +1,6 @@ import { AppliedCameraState, + CameraOrigin, ColorVariableDescriptor, GeometryPickMode, IRenderer, @@ -205,6 +206,10 @@ export class WasmRenderer implements IRenderer { }); } + addCameraSettledListener(callback: (origin: CameraOrigin) => void): () => void { + return this.#vtkScene.addCameraSettledListener(callback); + } + addFrameRenderedListener(callback: (fps: number) => void): () => void { return this.#vtkScene.addFrameRenderedListener(async (fps) => { callback(fps); diff --git a/src/ansys/visor/visor-client/src/wasm/CameraGestureTracker.js b/src/ansys/visor/visor-client/src/wasm/CameraGestureTracker.js new file mode 100644 index 00000000..ba9536b6 --- /dev/null +++ b/src/ansys/visor/visor-client/src/wasm/CameraGestureTracker.js @@ -0,0 +1,239 @@ +/** + * @desc Debounces raw camera `ModifiedEvent`s into a single "settled" report, + * and tags each report to `gesture` or `programmatic`. + * + * Called from the camera `ModifiedEvent` handler in `VtkScene.#setupCamera`, + * not through `addCameraChangedListener`, since that wrapper reads the whole + * wasm camera back on every intermediate event. + * + * - A drag or zoom raises many camera events in a row. Each one restarts a + * timer. When CAMERA_SETTLE_MS pass with no new event, the gesture is taken + * to be over and one report is sent. + * + * - The report is tagged `gesture` if the user was providing input while the + * events were coming in, and `programmatic` if not (for example, a camera the + * server pushed). "Providing input" means a mouse button is held on the + * canvas, or a wheel event or z/r keyup happened within the last CAMERA_SETTLE_MS. + * + * - That input check runs on every event as it arrives, and the result is + * remembered until the report. It cannot run when the timer fires, + * because by then the user has let go of the mouse and every drag would + * look programmatic. + * + * - The wasm wheel handler and `VtkScene`'s keyup handler run before this + * tracker's own listeners. So for a single wheel notch or a single z/r press, + * the camera event can arrive before the tracker has noticed the input. + * To cover that, a wheel event or a z/r keyup arriving while a report is + * pending marks that report `gesture` as well. + */ + +/** + * The settle debounce, in milliseconds, and equally the window during which a + * wheel event or a z/r keyup counts as active input. + * + * Tests pin the literal 300 and must not import this constant, so that changing + * it here fails a test rather than silently redefining what the tests assert. + * + * @type{number} + */ +export const CAMERA_SETTLE_MS = 300; + +/** + * Keys whose keyup arms the input window. + * + * Kept in step with the `z` / `r` cases of `VtkScene.#setupCamera`'s window + * `keyup` handler, which is what actually mutates the camera. If a key is + * added there, add it here. + * + * @type{ReadonlyArray} + */ +const CAMERA_KEYS = ['z', 'r']; + +export default class CameraGestureTracker { + /** + * Attaches its own input listeners. Every one of them is removed by + * `dispose()`. + * + * Mouse buttons are tracked on `canvasDiv`, in the capture phase, which is + * where the real user events land. They are deliberately *not* tracked on + * `canvas`: `VtkScene.#setupCamera`'s `applyMouseEvent` dispatches three + * synthetic `mouseup`s and a synthetic `mousedown` onto `canvas` for every + * real button event, so a canvas-side tracker would see releases that + * never happened. + * + * The wheel is tracked on `canvas`, which the synthetic mouse traffic does + * not touch. + * + * @param {HTMLElement} canvasDiv - the container that receives real mouse events + * @param {HTMLElement} canvas - the wasm render canvas + */ + constructor(canvasDiv, canvas) { + const onMouseDown = /**@param {MouseEvent} e*/ (e) => { + this.#heldButtons.add(e.button); + }; + const onMouseUp = /**@param {MouseEvent} e*/ (e) => { + this.#heldButtons.delete(e.button); + }; + // A `mouseout` is the existing sticky-mousedown release, and a window + // `blur` means the page no longer owns the input. Both clear *every* + // held button rather than only the one this event names: a MouseEvent + // for `mouseout` carries button 0, so clearing per-button would leave a + // middle- or right-drag latched as "input active" forever, and every + // later programmatic apply would be reported as a gesture. + const onInputLost = () => { + this.#heldButtons.clear(); + }; + const onWheel = () => { + this.#markImpulse(); + }; + const onKeyUp = /**@param {KeyboardEvent} e*/ (e) => { + if (e.key != null && CAMERA_KEYS.includes(e.key.toLowerCase())) { + this.#markImpulse(); + } + }; + + this.#addListener(canvasDiv, 'mousedown', onMouseDown, true); + this.#addListener(canvasDiv, 'mouseup', onMouseUp, true); + this.#addListener(canvasDiv, 'mouseout', onInputLost, true); + this.#addListener(canvas, 'wheel', onWheel, { passive: true }); + this.#addListener(window, 'keyup', onKeyUp, false); + this.#addListener(window, 'blur', onInputLost, false); + } + + /**@type{Array<()=>void>}*/ + #listenerRemovers = []; + /** + * Mouse buttons currently held down, by `MouseEvent.button` number. A Set + * rather than three booleans, so buttons 3 and 4 are handled the same way. + * @type{Set} + */ + #heldButtons = new Set(); + /**@type{boolean}*/ + #impulseActive = false; + /**@type{*}*/ + #impulseTimer = null; + /**@type{*}*/ + #settleTimer = null; + /** + * Whether any event in the pending settle window was a gesture. This is + * the whole of the "origin at event time" mechanism. + * @type{boolean} + */ + #sawGesture = false; + /**@type{boolean}*/ + #disposed = false; + /**@type{Mapvoid>}*/ + #settledListeners = new Map(); + + /** + * @param {EventTarget} target + * @param {string} type + * @param {(e:any)=>void} handler + * @param {boolean|AddEventListenerOptions} options + */ + #addListener = (target, type, handler, options) => { + target.addEventListener(type, handler, options); + this.#listenerRemovers.push(() => target.removeEventListener(type, handler, options)); + }; + + /** + * A wheel notch or a z/r press. Arms the input window for CAMERA_SETTLE_MS, + * and, because the camera event these raise may already have been noted + * before this handler ran, retroactively marks a pending settle window as + * a gesture. + */ + #markImpulse = () => { + if (this.#disposed) { + return; + } + this.#impulseActive = true; + if (this.#impulseTimer != null) { + clearTimeout(this.#impulseTimer); + } + this.#impulseTimer = setTimeout(() => { + this.#impulseTimer = null; + this.#impulseActive = false; + }, CAMERA_SETTLE_MS); + if (this.#settleTimer != null) { + this.#sawGesture = true; + } + }; + + /** + * @return {boolean} whether user input is active right now. + */ + #isInputActive = () => { + return this.#heldButtons.size > 0 || this.#impulseActive; + }; + + /** + * Called from the `ModifiedEvent` fan-out, once per raw camera event. + * Records this event's origin immediately, then restarts the debounce. + * @return {void} + */ + noteCameraEvent = () => { + if (this.#disposed) { + return; + } + if (this.#isInputActive()) { + this.#sawGesture = true; + } + if (this.#settleTimer != null) { + clearTimeout(this.#settleTimer); + } + this.#settleTimer = setTimeout(this.#reportSettled, CAMERA_SETTLE_MS); + }; + + /** + * @return {void} + */ + #reportSettled = () => { + const origin = this.#sawGesture ? 'gesture' : 'programmatic'; + this.#settleTimer = null; + this.#sawGesture = false; + for (const callback of this.#settledListeners.values()) { + callback(origin); + } + }; + + /** + * @param {(origin:'gesture'|'programmatic')=>void} handler + * @return {()=>void} a remover + */ + addSettledListener = (handler) => { + const remover = () => this.#settledListeners.delete(remover); + this.#settledListeners.set(remover, handler); + return remover; + }; + + /** + * Teardown. Clears the settled-listener map, the pending settle timer and + * the window's recorded origins, resets the input state, and removes every + * listener this class added. Idempotent. + * + * A settle armed before a scene rebuild must not fire afterwards: it would + * report a camera belonging to the previous scene, tagged as a user + * gesture, and the server would record it. + * + * @return {void} + */ + dispose = () => { + this.#disposed = true; + if (this.#settleTimer != null) { + clearTimeout(this.#settleTimer); + this.#settleTimer = null; + } + if (this.#impulseTimer != null) { + clearTimeout(this.#impulseTimer); + this.#impulseTimer = null; + } + this.#sawGesture = false; + this.#impulseActive = false; + this.#heldButtons.clear(); + this.#settledListeners.clear(); + for (const remover of this.#listenerRemovers) { + remover(); + } + this.#listenerRemovers = []; + }; +} diff --git a/src/ansys/visor/visor-client/src/wasm/VtkScene.js b/src/ansys/visor/visor-client/src/wasm/VtkScene.js index 4449e3d6..e8e95acf 100644 --- a/src/ansys/visor/visor-client/src/wasm/VtkScene.js +++ b/src/ansys/visor/visor-client/src/wasm/VtkScene.js @@ -5,6 +5,8 @@ * provided to the constructor to initialize the scene from an existing * remote one. */ +import CameraGestureTracker from './CameraGestureTracker.js'; + export default class VtkScene { /** * @private @@ -125,7 +127,10 @@ export default class VtkScene { this.#cameraChangedListeners.clear(); this.#viewerClickedListeners.clear(); this.#frameRenderedListeners.clear(); + this.#cameraGestureTracker?.dispose(); }; + /**@type{CameraGestureTracker|null}*/ + #cameraGestureTracker = null; /**@type{Mapvoid>}*/ #userObserverRemovers = new Map(); /**@type{Mapvoid>}*/ @@ -143,6 +148,17 @@ export default class VtkScene { this.#cameraChangedListeners.set(remover, handler); return remover; }; + /** + * Fires once per settle, CAMERA_SETTLE_MS after the last camera event, + * with the origin of that window: `gesture` if any event in the window occurred while user + * input was active, `programmatic` otherwise. See CameraGestureTracker. + * + * @param {(origin:'gesture'|'programmatic')=>void} handler + * @return {()=>void} a remover + */ + addCameraSettledListener = (handler) => { + return this.#cameraGestureTracker.addSettledListener(handler); + }; /** * @param {(actorId:number,ctrlKey:boolean,shiftKey:boolean,normX:number,normY:number)=>void} handler * @return {()=>void} @@ -387,6 +403,9 @@ export default class VtkScene { for (const callback of cameraChangedListeners.values()) { callback(camera); } + // Called directly rather than via addCameraChangedListener, + // which reads the whole wasm camera back on every event. + this.#cameraGestureTracker?.noteCameraEvent(); }); /**@type{boolean}*/ @@ -479,6 +498,9 @@ export default class VtkScene { // TODO: need to remove this event listener when the user disposes the WasmView object window.addEventListener('keyup', async (e) => { switch (e.key.toLowerCase()) { + // NOTE: the z/r key list is mirrored in CameraGestureTracker, + // which treats a keyup on either as active user input. Adding + // a camera-mutating key here means adding it there too. case 'z': await renderer.ResetCamera(); await renderWindow.Render(); @@ -506,6 +528,7 @@ export default class VtkScene { }, true ); + this.#cameraGestureTracker = new CameraGestureTracker(canvasDiv, canvas); }; #setupFpsMonitor = () => {