diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor b/Features/PhysicalAssets/LiveCameraScanner.razor index b1a4014..48b4a6d 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor +++ b/Features/PhysicalAssets/LiveCameraScanner.razor @@ -30,10 +30,29 @@ @if (!_isStreaming && !_isFrozen) {
-

Camera is inactive.

+ @if (_support is { HasMediaDevices: false }) + { + @* + Safari withholds the camera API entirely on a plain http:// + origin, so there is nothing here for a button to ask for. + Say what would make it work instead of offering a dead tap. + *@ +

This browser will not share a camera over an insecure connection.

+

+ Open Money Mirror over https://, or on localhost, + and the camera becomes available. +

+ } + else + { +

Camera is inactive.

+

+ Your browser will ask for permission — choose Allow to start scanning. +

+ }
} @@ -82,7 +101,32 @@ @if (_errorMessage is not null) { - + }
@@ -151,22 +195,67 @@ private bool _isModelReady; private string? _engineDevice; private string? _errorMessage; + private string? _errorCode; private string? _statusHint; + private CameraSupport? _support; private SamSegmentationResult? _lastResult; private string _cameraFacing = FacingEnvironment; private SamPromptType _currentPromptMode = SamPromptType.Positive; private readonly List _prompts = new(); + /// + /// Asks the browser what it is willing to do before the person taps anything, + /// so an origin with no camera API can say so rather than offer a button that + /// cannot work. + /// + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender) + { + return; + } + + _cameraModule ??= await JS.InvokeAsync("import", CameraModulePath); + _support = await _cameraModule.InvokeAsync("describeSupport"); + StateHasChanged(); + } + public async Task StartCameraAsync() { _errorMessage = null; + _errorCode = null; try { _cameraModule ??= await JS.InvokeAsync("import", CameraModulePath); - await _cameraModule.InvokeVoidAsync("startCamera", _video, _cameraFacing); + var result = await _cameraModule.InvokeAsync( + "openCamera", + _video, + _cameraFacing + ); + + if (!result.Ok) + { + _errorCode = result.Code; + _errorMessage = result.Message; + Logger.LogWarning( + "Camera could not be started ({Code}): {Message}", + result.Code, + result.Message + ); + return; + } + + _cameraFacing = result.Facing ?? _cameraFacing; _isStreaming = true; _isFrozen = false; + // The model is ~40 MB and has nothing to do with which camera is + // running, so a second start should not fetch it again. + if (_isModelReady) + { + return; + } + _statusHint = "Downloading MobileSAM model (~40 MB)..."; StateHasChanged(); @@ -197,6 +286,7 @@ public async Task SwitchCameraAsync() { _errorMessage = null; + _errorCode = null; if (_cameraModule is null) { @@ -206,8 +296,25 @@ try { var nextFacing = _cameraFacing == FacingEnvironment ? FacingUser : FacingEnvironment; - await _cameraModule.InvokeVoidAsync("switchCamera", _video, _cameraFacing); - _cameraFacing = nextFacing; + var result = await _cameraModule.InvokeAsync( + "flipCamera", + _video, + _cameraFacing + ); + + if (!result.Ok) + { + // Flipping tears the old stream down before asking for the new + // one, so a refusal here leaves no feed at all - worth explaining + // rather than leaving as a blank stage. + _errorCode = result.Code; + _errorMessage = result.Message; + _isStreaming = false; + _isFrozen = false; + return; + } + + _cameraFacing = result.Facing ?? nextFacing; } catch (Exception ex) { diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor.css b/Features/PhysicalAssets/LiveCameraScanner.razor.css index f60d1c8..81a54b9 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor.css +++ b/Features/PhysicalAssets/LiveCameraScanner.razor.css @@ -117,6 +117,32 @@ text-align: center; } +/* Sits on the dark viewfinder stage, so it borrows the placeholder's colour + and steps back from it rather than choosing one of its own. */ +.permission-help { + max-width: 24rem; + font-size: 0.875rem; + line-height: 1.5; + opacity: 0.85; +} + +.permission-help code { + font-size: 0.9em; +} + +/* Recovery steps live inside the denial alert, which is a light surface - these + inherit its colour, not the stage's. */ +.permission-steps { + margin: 0.5rem 0 0; + padding-left: 1.25rem; + font-size: 0.875rem; + line-height: 1.5; +} + +.permission-steps li + li { + margin-top: 0.25rem; +} + .action-bar { display: flex; flex-wrap: wrap; diff --git a/PhysicalAssets/CameraSupport.cs b/PhysicalAssets/CameraSupport.cs new file mode 100644 index 0000000..95798dd --- /dev/null +++ b/PhysicalAssets/CameraSupport.cs @@ -0,0 +1,25 @@ +namespace MoneyMirror.PhysicalAssets; + +/// +/// What the visitor's browser will allow before they tap anything, as reported +/// by describeSupport in camera-scanner.js. +/// +/// +/// False on a plain http:// origin. Safari withholds the camera API entirely +/// there, so the failure needs explaining rather than retrying. +/// +/// Whether navigator.mediaDevices.getUserMedia exists at all. +/// Whether to offer iOS-shaped recovery steps when permission is refused. +public record CameraSupport(bool SecureContext, bool HasMediaDevices, bool IsIos); + +/// +/// The outcome of asking the browser for a camera, as reported by +/// openCamera in camera-scanner.js. +/// +/// +/// The DOMException name - NotAllowedError for a refusal, +/// NotFoundError for a device with no camera, and so on. Interop would +/// drop this if the browser threw instead of reporting, and a refusal is the one +/// failure the person can actually do something about. +/// +public record CameraStartResult(bool Ok, string? Facing, string? Code, string? Message); diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs index 6304e03..94bcb66 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs @@ -22,9 +22,16 @@ public LiveCameraScannerTests() _context.Services.AddSingleton(_storage); _module = _context.JSInterop.SetupModule("./js/camera-scanner.js"); - _module.SetupVoid("startCamera", _ => true).SetVoidResult(); + _module + .Setup("describeSupport", _ => true) + .SetResult(new CameraSupport(SecureContext: true, HasMediaDevices: true, IsIos: true)); + _module + .Setup("openCamera", _ => true) + .SetResult(new CameraStartResult(Ok: true, Facing: "environment", null, null)); + _module + .Setup("flipCamera", _ => true) + .SetResult(new CameraStartResult(Ok: true, Facing: "user", null, null)); _module.SetupVoid("stopCamera", _ => true).SetVoidResult(); - _module.SetupVoid("switchCamera", _ => true).SetVoidResult(); _module.SetupVoid("freezeFrame", _ => true).SetVoidResult(); _module.SetupVoid("unfreezeFrame", _ => true).SetVoidResult(); _module @@ -50,7 +57,7 @@ public async Task StartCamera_InitializesEngineAndStream() await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); - _module.VerifyInvoke("startCamera", 1); + _module.VerifyInvoke("openCamera", 1); Assert.True(_samEngine.InitCalls >= 1); Assert.Contains("Camera active", cut.Markup); } @@ -128,7 +135,65 @@ public async Task SwitchCamera_CallsJsSwitchCamera() .Single(b => b.TextContent.Contains("Switch camera")); await switchButton.ClickAsync(new MouseEventArgs()); - _module.VerifyInvoke("switchCamera", 1); + _module.VerifyInvoke("flipCamera", 1); + } + + [Fact] + public async Task CameraRefused_ExplainsHowToGrantItBack() + { + using var context = new BunitContext(); + context.Services.AddLogging(); + context.Services.AddSingleton(new FakeSamEngine()); + context.Services.AddSingleton(new FakeStorage()); + + var module = context.JSInterop.SetupModule("./js/camera-scanner.js"); + module.SetupVoid("stopCamera", _ => true).SetVoidResult(); + module + .Setup("describeSupport", _ => true) + .SetResult(new CameraSupport(SecureContext: true, HasMediaDevices: true, IsIos: true)); + module + .Setup("openCamera", _ => true) + .SetResult( + new CameraStartResult( + Ok: false, + Facing: null, + Code: "NotAllowedError", + Message: "Camera access was denied." + ) + ); + + var cut = context.Render(); + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + + // Safari refuses again without prompting once it has been told no, so a + // second tap cannot help and the way back has to be spelled out. + Assert.Contains("Camera access was denied.", cut.Markup); + Assert.Contains("Website Settings", cut.Markup); + Assert.DoesNotContain("Camera active", cut.Markup); + } + + [Fact] + public void InsecureOrigin_OffersNoCameraButtonAtAll() + { + using var context = new BunitContext(); + context.Services.AddLogging(); + context.Services.AddSingleton(new FakeSamEngine()); + context.Services.AddSingleton(new FakeStorage()); + + var module = context.JSInterop.SetupModule("./js/camera-scanner.js"); + module.SetupVoid("stopCamera", _ => true).SetVoidResult(); + module + .Setup("describeSupport", _ => true) + .SetResult( + new CameraSupport(SecureContext: false, HasMediaDevices: false, IsIos: true) + ); + + var cut = context.Render(); + + // There is no camera API on a plain http:// origin, so a button could do + // nothing but fail. + Assert.DoesNotContain("Start live camera", cut.Markup); + Assert.Contains("insecure connection", cut.Markup); } [Fact] diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs index 7e99aa0..671c1f4 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs @@ -39,6 +39,12 @@ public VideoScanPickerTests() _context.Services.AddSingleton(_valuation); _context.Services.AddSingleton(_repository); _context.Services.AddSingleton(new FakeSamEngine()); + // The page also hosts LiveCameraScanner, which asks the browser what it + // supports on first render. + _context.JSInterop.SetupModule("./js/camera-scanner.js") + .Setup("describeSupport", _ => true) + .SetResult(new CameraSupport(SecureContext: true, HasMediaDevices: true, IsIos: false)); + _module = _context.JSInterop.SetupModule("./js/video-scan.js"); _module.Setup("prepare", _ => true).SetResult([0.5, 1.5]); _module.Setup("frameUrl", _ => true).SetResult("blob:test-frame"); diff --git a/tests/js/camera-scanner.test.mjs b/tests/js/camera-scanner.test.mjs index 758397e..7b3473c 100644 --- a/tests/js/camera-scanner.test.mjs +++ b/tests/js/camera-scanner.test.mjs @@ -7,9 +7,9 @@ const camera = await import(`data:text/javascript;base64,${source.toString("base const originalDescriptor = Object.getOwnPropertyDescriptor(globalThis, "navigator"); -function setMockNavigator(mediaDevices) { +function setMockNavigator(mediaDevices, userAgent = "", maxTouchPoints = 0) { Object.defineProperty(globalThis, "navigator", { - value: { mediaDevices }, + value: { mediaDevices, userAgent, maxTouchPoints }, configurable: true, writable: true, }); @@ -54,6 +54,7 @@ afterEach(() => { if (originalDescriptor) { Object.defineProperty(globalThis, "navigator", originalDescriptor); } + delete globalThis.isSecureContext; }); test("CameraDriver starts video stream with environment facing mode constraints", async () => { @@ -219,3 +220,69 @@ test("extractCutout creates cropped canvas with polygon clipping and returns blo assert.ok(drawn); assert.equal(paths.length, 4); }); + +test("openCamera reports a refusal by name instead of throwing it away", async () => { + const denial = new Error("Permission denied"); + denial.name = "NotAllowedError"; + + setMockNavigator({ + getUserMedia: mock.fn(async () => { + throw denial; + }), + }); + + const result = await camera.openCamera(createMockVideo()); + + // Interop flattens a thrown error down to its message, and the name is the + // part that says the person can grant this back. + assert.equal(result.ok, false); + assert.equal(result.code, "NotAllowedError"); + assert.match(result.message, /Camera access was denied/); +}); + +test("openCamera reports the facing mode it actually got", async () => { + setMockNavigator({ getUserMedia: mock.fn(async () => createMockStream()) }); + + const result = await camera.openCamera(createMockVideo(), camera.CameraFacing.User); + + assert.equal(result.ok, true); + assert.equal(result.facing, "user"); + assert.equal(result.code, null); +}); + +test("flipCamera reports a refusal rather than leaving a dead stage", async () => { + const failure = new Error("Camera is in use"); + failure.name = "NotReadableError"; + + setMockNavigator({ + getUserMedia: mock.fn(async () => { + throw failure; + }), + }); + + const result = await camera.flipCamera(createMockVideo(), camera.CameraFacing.Environment); + + assert.equal(result.ok, false); + assert.equal(result.code, "NotReadableError"); +}); + +test("describeSupport reports a withheld camera API instead of guessing", () => { + setMockNavigator(undefined); + globalThis.isSecureContext = false; + + const support = camera.describeSupport(); + + assert.equal(support.hasMediaDevices, false); + assert.equal(support.secureContext, false); +}); + +test("describeSupport recognises iOS, including an iPad claiming to be a Mac", () => { + setMockNavigator({ getUserMedia: () => {} }, "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0)"); + assert.equal(camera.describeSupport().isIos, true); + + setMockNavigator({ getUserMedia: () => {} }, "Mozilla/5.0 (Macintosh; Intel Mac OS X)", 5); + assert.equal(camera.describeSupport().isIos, true); + + setMockNavigator({ getUserMedia: () => {} }, "Mozilla/5.0 (Macintosh; Intel Mac OS X)", 0); + assert.equal(camera.describeSupport().isIos, false); +}); diff --git a/tests/js/mobile-sam.test.mjs b/tests/js/mobile-sam.test.mjs index 1fdbecf..c726b0f 100644 --- a/tests/js/mobile-sam.test.mjs +++ b/tests/js/mobile-sam.test.mjs @@ -100,6 +100,8 @@ afterEach(() => { globalThis.caches = originalCaches; globalThis.fetch = originalFetch; globalThis.document = originalDocument; + delete globalThis.crossOriginIsolated; + delete globalThis.ort; sam.disposeEngine(); }); @@ -160,15 +162,49 @@ test("OrtDriver attempts WebGPU and falls back to WASM if WebGPU is unsupported" }, }; + // A device that advertises a GPU, so the attempt is worth making here. + const originalNavigator = Object.getOwnPropertyDescriptor(globalThis, "navigator"); + Object.defineProperty(globalThis, "navigator", { + value: { gpu: {} }, + configurable: true, + writable: true, + }); + + try { + const driver = new sam.OrtDriver({ ort: mockOrt }); + const sessionResult = await driver.createSession( + mockOrt, + new ArrayBuffer(8), + sam.ExecutionDevice.Auto, + ); + + assert.equal(sessionResult.device, sam.ExecutionDevice.Wasm); + assert.equal(callCount, 2); + } finally { + if (originalNavigator) { + Object.defineProperty(globalThis, "navigator", originalNavigator); + } + } +}); + +test("OrtDriver does not attempt WebGPU on a device that has no GPU", async () => { + const mockOrt = createMockOrt(); const driver = new sam.OrtDriver({ ort: mockOrt }); + const sessionResult = await driver.createSession( mockOrt, new ArrayBuffer(8), sam.ExecutionDevice.Auto, ); + // The WebGPU runtime was never loaded here, so a session attempt could only + // allocate and then fail. assert.equal(sessionResult.device, sam.ExecutionDevice.Wasm); - assert.equal(callCount, 2); + assert.equal(mockOrt.InferenceSession.create.mock.callCount(), 1); + assert.deepEqual( + mockOrt.InferenceSession.create.mock.calls[0].arguments[1].executionProviders, + ["wasm"], + ); }); test("ImageProcessor resizes input preserving aspect ratio and normalizes RGB channels", () => { @@ -365,3 +401,95 @@ test("Module helpers provide singleton access for Blazor interop", async () => { const afterDispose = sam.getEngineStatus(); assert.equal(afterDispose.state, sam.EngineState.Uninitialized); }); + +test("OrtDriver loads the WebGPU runtime only where there is a GPU to use it", () => { + const withoutGpu = new sam.OrtDriver(); + // ort.all drags in a 23 MB WebAssembly binary; the plain WASM build is 12 MB, + // and on a phone that gap is the difference between running and being killed. + assert.match(withoutGpu.resolveCdnUrl(), /ort\.wasm\.bundle\.min\.mjs$/); + + const originalNavigator = Object.getOwnPropertyDescriptor(globalThis, "navigator"); + Object.defineProperty(globalThis, "navigator", { + value: { gpu: {} }, + configurable: true, + writable: true, + }); + + try { + assert.match(new sam.OrtDriver().resolveCdnUrl(), /ort\.webgpu\.bundle\.min\.mjs$/); + } finally { + if (originalNavigator) { + Object.defineProperty(globalThis, "navigator", originalNavigator); + } + } + + assert.equal( + new sam.OrtDriver({ cdnUrl: "https://example.com/ort.mjs" }).resolveCdnUrl(), + "https://example.com/ort.mjs", + ); +}); + +test("OrtDriver asks for a single WASM thread when the page is not cross-origin isolated", async () => { + const mockOrt = createMockOrt(); + mockOrt.env.wasm.numThreads = 4; + + globalThis.crossOriginIsolated = false; + + globalThis.ort = mockOrt; + + const driver = new sam.OrtDriver(); + await driver.getOrt(); + + // Threads need SharedArrayBuffer, which needs COOP/COEP headers the app does + // not send. Asking for four buys worker infrastructure and nothing else. + assert.equal(mockOrt.env.wasm.numThreads, 1); +}); + +test("MobileSamEngine releases each model before reaching for the next", async () => { + const mockOrt = createMockOrt(); + const order = []; + + const storage = { + loadBuffer: mock.fn(async (url) => { + order.push(`load:${url.includes("encoder") ? "encoder" : "decoder"}`); + return new ArrayBuffer(16); + }), + }; + const driver = { + getOrt: async () => mockOrt, + createSession: mock.fn(async () => { + order.push("create"); + return { + session: await mockOrt.InferenceSession.create(), + device: sam.ExecutionDevice.Wasm, + }; + }), + }; + + const engine = new sam.MobileSamEngine({ + storage, + driver, + processor: { prepareTensor: () => ({}) }, + }); + await engine.init({ + device: sam.ExecutionDevice.Wasm, + encoderUrl: "https://example.com/encoder.onnx", + decoderUrl: "https://example.com/decoder.onnx", + }); + + // Strictly one model in flight: loading both up front held two ONNX buffers + // and the runtime's copy of each at the same time, roughly doubling the peak. + assert.deepEqual(order, ["load:encoder", "create", "load:decoder", "create"]); +}); + +test("ImageProcessor hands the scratch canvas back once the pixels are copied out", () => { + const canvas = createMockCanvas(512, 288); + globalThis.document = { createElement: () => canvas }; + + new sam.ImageProcessor().prepareTensor({ width: 1920, height: 1080 }, createMockOrt()); + + // Safari keeps canvas backing stores alive well past the last reference to + // them, so this one is given up explicitly. + assert.equal(canvas.width, 0); + assert.equal(canvas.height, 0); +}); diff --git a/wwwroot/js/camera-scanner.js b/wwwroot/js/camera-scanner.js index 6c31eb3..f091d69 100644 --- a/wwwroot/js/camera-scanner.js +++ b/wwwroot/js/camera-scanner.js @@ -47,7 +47,12 @@ export class CameraDriver { height: video.videoHeight || DEFAULT_IDEAL_HEIGHT, }; } catch (error) { - throw new Error(this.#mapErrorMessage(error)); + // The name is what separates a refusal from a missing device, and the + // two want very different advice, so keep it rather than flattening + // everything into one sentence. + const wrapped = new Error(this.#mapErrorMessage(error)); + wrapped.code = error?.name || "UnknownError"; + throw wrapped; } } @@ -208,6 +213,59 @@ export function stopCamera(video) { defaultDriver.stop(video); } +/** + * What this browser is willing to do, before anyone taps anything. + * + * WebKit withholds navigator.mediaDevices entirely outside a secure context, so + * over plain http:// on a LAN address there is no camera API to refuse and no + * prompt to show - only an undefined property. Knowing that up front lets the + * page explain itself instead of offering a button that cannot work. + */ +export function describeSupport() { + const agent = navigator?.userAgent || ""; + + // iPadOS reports itself as a Mac; the touch points are what give it away. + const isIos = + /iPad|iPhone|iPod/.test(agent) || + (agent.includes("Macintosh") && (navigator?.maxTouchPoints || 0) > 1); + + return { + secureContext: typeof isSecureContext === "boolean" ? isSecureContext : true, + hasMediaDevices: !!navigator?.mediaDevices?.getUserMedia, + isIos, + }; +} + +/** + * Open the camera, reporting a refusal instead of throwing one. + * + * Interop turns a thrown error into a JSException carrying only its message, so + * the DOMException name - the part that says whether the person refused or the + * device simply has no camera - would be lost on the way across. Handing back a + * result keeps it. + */ +export async function openCamera(video, facing = CameraFacing.Environment) { + return await attemptCamera(() => defaultDriver.start(video, facing)); +} + +export async function flipCamera(video, currentFacing) { + return await attemptCamera(() => defaultDriver.switchFacing(video, currentFacing)); +} + +async function attemptCamera(run) { + try { + const result = await run(); + return { ok: true, facing: result.facing, code: null, message: null }; + } catch (error) { + return { + ok: false, + facing: null, + code: error?.code || "UnknownError", + message: error?.message || "Failed to start camera feed.", + }; + } +} + export function freezeFrame(video, canvas) { return defaultCapture.freeze(video, canvas); } diff --git a/wwwroot/js/mobile-sam.js b/wwwroot/js/mobile-sam.js index d769f07..231e105 100644 --- a/wwwroot/js/mobile-sam.js +++ b/wwwroot/js/mobile-sam.js @@ -24,10 +24,18 @@ const DEFAULT_MODEL_BASE_URL = "https://huggingface.co/Acly/MobileSAM/resolve/ma const DEFAULT_ENCODER_PATH = "mobile_sam_image_encoder.onnx"; const DEFAULT_DECODER_PATH = "sam_mask_decoder_single.onnx"; const CACHE_NAME = "mobilesam-model-v1"; -const ORT_CDN_URL = - "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.0/dist/ort.all.bundle.min.mjs"; const ORT_WASM_PATH = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.0/dist/"; +// onnxruntime publishes one bundle per set of execution providers, and they are +// not close in size. ort.all pulls in WebGPU, WebGL, WebNN and the training +// runtime, and with them the 23 MB JSEP WebAssembly binary - which it loads +// whether or not the device has a GPU to point it at. The plain WASM build is +// 12 MB. That difference is the whole budget on a phone: iOS gives a tab a few +// hundred megabytes and kills it outright when it goes over, with no error and +// no way to catch it, so the page simply reloads itself mid-tap. +const ORT_WEBGPU_BUNDLE = "ort.webgpu.bundle.min.mjs"; +const ORT_WASM_BUNDLE = "ort.wasm.bundle.min.mjs"; + const TARGET_SIZE = 1024; const CHANNELS = 3; const LOW_RES_SIZE = 256; @@ -95,13 +103,18 @@ export class ModelStorage { if ("caches" in globalThis) { try { + // Response copies what it is handed, so slicing first meant three + // copies of a 40 MB model alive at once: ours, the slice, and the + // body. Hand it the buffer directly and there are two. const cache = await globalThis.caches.open(this.#cacheName); - const cacheResponse = new Response(buffer.slice(0), { - headers: { - "content-type": "application/octet-stream", - }, - }); - await cache.put(url, cacheResponse); + await cache.put( + url, + new Response(buffer, { + headers: { + "content-type": "application/octet-stream", + }, + }), + ); } catch { // Storage quota exceeded or private mode restriction } @@ -151,10 +164,21 @@ export class OrtDriver { constructor(options = {}) { this.#ort = options.ort || null; - this.#cdnUrl = options.cdnUrl || ORT_CDN_URL; + this.#cdnUrl = options.cdnUrl || null; this.#wasmPath = options.wasmPath || ORT_WASM_PATH; } + // Loading the WebGPU runtime on a device with no navigator.gpu costs ~11 MB + // of WebAssembly that can never run anything. + resolveCdnUrl() { + if (this.#cdnUrl) { + return this.#cdnUrl; + } + + const bundle = globalThis.navigator?.gpu ? ORT_WEBGPU_BUNDLE : ORT_WASM_BUNDLE; + return `${ORT_WASM_PATH}${bundle}`; + } + async getOrt() { if (this.#ort) { return this.#ort; @@ -166,7 +190,7 @@ export class OrtDriver { return this.#ort; } - const loaded = await import(/* webpackIgnore: true */ this.#cdnUrl); + const loaded = await import(/* webpackIgnore: true */ this.resolveCdnUrl()); this.#ort = loaded.default || loaded; this.#configureWasm(this.#ort); return this.#ort; @@ -181,8 +205,16 @@ export class OrtDriver { ort.env.wasm.wasmPaths = this.#wasmPath; } - const cores = navigator.hardwareConcurrency || 2; - ort.env.wasm.numThreads = Math.min(4, cores); + // Threaded WASM runs on SharedArrayBuffer, which the browser only exposes + // to a cross-origin-isolated page - COOP and COEP headers this app does + // not send. Asking for threads anyway has onnxruntime stand up worker + // and memory infrastructure it can never use. + const isolated = + globalThis.crossOriginIsolated === true && typeof SharedArrayBuffer !== "undefined"; + + ort.env.wasm.numThreads = isolated + ? Math.min(4, globalThis.navigator?.hardwareConcurrency || 2) + : 1; } async createSession(ort, bufferOrUrl, preferredDevice = ExecutionDevice.Auto) { @@ -198,6 +230,13 @@ export class OrtDriver { } async #tryWebGpuThenWasm(ort, bufferOrUrl, strictGpu) { + // resolveCdnUrl only loads the WebGPU runtime where the device reported a + // GPU, so without one this attempt cannot do anything except fail - after + // allocating its way there, which is the part worth avoiding. + if (!strictGpu && !globalThis.navigator?.gpu) { + return await this.#createWasmSession(ort, bufferOrUrl); + } + try { const session = await ort.InferenceSession.create(bufferOrUrl, { executionProviders: [ExecutionDevice.WebGpu], @@ -237,6 +276,13 @@ export class ImageProcessor { const imageData = ctx.getImageData(0, 0, scaledWidth, scaledHeight); const pixels = imageData.data; + // Safari holds on to canvas backing stores long after the element is + // unreachable, and this one exists only for the pixels just copied out + // of it. Zeroing the dimensions releases it now rather than whenever a + // collection happens to run. + canvas.width = 0; + canvas.height = 0; + const totalPixels = scaledWidth * scaledHeight; const tensorData = new Float32Array(totalPixels * CHANNELS); @@ -561,15 +607,33 @@ export class MobileSamEngine { new URL(options.decoderPath || DEFAULT_DECODER_PATH, baseUrl).href; const preferredDevice = options.device || ExecutionDevice.Auto; - const [encoderBuffer, decoderBuffer] = await Promise.all([ - this.#storage.loadBuffer(encoderUrl, options.onEncoderProgress), - this.#storage.loadBuffer(decoderUrl, options.onDecoderProgress), - ]); - - const [encoderResult, decoderResult] = await Promise.all([ - this.#driver.createSession(this.#ort, encoderBuffer, preferredDevice), - this.#driver.createSession(this.#ort, decoderBuffer, preferredDevice), - ]); + // One model at a time. Loading both in parallel held two ONNX buffers + // and the runtime's own copy of each in memory simultaneously, which + // roughly doubles the peak for no wall-clock gain worth having - the + // download is the slow part and it is cached after the first run. + // Releasing each buffer before reaching for the next keeps the high + // water mark at one model rather than two. + let encoderBuffer = await this.#storage.loadBuffer( + encoderUrl, + options.onEncoderProgress, + ); + const encoderResult = await this.#driver.createSession( + this.#ort, + encoderBuffer, + preferredDevice, + ); + encoderBuffer = null; + + let decoderBuffer = await this.#storage.loadBuffer( + decoderUrl, + options.onDecoderProgress, + ); + const decoderResult = await this.#driver.createSession( + this.#ort, + decoderBuffer, + preferredDevice, + ); + decoderBuffer = null; this.#encoderSession = encoderResult.session; this.#decoderSession = decoderResult.session;