From 58f79be44ecee085a299c0f12cf8fb90bbd0ddc7 Mon Sep 17 00:00:00 2001
From: Elliott Imhoff <contact@elliottimhoff.com>
Date: Mon, 31 Aug 2026 15:21:42 -0500
Subject: [PATCH 1/3] Support cropped bitmasks

---
 .github/tasks.md                     |  37 ++
 index.d.ts                           |  11 +
 src/index.js                         | 263 ++++++++++---
 src/initializer.ts                   |  13 +-
 src/mask_utils.ts                    | 545 ++++++++++++++++++++++-----
 src/toolbox.ts                       |  21 +-
 src/toolbox_items/annotation_list.ts |   3 +
 tests/mask_utils.test.js             |  27 +-
 8 files changed, 743 insertions(+), 177 deletions(-)

diff --git a/.github/tasks.md b/.github/tasks.md
index 826c5841..8e08c5df 100644
--- a/.github/tasks.md
+++ b/.github/tasks.md
@@ -1,3 +1,40 @@
 ## Tasks
 
+Bitmask/segmentation viewer performance work, driven by the model-registry
+integration. Items 1/3/4/5 touch this repo; item 2 is model-registry only.
+
+- [x] 1. Cache the bitmask hover outline on the render object
+  - `draw_bitmask` allocates a canvas and does 8 dilation blits on every draw
+    while hovered. Cache on `_mask_render`, which already invalidates on mask
+    version + color change.
+- [x] 2. (model-registry) Drop `wrapperSize` from `viewerKey`
+  - A container resize currently forces a full ULabel rebuild.
+- [x] 3. Windowed `ULabelMask`
+  - Store pixels for a sub-rectangle instead of the full frame, keeping the
+    public API in image coordinates. Removes the `objects x width x height`
+    memory bound, so model-registry can go back to one annotation per Encord
+    object instead of one merged mask per class.
+  - Accept an optional `box` on the raw payload so callers can hand over an
+    already-cropped buffer with no copy.
+  - Grow the window on paint so editing still works.
+  - Verified in the browser on run #3: GT renders 99 separate objects on the
+    densest item sampled (was 1 merged mask per class), item-to-item switching
+    ~90 ms, heap flat around 1 GB with no OOM.
+- [x] 4. Swap subtasks in place instead of rebuilding the instance
+  - Add `replace_subtasks`, so a GT/Pred/Diff switch reuses the decoded image,
+    listeners and toolbox rather than running `destroy()` + `init()`.
+  - Only the annotation set can change: `replace_subtasks` returns `null` when
+    the subtask shape (keys, allowed modes, class defs) differs, so the caller
+    knows it still has to rebuild.
+  - Verified in the browser on run #3: four consecutive confidence-threshold
+    changes in pred mode produced zero rebuilds (previously one full
+    `destroy()` + `init()` each). Mode switches on that run still rebuild
+    because its GT `Row` class is a polyline while pred renders it as a
+    bitmask, which is a genuine shape change.
+- [x] 5. Decode RLE off the main thread
+  - (model-registry) move `rleRecordToRawMask` into a worker and transfer the
+    cropped buffers back.
+  - Verified in the browser: GT still renders after the move, heap 193 MB on a
+    fresh load, no page errors.
+
 
diff --git a/index.d.ts b/index.d.ts
index a22ca5c2..01038817 100644
--- a/index.d.ts
+++ b/index.d.ts
@@ -382,6 +382,17 @@ export class ULabel {
     public readjust_subtask_opacities(): void;
     public set_subtask(st_key: string): void;
     public switch_to_next_subtask(): void;
+    /**
+     * Swap in a new set of subtask specs without tearing the instance down,
+     * reusing the decoded image, canvases, toolbox and listeners. Subtasks
+     * whose annotations are already loaded are skipped.
+     *
+     * Resolves to the keys that were swapped, or to null — leaving the instance
+     * untouched — when the subtask keys, classes or allowed modes differ from
+     * what the instance was built with, since those are baked into the DOM and
+     * event bindings. Callers should rebuild the instance in that case.
+     */
+    public replace_subtasks(subtasks: ULabelSubtasks): Promise<string[] | null>;
 
     // Annotations
     public get_annotations(subtask: string): ULabelAnnotation[];
diff --git a/src/index.js b/src/index.js
index 74daee59..709754cc 100644
--- a/src/index.js
+++ b/src/index.js
@@ -60,6 +60,9 @@ jQuery.fn.outer_html = function () {
 // Valid brush overlap modes for bitmask painting (see set_brush_overlap_mode).
 const BRUSH_OVERLAP_MODES = ["none", "exclude", "overwrite"];
 
+// Width, in image pixels, of the contour drawn around a hovered bitmask.
+const BITMASK_OUTLINE_BORDER = 2;
+
 export class ULabel {
     static version() {
         return ULABEL_VERSION;
@@ -371,6 +374,11 @@ export class ULabel {
                 data: new Uint8Array(raw_payload.data),
                 size: [raw_payload.size[0], raw_payload.size[1]],
             };
+            // A cropped payload's box has to survive the clone, or the buffer would be
+            // misread as full-frame.
+            if (raw_payload.box !== undefined) {
+                cloned.spatial_payload.box = { ...raw_payload.box };
+            }
             return cloned;
         }
         return JSON.parse(JSON.stringify(raw));
@@ -2108,7 +2116,7 @@ export class ULabel {
         // the cache; only a mask edit (version bump) or color change forces a rebuild.
         let render = annotation_object["_mask_render"];
         if (render == null || render.mask !== mask || render.version !== mask.version || render.color !== color) {
-            render = this.build_bitmask_render(annotation_object, mask, color);
+            render = this.build_bitmask_render(mask, color);
             if (render === null) return; // Empty mask, nothing to draw
             this.set_bitmask_render(annotation_object, render);
         }
@@ -2132,53 +2140,53 @@ export class ULabel {
         ctx.globalAlpha = 1.0;
 
         if (this.is_annotation_hovered(annotation_object)) {
-            // Build a white contour by dilating the mask shape and cutting out the interior
-            const border = 2;
-            const ow = render.box_width + border * 2;
-            const oh = render.box_height + border * 2;
-            const outline = document.createElement("canvas");
-            outline.width = ow;
-            outline.height = oh;
-            const octx = outline.getContext("2d");
-            const dirs = [[-1, 0], [1, 0], [0, -1], [0, 1], [-1, -1], [-1, 1], [1, -1], [1, 1]];
-            for (const [ox, oy] of dirs) {
-                octx.drawImage(render.canvas, border + ox * border, border + oy * border);
-            }
-            octx.globalCompositeOperation = "source-in";
-            octx.fillStyle = "white";
-            octx.fillRect(0, 0, ow, oh);
-            octx.globalCompositeOperation = "destination-out";
-            octx.drawImage(render.canvas, border, border);
-            const blit_x = (render.tlx + diffX - border) * px_per_px;
-            const blit_y = (render.tly + diffY - border) * px_per_px;
-            ctx.drawImage(outline, blit_x, blit_y, ow * px_per_px, oh * px_per_px);
-        }
-    }
-
-    // Build a native-resolution, class-colored stencil for a bitmask's bounding box.
-    // Returns { canvas, tlx, tly, box_width, box_height, mask, version, color } or null if empty.
-    build_bitmask_render(annotation_object, mask, color) {
-        const image_width = this.config["image_width"];
-        const image_height = this.config["image_height"];
+            const outline = this.get_bitmask_outline(render);
+            const blit_x = (render.tlx + diffX - BITMASK_OUTLINE_BORDER) * px_per_px;
+            const blit_y = (render.tly + diffY - BITMASK_OUTLINE_BORDER) * px_per_px;
+            ctx.drawImage(outline, blit_x, blit_y, outline.width * px_per_px, outline.height * px_per_px);
+        }
+    }
 
-        // Only rasterize the mask's bounding box rather than the whole image. The containing
-        // box is maintained as a superset of the foreground (see rebuild_bitmask_containing_box),
-        // so every painted pixel is covered. Fall back to a full scan only if it is missing.
-        let box = annotation_object["containing_box"];
-        if (box == null) {
-            box = mask.get_bounding_box();
-            if (box === null) return null;
+    // Build (once per render) a white contour by dilating the mask shape and cutting out the
+    // interior. Cached on the render, which is already discarded whenever the mask version or
+    // class color changes, so hovering never re-rasterizes.
+    get_bitmask_outline(render) {
+        if (render.outline != null) return render.outline;
+
+        const border = BITMASK_OUTLINE_BORDER;
+        const ow = render.box_width + border * 2;
+        const oh = render.box_height + border * 2;
+        const outline = document.createElement("canvas");
+        outline.width = ow;
+        outline.height = oh;
+        const octx = outline.getContext("2d");
+        const dirs = [[-1, 0], [1, 0], [0, -1], [0, 1], [-1, -1], [-1, 1], [1, -1], [1, 1]];
+        for (const [ox, oy] of dirs) {
+            octx.drawImage(render.canvas, border + ox * border, border + oy * border);
         }
+        octx.globalCompositeOperation = "source-in";
+        octx.fillStyle = "white";
+        octx.fillRect(0, 0, ow, oh);
+        octx.globalCompositeOperation = "destination-out";
+        octx.drawImage(render.canvas, border, border);
 
-        const tlx = Math.max(0, Math.floor(box.tlx));
-        const tly = Math.max(0, Math.floor(box.tly));
-        const brx = Math.min(image_width - 1, Math.ceil(box.brx));
-        const bry = Math.min(image_height - 1, Math.ceil(box.bry));
-        const box_width = brx - tlx + 1;
-        const box_height = bry - tly + 1;
-        if (box_width <= 0 || box_height <= 0) return null;
+        render.outline = outline;
+        return outline;
+    }
 
-        // Build an opaque white stencil of just the box region at native resolution
+    // Build a native-resolution, class-colored stencil for a bitmask's stored window.
+    // Returns { canvas, tlx, tly, box_width, box_height, mask, version, color } or null if empty.
+    build_bitmask_render(mask, color) {
+        // A mask only holds pixels for its window, which is already a superset of the
+        // foreground, so the window is exactly the region worth rasterizing.
+        const box = mask.get_window_box();
+        if (box === null) return null; // Empty mask, nothing to draw
+
+        const box_width = mask.window_width;
+        const box_height = mask.window_height;
+
+        // Stencil the window at native resolution. Only alpha is written; the tint below
+        // fills color wherever alpha survives, so this is one store per foreground pixel.
         const offscreen = document.createElement("canvas");
         offscreen.width = box_width;
         offscreen.height = box_height;
@@ -2186,17 +2194,9 @@ export class ULabel {
         const image_data = offscreen_ctx.createImageData(box_width, box_height);
         const data = image_data.data;
         const mask_data = mask.data;
-        for (let y = tly; y <= bry; y++) {
-            const mask_row = y * image_width;
-            const local_row = (y - tly) * box_width;
-            for (let x = tlx; x <= brx; x++) {
-                if (mask_data[mask_row + x] !== 0) {
-                    const j = (local_row + (x - tlx)) * 4;
-                    data[j] = 255;
-                    data[j + 1] = 255;
-                    data[j + 2] = 255;
-                    data[j + 3] = 255;
-                }
+        for (let i = 0, j = 3; i < mask_data.length; i++, j += 4) {
+            if (mask_data[i] !== 0) {
+                data[j] = 255;
             }
         }
         offscreen_ctx.putImageData(image_data, 0, 0);
@@ -2209,8 +2209,8 @@ export class ULabel {
 
         return {
             canvas: offscreen,
-            tlx: tlx,
-            tly: tly,
+            tlx: box.tlx,
+            tly: box.tly,
             box_width: box_width,
             box_height: box_height,
             mask: mask,
@@ -7490,6 +7490,155 @@ export class ULabel {
         this.subtasks[subtask]["state"]["annotation_contexts"] = {};
     }
 
+    /**
+     * Whether `subtask_specs` describes the same subtask layer this instance was
+     * built with, differing only in annotations.
+     *
+     * Subtask keys, class definitions and allowed modes are baked into DOM ids,
+     * toolbox tabs and event bindings at init time, so those must match for an
+     * in-place swap to be safe.
+     */
+    _subtask_shape_matches(subtask_specs) {
+        const old_keys = Object.keys(this.subtasks);
+        const new_keys = Object.keys(subtask_specs);
+        if (old_keys.length !== new_keys.length) return false;
+
+        for (const st of new_keys) {
+            const current = this.subtasks[st];
+            if (current === undefined) return false;
+
+            const spec = subtask_specs[st];
+            const modes = spec["allowed_modes"] ?? [];
+            if (modes.length !== current["allowed_modes"].length) return false;
+            for (let i = 0; i < modes.length; i++) {
+                if (modes[i] !== current["allowed_modes"][i]) return false;
+            }
+
+            const classes = spec["classes"] ?? [];
+            const class_defs = current["class_defs"];
+            if (classes.length !== class_defs.length) return false;
+            for (let i = 0; i < classes.length; i++) {
+                const raw = classes[i];
+                // Classes may be given as bare ids, in which case only the id
+                // can differ from what was processed at init.
+                const id = typeof raw === "object" ? raw["id"] : raw;
+                if (id !== class_defs[i]["id"]) return false;
+                if (typeof raw !== "object") continue;
+                if (raw["name"] !== undefined && raw["name"] !== class_defs[i]["name"]) return false;
+                if (raw["color"] !== undefined && raw["color"] !== class_defs[i]["color"]) return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Whether `annotations` is the set already loaded on `subtask`, compared by
+     * id and order. Lets replace_subtasks skip untouched subtasks, so pushing a
+     * new spec for one class doesn't re-import and redraw its siblings.
+     */
+    _subtask_annotations_unchanged(subtask, annotations) {
+        const ordering = this.subtasks[subtask]["annotations"]["ordering"];
+        if (ordering.length !== annotations.length) return false;
+        for (let i = 0; i < ordering.length; i++) {
+            if (ordering[i] !== annotations[i]["id"]) return false;
+        }
+        return true;
+    }
+
+    /**
+     * Swap in a new set of subtask specs without tearing the instance down.
+     *
+     * Reuses the decoded image, the subtask canvases, the toolbox and every
+     * listener, rebuilding only the annotation layer — far cheaper than
+     * destroy() + new ULabel() + init() when a host application switches
+     * between views of the same image. Subtasks whose annotations are already
+     * loaded are left alone, so pushing a whole spec set costs only the
+     * subtasks that actually changed.
+     *
+     * Only annotations (and `read_only` / `inactive_opacity`) can change this
+     * way. If the subtask keys, classes or allowed modes differ, the DOM and
+     * toolbox no longer describe the incoming layer, so this returns false and
+     * leaves the instance untouched; the caller should rebuild instead.
+     *
+     * @param {object} subtask_specs Same shape as the constructor's `subtasks`.
+     * @returns {Promise<string[] | null>} the keys that were swapped, or null if
+     *     the caller must rebuild the instance.
+     */
+    async replace_subtasks(subtask_specs) {
+        if (this.is_destroyed) {
+            log_message("replace_subtasks called on a destroyed ULabel instance", LogLevel.WARNING, true);
+            return null;
+        }
+        if (!this.is_init) return null;
+        if (!this._subtask_shape_matches(subtask_specs)) return null;
+
+        const stale = [];
+        for (const st in subtask_specs) {
+            const incoming = subtask_specs[st]["resume_from"] ?? [];
+            if (!this._subtask_annotations_unchanged(st, incoming)) stale.push(st);
+        }
+        if (stale.length === 0) return [];
+
+        const container = document.getElementById(this.config["container_id"]);
+        ULabelLoader.add_loader_div(container);
+        // Yield so the browser can paint the loader before the heavy synchronous work below
+        await ULabelLoader.wait_for_render();
+
+        // Recheck: destroy() (manual or auto) may have run during the paint yield.
+        if (this.is_destroyed) return stale;
+
+        try {
+            for (const st of stale) {
+                const spec = subtask_specs[st];
+                // Undo/redo won't survive a wholesale annotation swap.
+                this.reset_interaction_state(st);
+                this.subtasks[st]["actions"]["stream"] = [];
+                this.subtasks[st]["actions"]["undone_stack"] = [];
+                this._clear_subtask_annotation_canvases(st);
+
+                ULabel.process_resume_from(this, st, { resume_from: spec["resume_from"] ?? [] });
+
+                if (spec["read_only"] !== undefined) {
+                    this.subtasks[st]["read_only"] = spec["read_only"];
+                }
+                if (spec["inactive_opacity"] !== undefined) {
+                    this.subtasks[st]["inactive_opacity"] = spec["inactive_opacity"];
+                }
+                // Keep the raw config in step so a later get/set round trip sees
+                // the annotations that are actually on screen.
+                this.config.subtasks[st] = spec;
+            }
+
+            // Yield the event loop so the loader's reveal timer can fire if the work above
+            // took long enough to cross the reveal threshold.
+            await new Promise((resolve) => setTimeout(resolve, 0));
+            if (this.is_destroyed) return stale;
+
+            if (!this.config.allow_annotations_outside_image) {
+                const image_width = this.config["image_width"];
+                const image_height = this.config["image_height"];
+                for (const st of stale) {
+                    for (const anno of Object.values(this.subtasks[st]["annotations"]["access"])) {
+                        anno.clamp_annotation_to_image_bounds(image_width, image_height);
+                    }
+                }
+            }
+
+            for (const st of stale) {
+                initialize_annotation_canvases(this, st);
+                this.redraw_all_annotations(st);
+            }
+            this.readjust_subtask_opacities();
+            // Calculate distances for all annotations if FilterDistance is present
+            this.update_filter_distance(null, false, true);
+            // Update class counter in toolbox
+            this.toolbox.redraw_update_items(this);
+        } finally {
+            ULabelLoader.remove_loader_div();
+        }
+        return stale;
+    }
+
     // Change frame
     update_frame(delta = null, new_frame = null) {
         if (this.config["image_data"]["frames"].length === 1) {
diff --git a/src/initializer.ts b/src/initializer.ts
index 3ccde13c..ff690e0b 100644
--- a/src/initializer.ts
+++ b/src/initializer.ts
@@ -208,18 +208,6 @@ export async function ulabel_init(
     // Create listers to manipulate and export this object
     create_ulabel_listeners(ulabel);
 
-    // Restore toolbox collapsed state from localStorage
-    const is_collapsed = get_local_storage_item("ulabel_toolbox_collapsed");
-    if (is_collapsed === "true") {
-        const toolbox = $("#" + ulabel.config["toolbox_id"]);
-        const container = $(".full_ulabel_container_");
-        const btn = $(".toolbox-collapse-btn");
-        toolbox.addClass("collapsed");
-        container.addClass("toolbox-collapsed");
-        btn.text("▶");
-        btn.attr("title", "Expand toolbox");
-    }
-
     ulabel.handle_toolbox_overflow();
 
     // Set the canvas elements in the correct stacking order given current subtask
@@ -233,6 +221,7 @@ export async function ulabel_init(
     ulabel.is_init = true;
 
     ulabel.show_initial_crop();
+    $(".full_ulabel_container_").addClass("ulabel-cropped");
     ulabel.update_frame();
 
     // Draw demo annotation
diff --git a/src/mask_utils.ts b/src/mask_utils.ts
index a86ae8c8..fde5e2dd 100644
--- a/src/mask_utils.ts
+++ b/src/mask_utils.ts
@@ -1,8 +1,15 @@
 // Utilities for raster "bitmask" segmentation annotations.
 //
-// A bitmask annotation stores a per-pixel binary occupancy grid the size of the
-// image. At runtime the grid is held as a row-major Uint8Array (values 0 or 1).
-// For serialization it is encoded as COCO-style, column-major run-length counts.
+// A bitmask annotation covers a per-pixel binary occupancy grid the size of the
+// image, but pixels are only *stored* for a sub-rectangle of it -- the mask's
+// "window". Everything outside the window is background by definition. The
+// public API is entirely in image coordinates, so callers never see the window;
+// it exists so memory scales with the area an annotation actually covers rather
+// than with the image, which is what lets a frame hold hundreds of objects.
+//
+// Within the window the grid is a row-major Uint8Array (values 0 or 1). For
+// serialization it is encoded as COCO-style, column-major run-length counts over
+// the full image, so the wire format is unchanged.
 
 // COCO-style run-length encoding of a binary mask.
 // - counts: alternating run lengths (in column-major / Fortran order) that always
@@ -15,20 +22,38 @@ export type ULabelMaskPayload = {
 };
 
 // Raw pixel-buffer form of a bitmask payload. Row-major, one byte per pixel
-// (non-zero = foreground). `size` is [height, width] to match ULabelMaskPayload.
-// Accepted as an alternative to the RLE form on load; callers that already have
-// the mask as a Uint8Array avoid the encode-then-decode round-trip.
+// (non-zero = foreground). `size` is [height, width] of the *image*, to match
+// ULabelMaskPayload. Accepted as an alternative to the RLE form on load; callers
+// that already have the mask as a Uint8Array avoid the encode-then-decode
+// round-trip.
+//
+// If `box` is given, `data` covers only that inclusive image-space rectangle
+// (row stride `brx - tlx + 1`) rather than the whole image, and is adopted as the
+// mask's window as-is. Producing a cropped payload is the cheapest way to import
+// a dense frame: nothing full-size is ever allocated.
 export type ULabelRawMaskPayload = {
     data: Uint8Array;
     size: [number, number];
+    box?: BoundingBox;
 };
 
+// Duck-type check for a bounding box of integer bounds.
+function is_bounding_box(box: unknown): box is BoundingBox {
+    if (box === null || typeof box !== "object") return false;
+    const b = box as Record<string, unknown>;
+    for (const key of ["tlx", "tly", "brx", "bry"]) {
+        if (typeof b[key] !== "number" || !Number.isInteger(b[key])) return false;
+    }
+    return true;
+}
+
 // Duck-type check for the raw payload shape.
 export function is_raw_mask_payload(payload: unknown): payload is ULabelRawMaskPayload {
     if (payload === null || typeof payload !== "object") return false;
-    const p = payload as { data?: unknown; size?: unknown };
+    const p = payload as { data?: unknown; size?: unknown; box?: unknown };
     if (!(p.data instanceof Uint8Array) && !(p.data instanceof Uint8ClampedArray)) return false;
     if (!Array.isArray(p.size) || p.size.length !== 2) return false;
+    if (p.box !== undefined && !is_bounding_box(p.box)) return false;
     return Number.isInteger(p.size[0]) && Number.isInteger(p.size[1]);
 }
 
@@ -49,37 +74,148 @@ function clamp_int(value: number, min: number, max: number): number {
 }
 
 export class ULabelMask {
+    // Stored pixels for the window only; length is window_width * window_height.
     public data: Uint8Array;
+    // Full image dimensions -- the mask's coordinate space, not its allocation.
     public readonly width: number;
     public readonly height: number;
     // Bumped by every mutating method so render caches can detect changes by comparison.
     public version: number = 0;
 
-    constructor(width: number, height: number, data?: Uint8Array) {
+    // Window origin and extent, in image coordinates. A zero-area window means the
+    // mask is entirely background and holds no buffer.
+    private win_x: number = 0;
+    private win_y: number = 0;
+    private win_w: number = 0;
+    private win_h: number = 0;
+
+    // `data` without `box` is treated as a full-frame buffer, preserving the
+    // original constructor contract. With `box`, `data` covers just that
+    // rectangle. With neither, the mask starts empty and grows as it is painted.
+    constructor(width: number, height: number, data?: Uint8Array, box?: BoundingBox) {
         this.width = width;
         this.height = height;
-        if (data !== undefined) {
+
+        if (data === undefined) {
+            this.data = new Uint8Array(0);
+            return;
+        }
+
+        if (box === undefined) {
             if (data.length !== width * height) {
                 throw new Error(
                     `Mask data length ${data.length} does not match dimensions ${width}x${height}`,
                 );
             }
+            this.win_w = width;
+            this.win_h = height;
             this.data = data;
-        } else {
-            this.data = new Uint8Array(width * height);
+            return;
+        }
+
+        const win_w = box.brx - box.tlx + 1;
+        const win_h = box.bry - box.tly + 1;
+        if (data.length !== win_w * win_h) {
+            throw new Error(
+                `Mask data length ${data.length} does not match window ${win_w}x${win_h}`,
+            );
         }
+        if (box.tlx < 0 || box.tly < 0 || box.brx >= width || box.bry >= height) {
+            throw new Error(
+                `Mask window [${box.tlx}, ${box.tly}, ${box.brx}, ${box.bry}] lies outside ${width}x${height}`,
+            );
+        }
+        this.win_x = box.tlx;
+        this.win_y = box.tly;
+        this.win_w = win_w;
+        this.win_h = win_h;
+        this.data = data;
     }
 
-    // Create an empty (all-background) mask.
+    // Create an empty (all-background) mask. Allocates nothing until painted.
     public static create_empty(width: number, height: number): ULabelMask {
         return new ULabelMask(width, height);
     }
 
-    public get_pixel(x: number, y: number): number {
-        if (x < 0 || y < 0 || x >= this.width || y >= this.height) {
-            return 0;
+    public get window_x(): number {
+        return this.win_x;
+    }
+
+    public get window_y(): number {
+        return this.win_y;
+    }
+
+    public get window_width(): number {
+        return this.win_w;
+    }
+
+    public get window_height(): number {
+        return this.win_h;
+    }
+
+    // The window as an inclusive image-space box, or null if the mask is empty.
+    // Renderers use this to walk `data` directly instead of probing get_pixel.
+    public get_window_box(): BoundingBox | null {
+        if (this.win_w === 0 || this.win_h === 0) return null;
+        return {
+            tlx: this.win_x,
+            tly: this.win_y,
+            brx: this.win_x + this.win_w - 1,
+            bry: this.win_y + this.win_h - 1,
+        };
+    }
+
+    // Index into `data` for an image coordinate, or -1 if outside the window.
+    private idx(x: number, y: number): number {
+        const lx = x - this.win_x;
+        const ly = y - this.win_y;
+        if (lx < 0 || ly < 0 || lx >= this.win_w || ly >= this.win_h) return -1;
+        return ly * this.win_w + lx;
+    }
+
+    // Grow the window so it contains `box` (clamped to the image), reallocating and
+    // copying existing rows across. No-op when already covered.
+    private ensure_window(box: BoundingBox): void {
+        const tlx = Math.max(0, Math.floor(box.tlx));
+        const tly = Math.max(0, Math.floor(box.tly));
+        const brx = Math.min(this.width - 1, Math.ceil(box.brx));
+        const bry = Math.min(this.height - 1, Math.ceil(box.bry));
+        if (brx < tlx || bry < tly) return;
+
+        if (this.win_w === 0 || this.win_h === 0) {
+            this.win_x = tlx;
+            this.win_y = tly;
+            this.win_w = brx - tlx + 1;
+            this.win_h = bry - tly + 1;
+            this.data = new Uint8Array(this.win_w * this.win_h);
+            return;
+        }
+
+        const cur_brx = this.win_x + this.win_w - 1;
+        const cur_bry = this.win_y + this.win_h - 1;
+        if (tlx >= this.win_x && tly >= this.win_y && brx <= cur_brx && bry <= cur_bry) return;
+
+        const new_x = Math.min(this.win_x, tlx);
+        const new_y = Math.min(this.win_y, tly);
+        const new_w = Math.max(cur_brx, brx) - new_x + 1;
+        const new_h = Math.max(cur_bry, bry) - new_y + 1;
+        const grown = new Uint8Array(new_w * new_h);
+        const row_offset = this.win_x - new_x;
+        for (let ly = 0; ly < this.win_h; ly++) {
+            const src = ly * this.win_w;
+            const dst = (ly + this.win_y - new_y) * new_w + row_offset;
+            grown.set(this.data.subarray(src, src + this.win_w), dst);
         }
-        return this.data[y * this.width + x];
+        this.win_x = new_x;
+        this.win_y = new_y;
+        this.win_w = new_w;
+        this.win_h = new_h;
+        this.data = grown;
+    }
+
+    public get_pixel(x: number, y: number): number {
+        const i = this.idx(x, y);
+        return i < 0 ? 0 : this.data[i];
     }
 
     public set_pixel(x: number, y: number, value: number): void {
@@ -87,7 +223,13 @@ export class ULabelMask {
             return;
         }
         this.version++;
-        this.data[y * this.width + x] = value ? 1 : 0;
+        if (value) {
+            this.ensure_window({ tlx: x, tly: y, brx: x, bry: y });
+        }
+        const i = this.idx(x, y);
+        // Erasing outside the window is already a no-op
+        if (i < 0) return;
+        this.data[i] = value ? 1 : 0;
     }
 
     // Paint (value = 1) or erase (value = 0) a filled circle into the mask.
@@ -100,14 +242,21 @@ export class ULabelMask {
         const max_x = clamp_int(cx + r, 0, this.width - 1);
         const min_y = clamp_int(cy - r, 0, this.height - 1);
         const max_y = clamp_int(cy + r, 0, this.height - 1);
+        const circle_box = { tlx: min_x, tly: min_y, brx: max_x, bry: max_y };
+        if (v === 1) {
+            this.ensure_window(circle_box);
+        }
+        const b = this.clamp_box_to_window(circle_box);
+        if (b === null) return false;
         const r_sq = r * r;
         let changed = false;
-        for (let y = min_y; y <= max_y; y++) {
+        for (let y = b.y0; y <= b.y1; y++) {
             const dy = y - cy;
-            for (let x = min_x; x <= max_x; x++) {
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            for (let x = b.x0; x <= b.x1; x++) {
                 const dx = x - cx;
                 if (dx * dx + dy * dy <= r_sq) {
-                    const idx = y * this.width + x;
+                    const idx = row + x;
                     if (this.data[idx] !== v) {
                         this.data[idx] = v;
                         changed = true;
@@ -129,16 +278,20 @@ export class ULabelMask {
     // True if any foreground pixel lies within the given circle.
     public has_foreground_in_circle(cx: number, cy: number, radius: number): boolean {
         const r = Math.max(0, radius);
-        const min_x = clamp_int(cx - r, 0, this.width - 1);
-        const max_x = clamp_int(cx + r, 0, this.width - 1);
-        const min_y = clamp_int(cy - r, 0, this.height - 1);
-        const max_y = clamp_int(cy + r, 0, this.height - 1);
+        const b = this.clamp_box_to_window({
+            tlx: clamp_int(cx - r, 0, this.width - 1),
+            tly: clamp_int(cy - r, 0, this.height - 1),
+            brx: clamp_int(cx + r, 0, this.width - 1),
+            bry: clamp_int(cy + r, 0, this.height - 1),
+        });
+        if (b === null) return false;
         const r_sq = r * r;
-        for (let y = min_y; y <= max_y; y++) {
+        for (let y = b.y0; y <= b.y1; y++) {
             const dy = y - cy;
-            for (let x = min_x; x <= max_x; x++) {
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            for (let x = b.x0; x <= b.x1; x++) {
                 const dx = x - cx;
-                if (dx * dx + dy * dy <= r_sq && this.data[y * this.width + x] !== 0) {
+                if (dx * dx + dy * dy <= r_sq && this.data[row + x] !== 0) {
                     return true;
                 }
             }
@@ -149,52 +302,69 @@ export class ULabelMask {
     // Axis-aligned bounding box of foreground pixels, or null if empty.
     // Returned as { tlx, tly, brx, bry } in image pixel coordinates.
     public get_bounding_box(): BoundingBox | null {
-        let min_x = this.width;
-        let min_y = this.height;
+        let min_x = this.win_w;
+        let min_y = this.win_h;
         let max_x = -1;
         let max_y = -1;
-        for (let y = 0; y < this.height; y++) {
-            const row = y * this.width;
-            for (let x = 0; x < this.width; x++) {
-                if (this.data[row + x] !== 0) {
-                    if (x < min_x) min_x = x;
-                    if (x > max_x) max_x = x;
-                    if (y < min_y) min_y = y;
-                    if (y > max_y) max_y = y;
+        for (let ly = 0; ly < this.win_h; ly++) {
+            const row = ly * this.win_w;
+            for (let lx = 0; lx < this.win_w; lx++) {
+                if (this.data[row + lx] !== 0) {
+                    if (lx < min_x) min_x = lx;
+                    if (lx > max_x) max_x = lx;
+                    if (ly < min_y) min_y = ly;
+                    if (ly > max_y) max_y = ly;
                 }
             }
         }
         if (max_x < 0) {
             return null;
         }
-        return { tlx: min_x, tly: min_y, brx: max_x, bry: max_y };
+        return {
+            tlx: min_x + this.win_x,
+            tly: min_y + this.win_y,
+            brx: max_x + this.win_x,
+            bry: max_y + this.win_y,
+        };
     }
 
     // Return a new mask with all foreground pixels shifted by (dx, dy) image pixels.
     // Pixels shifted outside the image are dropped.
     public translate(dx: number, dy: number): ULabelMask {
         const shifted = new ULabelMask(this.width, this.height);
+        const box = this.get_window_box();
+        if (box === null) return shifted;
         const idx = Math.round(dx);
         const idy = Math.round(dy);
-        for (let y = 0; y < this.height; y++) {
-            const ny = y + idy;
+        if (box.brx + idx < 0 || box.bry + idy < 0) return shifted;
+        if (box.tlx + idx >= this.width || box.tly + idy >= this.height) return shifted;
+        shifted.ensure_window({
+            tlx: box.tlx + idx,
+            tly: box.tly + idy,
+            brx: box.brx + idx,
+            bry: box.bry + idy,
+        });
+        for (let ly = 0; ly < this.win_h; ly++) {
+            const ny = ly + this.win_y + idy;
             if (ny < 0 || ny >= this.height) continue;
-            const src_row = y * this.width;
-            const dst_row = ny * this.width;
-            for (let x = 0; x < this.width; x++) {
-                if (this.data[src_row + x] !== 0) {
-                    const nx = x + idx;
+            const src_row = ly * this.win_w;
+            for (let lx = 0; lx < this.win_w; lx++) {
+                if (this.data[src_row + lx] !== 0) {
+                    const nx = lx + this.win_x + idx;
                     if (nx < 0 || nx >= this.width) continue;
-                    shifted.data[dst_row + nx] = 1;
+                    const i = shifted.idx(nx, ny);
+                    if (i >= 0) shifted.data[i] = 1;
                 }
             }
         }
         return shifted;
     }
 
-    // Return a copy of this mask.
+    // Return a copy of this mask, window and all.
     public clone(): ULabelMask {
-        return new ULabelMask(this.width, this.height, this.data.slice());
+        const box = this.get_window_box();
+        if (box === null) return new ULabelMask(this.width, this.height);
+        return new ULabelMask(this.width, this.height, this.data.slice(), box);
     }
 
     // Ensure another mask has the same dimensions as this one.
@@ -206,16 +376,41 @@ export class ULabelMask {
         }
     }
 
+    // Clamp a box to the intersection of the image and this mask's window, returning
+    // integer inclusive image-space bounds or null if empty.
+    private clamp_box_to_window(box: BoundingBox): { x0: number; y0: number; x1: number; y1: number } | null {
+        if (this.win_w === 0 || this.win_h === 0) return null;
+        const x0 = Math.max(this.win_x, Math.floor(box.tlx));
+        const y0 = Math.max(this.win_y, Math.floor(box.tly));
+        const x1 = Math.min(this.win_x + this.win_w - 1, Math.ceil(box.brx));
+        const y1 = Math.min(this.win_y + this.win_h - 1, Math.ceil(box.bry));
+        if (x1 < x0 || y1 < y0) return null;
+        return { x0, y0, x1, y1 };
+    }
+
+    // Inclusive image-space bounds covered by both masks' windows, or null.
+    private window_overlap(other: ULabelMask): { x0: number; y0: number; x1: number; y1: number } | null {
+        const box = other.get_window_box();
+        if (box === null) return null;
+        return this.clamp_box_to_window(box);
+    }
+
     // Remove another mask's foreground from this one (this = this AND NOT other).
     // Returns true if any pixel changed.
     public subtract(other: ULabelMask): boolean {
         this.assert_same_dims(other);
         this.version++;
+        const b = this.window_overlap(other);
+        if (b === null) return false;
         let changed = false;
-        for (let i = 0; i < this.data.length; i++) {
-            if (this.data[i] !== 0 && other.data[i] !== 0) {
-                this.data[i] = 0;
-                changed = true;
+        for (let y = b.y0; y <= b.y1; y++) {
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            const other_row = (y - other.win_y) * other.win_w - other.win_x;
+            for (let x = b.x0; x <= b.x1; x++) {
+                if (this.data[row + x] !== 0 && other.data[other_row + x] !== 0) {
+                    this.data[row + x] = 0;
+                    changed = true;
+                }
             }
         }
         return changed;
@@ -227,9 +422,10 @@ export class ULabelMask {
     // pixel changed. Used to apply ULabel's polygon/bbox delete modes to raster masks.
     public subtract_polygon(polygon: [number, number][]): boolean {
         if (polygon.length < 3) return false;
+        if (this.win_w === 0 || this.win_h === 0) return false;
         this.version++;
 
-        // Restrict work to the polygon's vertical extent, clamped to the image.
+        // Restrict work to the polygon's vertical extent, clamped to the window.
         let min_py = Infinity;
         let max_py = -Infinity;
         for (let i = 0; i < polygon.length; i++) {
@@ -237,8 +433,8 @@ export class ULabelMask {
             if (py < min_py) min_py = py;
             if (py > max_py) max_py = py;
         }
-        const y_start = Math.max(0, Math.ceil(min_py));
-        const y_end = Math.min(this.height - 1, Math.floor(max_py));
+        const y_start = Math.max(this.win_y, Math.ceil(min_py));
+        const y_end = Math.min(this.win_y + this.win_h - 1, Math.floor(max_py));
 
         let changed = false;
         const n = polygon.length;
@@ -257,10 +453,10 @@ export class ULabelMask {
             }
             if (xs.length < 2) continue;
             xs.sort((a, b) => a - b);
-            const row = y * this.width;
+            const row = (y - this.win_y) * this.win_w - this.win_x;
             for (let k = 0; k + 1 < xs.length; k += 2) {
-                const x_start = Math.max(0, Math.ceil(xs[k]));
-                const x_end = Math.min(this.width - 1, Math.floor(xs[k + 1]));
+                const x_start = Math.max(this.win_x, Math.ceil(xs[k]));
+                const x_end = Math.min(this.win_x + this.win_w - 1, Math.floor(xs[k + 1]));
                 for (let x = x_start; x <= x_end; x++) {
                     if (this.data[row + x] !== 0) {
                         this.data[row + x] = 0;
@@ -276,9 +472,18 @@ export class ULabelMask {
     public add_mask(other: ULabelMask): void {
         this.assert_same_dims(other);
         this.version++;
-        for (let i = 0; i < this.data.length; i++) {
-            if (other.data[i] !== 0) {
-                this.data[i] = 1;
+        const other_box = other.get_window_box();
+        if (other_box === null) return;
+        this.ensure_window(other_box);
+        const b = this.window_overlap(other);
+        if (b === null) return;
+        for (let y = b.y0; y <= b.y1; y++) {
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            const other_row = (y - other.win_y) * other.win_w - other.win_x;
+            for (let x = b.x0; x <= b.x1; x++) {
+                if (other.data[other_row + x] !== 0) {
+                    this.data[row + x] = 1;
+                }
             }
         }
     }
@@ -287,9 +492,24 @@ export class ULabelMask {
     public intersect(other: ULabelMask): void {
         this.assert_same_dims(other);
         this.version++;
-        for (let i = 0; i < this.data.length; i++) {
-            if (other.data[i] === 0) {
-                this.data[i] = 0;
+        const b = this.window_overlap(other);
+        if (b === null) {
+            this.data.fill(0);
+            return;
+        }
+        const win_x1 = this.win_x + this.win_w - 1;
+        for (let y = this.win_y; y <= this.win_y + this.win_h - 1; y++) {
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            // Rows the other mask doesn't reach are cleared wholesale.
+            if (y < b.y0 || y > b.y1) {
+                this.data.fill(0, row + this.win_x, row + win_x1 + 1);
+                continue;
+            }
+            const other_row = (y - other.win_y) * other.win_w - other.win_x;
+            for (let x = this.win_x; x <= win_x1; x++) {
+                if (x < b.x0 || x > b.x1 || other.data[other_row + x] === 0) {
+                    this.data[row + x] = 0;
+                }
             }
         }
     }
@@ -297,20 +517,28 @@ export class ULabelMask {
     // True if this mask shares any foreground pixel with another.
     public intersects(other: ULabelMask): boolean {
         this.assert_same_dims(other);
-        for (let i = 0; i < this.data.length; i++) {
-            if (this.data[i] !== 0 && other.data[i] !== 0) {
-                return true;
+        const b = this.window_overlap(other);
+        if (b === null) return false;
+        for (let y = b.y0; y <= b.y1; y++) {
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            const other_row = (y - other.win_y) * other.win_w - other.win_x;
+            for (let x = b.x0; x <= b.x1; x++) {
+                if (this.data[row + x] !== 0 && other.data[other_row + x] !== 0) {
+                    return true;
+                }
             }
         }
         return false;
     }
 
-    // Clamp a bounding box to the image, returning integer inclusive bounds or null if empty.
-    private clamp_box_to_image(box: BoundingBox): { x0: number; y0: number; x1: number; y1: number } | null {
-        const x0 = Math.max(0, Math.floor(box.tlx));
-        const y0 = Math.max(0, Math.floor(box.tly));
-        const x1 = Math.min(this.width - 1, Math.ceil(box.brx));
-        const y1 = Math.min(this.height - 1, Math.ceil(box.bry));
+    // Inclusive bounds covered by `box` and both masks' windows, or null.
+    private overlap_in_box(other: ULabelMask, box: BoundingBox): { x0: number; y0: number; x1: number; y1: number } | null {
+        const b = this.window_overlap(other);
+        if (b === null) return null;
+        const x0 = Math.max(b.x0, Math.floor(box.tlx));
+        const y0 = Math.max(b.y0, Math.floor(box.tly));
+        const x1 = Math.min(b.x1, Math.ceil(box.brx));
+        const y1 = Math.min(b.y1, Math.ceil(box.bry));
         if (x1 < x0 || y1 < y0) return null;
         return { x0, y0, x1, y1 };
     }
@@ -318,13 +546,13 @@ export class ULabelMask {
     // True if any pixel within `box` is foreground in both masks. O(box area).
     public intersects_in_box(other: ULabelMask, box: BoundingBox): boolean {
         this.assert_same_dims(other);
-        const b = this.clamp_box_to_image(box);
+        const b = this.overlap_in_box(other, box);
         if (b === null) return false;
         for (let y = b.y0; y <= b.y1; y++) {
-            const row = y * this.width;
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            const other_row = (y - other.win_y) * other.win_w - other.win_x;
             for (let x = b.x0; x <= b.x1; x++) {
-                const i = row + x;
-                if (this.data[i] !== 0 && other.data[i] !== 0) return true;
+                if (this.data[row + x] !== 0 && other.data[other_row + x] !== 0) return true;
             }
         }
         return false;
@@ -334,16 +562,16 @@ export class ULabelMask {
     // Returns true if any pixel changed. O(box area).
     public subtract_in_box(other: ULabelMask, box: BoundingBox): boolean {
         this.assert_same_dims(other);
-        const b = this.clamp_box_to_image(box);
+        const b = this.overlap_in_box(other, box);
         if (b === null) return false;
         this.version++;
         let changed = false;
         for (let y = b.y0; y <= b.y1; y++) {
-            const row = y * this.width;
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            const other_row = (y - other.win_y) * other.win_w - other.win_x;
             for (let x = b.x0; x <= b.x1; x++) {
-                const i = row + x;
-                if (this.data[i] !== 0 && other.data[i] !== 0) {
-                    this.data[i] = 0;
+                if (this.data[row + x] !== 0 && other.data[other_row + x] !== 0) {
+                    this.data[row + x] = 0;
                     changed = true;
                 }
             }
@@ -356,16 +584,23 @@ export class ULabelMask {
     public subtract_intersection_in_box(a: ULabelMask, b: ULabelMask, box: BoundingBox): boolean {
         this.assert_same_dims(a);
         this.assert_same_dims(b);
-        const bx = this.clamp_box_to_image(box);
-        if (bx === null) return false;
+        const bounds = this.overlap_in_box(a, box);
+        const b_box = b.get_window_box();
+        if (bounds === null || b_box === null) return false;
+        const x0 = Math.max(bounds.x0, b_box.tlx);
+        const y0 = Math.max(bounds.y0, b_box.tly);
+        const x1 = Math.min(bounds.x1, b_box.brx);
+        const y1 = Math.min(bounds.y1, b_box.bry);
+        if (x1 < x0 || y1 < y0) return false;
         this.version++;
         let changed = false;
-        for (let y = bx.y0; y <= bx.y1; y++) {
-            const row = y * this.width;
-            for (let x = bx.x0; x <= bx.x1; x++) {
-                const i = row + x;
-                if (this.data[i] !== 0 && a.data[i] !== 0 && b.data[i] !== 0) {
-                    this.data[i] = 0;
+        for (let y = y0; y <= y1; y++) {
+            const row = (y - this.win_y) * this.win_w - this.win_x;
+            const a_row = (y - a.win_y) * a.win_w - a.win_x;
+            const b_row = (y - b.win_y) * b.win_w - b.win_x;
+            for (let x = x0; x <= x1; x++) {
+                if (this.data[row + x] !== 0 && a.data[a_row + x] !== 0 && b.data[b_row + x] !== 0) {
+                    this.data[row + x] = 0;
                     changed = true;
                 }
             }
@@ -373,7 +608,7 @@ export class ULabelMask {
         return changed;
     }
 
-    // Encode to COCO-style, column-major run-length counts.
+    // Encode to COCO-style, column-major run-length counts over the full image.
     public to_rle(): ULabelMaskPayload {
         const counts: number[] = [];
         // Runs always start with background (0). Compare foreground-truthiness rather
@@ -381,9 +616,24 @@ export class ULabelMask {
         // (e.g. 0/255 masks, multi-valued upstream buffers) still encode correctly.
         let current_is_fg = false;
         let run = 0;
+        const win_x1 = this.win_x + this.win_w;
+        const win_y1 = this.win_y + this.win_h;
         for (let x = 0; x < this.width; x++) {
+            // Columns outside the window are background end to end; skip the scan.
+            if (x < this.win_x || x >= win_x1) {
+                if (current_is_fg) {
+                    counts.push(run);
+                    current_is_fg = false;
+                    run = this.height;
+                } else {
+                    run += this.height;
+                }
+                continue;
+            }
+            const col = x - this.win_x;
             for (let y = 0; y < this.height; y++) {
-                const is_fg = this.data[y * this.width + x] !== 0;
+                const is_fg = y >= this.win_y && y < win_y1 &&
+                    this.data[(y - this.win_y) * this.win_w + col] !== 0;
                 if (is_fg === current_is_fg) {
                     run++;
                 } else {
@@ -437,13 +687,57 @@ export class ULabelMask {
         }
     }
 
-    // Decode a COCO-style RLE payload into a mask.
+    // Bounding box of the foreground implied by a column-major RLE, without decoding
+    // it. Walks runs rather than pixels, so this is O(runs).
+    private static rle_bounding_box(payload: ULabelMaskPayload): BoundingBox | null {
+        const [height] = payload.size;
+        if (height === 0) return null;
+        let min_x = Infinity;
+        let min_y = Infinity;
+        let max_x = -1;
+        let max_y = -1;
+        let idx = 0;
+        let value = 0;
+        for (let c = 0; c < payload.counts.length; c++) {
+            const run = payload.counts[c];
+            if (value === 1 && run > 0) {
+                const last = idx + run - 1;
+                const first_x = Math.floor(idx / height);
+                const last_x = Math.floor(last / height);
+                if (first_x < min_x) min_x = first_x;
+                if (last_x > max_x) max_x = last_x;
+                if (last_x > first_x) {
+                    // Crossing a column boundary means the run reaches the bottom of its
+                    // first column and the top of its last, so it spans every row.
+                    min_y = 0;
+                    max_y = height - 1;
+                } else {
+                    const first_y = idx % height;
+                    const last_y = last % height;
+                    if (first_y < min_y) min_y = first_y;
+                    if (last_y > max_y) max_y = last_y;
+                }
+            }
+            idx += run;
+            value = value === 0 ? 1 : 0;
+        }
+        if (max_x < 0) return null;
+        return { tlx: min_x, tly: min_y, brx: max_x, bry: max_y };
+    }
+
+    // Decode a COCO-style RLE payload into a mask, allocating only the foreground's
+    // bounding box rather than the whole frame.
     public static from_rle(payload: ULabelMaskPayload, validate: boolean = true): ULabelMask {
         if (validate) {
             ULabelMask.validate_rle(payload);
         }
         const [height, width] = payload.size;
-        const mask = new ULabelMask(width, height);
+        const box = ULabelMask.rle_bounding_box(payload);
+        if (box === null) return new ULabelMask(width, height);
+
+        const win_w = box.brx - box.tlx + 1;
+        const data = new Uint8Array(win_w * (box.bry - box.tly + 1));
+        const mask = new ULabelMask(width, height, data, box);
         let idx = 0; // column-major index
         let value = 0;
         for (let c = 0; c < payload.counts.length; c++) {
@@ -453,7 +747,7 @@ export class ULabelMask {
                     const col_idx = idx + k;
                     const x = Math.floor(col_idx / height);
                     const y = col_idx % height;
-                    mask.data[y * width + x] = 1;
+                    data[(y - box.tly) * win_w + (x - box.tlx)] = 1;
                 }
             }
             idx += run;
@@ -471,6 +765,17 @@ export class ULabelMask {
         if (height < 0 || width < 0) {
             throw new Error(`Invalid raw mask size: expected non-negative integers, got [${height}, ${width}]`);
         }
+        if (payload.box !== undefined) {
+            const box = payload.box;
+            if (box.tlx < 0 || box.tly < 0 || box.brx >= width || box.bry >= height) {
+                throw new Error(`Invalid raw mask box [${box.tlx}, ${box.tly}, ${box.brx}, ${box.bry}] for ${height}x${width}`);
+            }
+            const expected = (box.brx - box.tlx + 1) * (box.bry - box.tly + 1);
+            if (payload.data.length !== expected) {
+                throw new Error(`Invalid raw mask data length: expected ${expected} bytes for box, got ${payload.data.length}`);
+            }
+            return;
+        }
         const expected = height * width;
         if (payload.data.length !== expected) {
             throw new Error(`Invalid raw mask data length: expected ${expected} bytes for ${height}x${width}, got ${payload.data.length}`);
@@ -478,14 +783,56 @@ export class ULabelMask {
     }
 
     // Wrap a raw pixel-buffer payload as a ULabelMask.
-    // By default copies the input so the caller can safely mutate their own array;
-    // pass `copy: false` when the caller (e.g. process_resume_from) already copied.
+    //
+    // With `box`, the buffer is already cropped and is adopted as the window -- by
+    // default copied so the caller can safely mutate their own array; pass
+    // `copy: false` when the caller (e.g. process_resume_from) already copied.
+    //
+    // Without `box`, the buffer is full-frame and is cropped to its bounding box on
+    // import, so the full-size allocation becomes garbage immediately instead of
+    // being retained for the life of the annotation. `copy` is moot in that case;
+    // the crop is always a fresh buffer.
     public static from_raw(payload: ULabelRawMaskPayload, validate: boolean = true, copy: boolean = true): ULabelMask {
         if (validate) {
             ULabelMask.validate_raw(payload);
         }
         const [height, width] = payload.size;
-        const data = copy ? new Uint8Array(payload.data) : payload.data;
-        return new ULabelMask(width, height, data);
+
+        if (payload.box !== undefined) {
+            const data = copy ? new Uint8Array(payload.data) : payload.data;
+            return new ULabelMask(width, height, data, payload.box);
+        }
+
+        const src = payload.data;
+        let min_x = width;
+        let min_y = height;
+        let max_x = -1;
+        let max_y = -1;
+        for (let y = 0; y < height; y++) {
+            const row = y * width;
+            for (let x = 0; x < width; x++) {
+                if (src[row + x] !== 0) {
+                    if (x < min_x) min_x = x;
+                    if (x > max_x) max_x = x;
+                    if (y < min_y) min_y = y;
+                    if (y > max_y) max_y = y;
+                }
+            }
+        }
+        if (max_x < 0) return new ULabelMask(width, height);
+
+        const win_w = max_x - min_x + 1;
+        const win_h = max_y - min_y + 1;
+        const data = new Uint8Array(win_w * win_h);
+        for (let y = 0; y < win_h; y++) {
+            const src_start = (y + min_y) * width + min_x;
+            data.set(src.subarray(src_start, src_start + win_w), y * win_w);
+        }
+        return new ULabelMask(width, height, data, {
+            tlx: min_x,
+            tly: min_y,
+            brx: max_x,
+            bry: max_y,
+        });
     }
 }
diff --git a/src/toolbox.ts b/src/toolbox.ts
index be7355fd..022cc132 100644
--- a/src/toolbox.ts
+++ b/src/toolbox.ts
@@ -124,6 +124,12 @@ export class Toolbox {
             height: 100%;
         }
 
+        /* The image is inserted at its natural size and only scaled once the
+           initial crop lands, so keep it hidden until then. */
+        .full_ulabel_container_:not(.ulabel-cropped) .imwrap_cls {
+            visibility: hidden;
+        }
+
         #toolbox {
             width: 320px;
             background-color: white;
@@ -272,17 +278,26 @@ export class Toolbox {
         images: string,
         ULABEL_VERSION: string,
     ): string {
+        // Bake the persisted collapsed state into the markup. Applying it after
+        // init instead paints a frame with the toolbox expanded, which reads as
+        // a flicker every time the instance is rebuilt.
+        const is_collapsed = get_local_storage_item("ulabel_toolbox_collapsed") === "true";
+        const container_class = is_collapsed ? "full_ulabel_container_ toolbox-collapsed" : "full_ulabel_container_";
+        const toolbox_class = is_collapsed ? "toolbox_cls collapsed" : "toolbox_cls";
+        const collapse_arrow = is_collapsed ? "◀" : "▶";
+        const collapse_title = is_collapsed ? "Expand toolbox" : "Collapse toolbox";
+
         // Setup base div and ULabel version header
         let toolbox_html = `
-        <div class="full_ulabel_container_">
+        <div class="${container_class}">
             ${frame_annotation_dialogs}
             <div id="${ulabel.config["annbox_id"]}" class="annbox_cls">
                 <div id="${ulabel.config["imwrap_id"]}" class="imwrap_cls ${ulabel.config["imgsz_class"]}">
                     ${images}
                 </div>
             </div>
-            <button class="toolbox-collapse-btn" title="Collapse toolbox">▶</button>
-            <div id="${ulabel.config["toolbox_id"]}" class="toolbox_cls">
+            <button class="toolbox-collapse-btn" title="${collapse_title}">${collapse_arrow}</button>
+            <div id="${ulabel.config["toolbox_id"]}" class="${toolbox_class}">
                 <div class="toolbox-header-container">
                     <div class="toolbox-name-header">
                         <h1 class="toolname"><a class="repo-anchor" href="https://github.com/SenteraLLC/ulabel">ULabel</a> <span class="version-number">v${ULABEL_VERSION}</span></h1><!--
diff --git a/src/toolbox_items/annotation_list.ts b/src/toolbox_items/annotation_list.ts
index 0054e9d4..0528561e 100644
--- a/src/toolbox_items/annotation_list.ts
+++ b/src/toolbox_items/annotation_list.ts
@@ -702,6 +702,9 @@ export class AnnotationListToolboxItem extends ToolboxItem {
      * Called when the edit_candidate changes on the canvas
      */
     private sync_highlight_from_canvas() {
+        // A frame scheduled by the annbox mousemove handler can still be
+        // pending when the instance is destroyed, which nulls this reference.
+        if (!this.ulabel) return;
         const current_subtask = this.ulabel.get_current_subtask();
         if (!current_subtask) return;
 
diff --git a/tests/mask_utils.test.js b/tests/mask_utils.test.js
index 1f997761..a170c80a 100644
--- a/tests/mask_utils.test.js
+++ b/tests/mask_utils.test.js
@@ -7,8 +7,11 @@ describe("ULabelMask", () => {
             const mask = ULabelMask.create_empty(4, 3);
             expect(mask.width).toBe(4);
             expect(mask.height).toBe(3);
-            expect(mask.data.length).toBe(12);
             expect(mask.is_empty()).toBe(true);
+            // Storage is windowed to the foreground, so an empty mask holds no
+            // pixels at all -- that is what keeps hundreds of objects viable.
+            expect(mask.get_window_box()).toBeNull();
+            expect(mask.data.length).toBe(0);
         });
 
         test("throws when provided data length mismatches dimensions", () => {
@@ -168,10 +171,18 @@ describe("ULabelMask", () => {
             const rle = mask.to_rle();
             expect(rle.size).toEqual([2, 3]);
             expect(rle.counts).toEqual([0, 4, 2]);
-            // Decoding the RLE must produce the same foreground pattern.
+            // Decoding the RLE must produce the same foreground pattern. Compare
+            // in image coordinates: storage is windowed to the bounding box, so
+            // the two masks' buffers need not share a layout.
             const restored = ULabelMask.from_rle(rle);
-            const fg_pattern = Array.from(restored.data).map((b) => (b !== 0 ? 1 : 0));
-            const original_fg = Array.from(data).map((b) => (b !== 0 ? 1 : 0));
+            const fg_pattern = [];
+            const original_fg = [];
+            for (let y = 0; y < 2; y++) {
+                for (let x = 0; x < 3; x++) {
+                    fg_pattern.push(restored.get_pixel(x, y) !== 0 ? 1 : 0);
+                    original_fg.push(data[y * 3 + x] !== 0 ? 1 : 0);
+                }
+            }
             expect(fg_pattern).toEqual(original_fg);
         });
     });
@@ -356,10 +367,14 @@ describe("ULabelMask", () => {
         });
 
         test("from_raw with copy=false references the caller's buffer", () => {
+            // Zero-copy needs an already-cropped payload: without a `box` the
+            // mask has to crop the full frame down to its window, which by
+            // definition allocates.
             const data = new Uint8Array([0, 0, 0, 0]);
-            const mask = ULabelMask.from_raw({ data, size: [2, 2] }, true, false);
+            const box = { tlx: 0, tly: 0, brx: 1, bry: 1 };
+            const mask = ULabelMask.from_raw({ data, size: [2, 2], box }, true, false);
             data[0] = 1;
-            expect(mask.data[0]).toBe(1);
+            expect(mask.get_pixel(0, 0)).toBe(1);
         });
 
         test("validate_raw rejects wrong-length buffers", () => {

From 4d628db659e828798bd41c8df071862047277f10 Mon Sep 17 00:00:00 2001
From: Elliott Imhoff <contact@elliottimhoff.com>
Date: Wed, 2 Sep 2026 15:37:14 -0500
Subject: [PATCH 2/3] Fix bugs

---
 .github/tasks.md                   |  60 ++++++++++
 index.d.ts                         |  62 ++++++++++
 src/annotation.ts                  |   5 +
 src/annotation_operators.ts        |  28 +++++
 src/blobs.js                       |   8 +-
 src/canvas_utils.ts                |   1 +
 src/configuration.ts               |   9 ++
 src/index.js                       | 180 ++++++++++++++++++++++++-----
 src/listeners.ts                   |  14 +++
 tests/annotation_operators.test.js |  61 ++++++++++
 10 files changed, 401 insertions(+), 27 deletions(-)

diff --git a/.github/tasks.md b/.github/tasks.md
index 8e08c5df..53124f4e 100644
--- a/.github/tasks.md
+++ b/.github/tasks.md
@@ -37,4 +37,64 @@ integration. Items 1/3/4/5 touch this repo; item 2 is model-registry only.
   - Verified in the browser: GT still renders after the move, heap 193 MB on a
     fresh load, no page errors.
 
+## Architecture: subtask per data set, class as class
+
+The viewer models GT/Pred/Diff as *per-class* subtasks whose set changes with
+the view mode, so a mode switch is a subtask-shape change and forces a rebuild.
+Diff goes further and replaces the class with the outcome (`FP`/`FN`/`TP`), so
+class identity is destroyed and "false negatives for Crop" is inexpressible.
+
+Target: three fixed subtasks (`groundtruth`, `prediction`, `diff`), each with
+the real class defs, present in every mode with only their annotations
+swapping. Outcome moves to annotation metadata. This is also what diff-driven
+groundtruth editing needs, since applying a diff region to a GT mask requires
+both loaded together with class identity intact.
+
+- [x] 6. Per-annotation color resolver
+  - `get_annotation_color` looks up `color_info[class_id]`. Add an optional
+    per-annotation hook so the diff subtask can color by outcome while keeping
+    real classes. Every draw path already funnels through this one function.
+  - Added `annotation_color_resolver` to `Configuration` and the constructor
+    args. Returning `null` falls back to the class color, and the confidence
+    gradient still applies either way.
+- [x] 7. Class-aware annotation canvases
+  - `get_next_available_canvas_id` packs annotations into the first non-full
+    canvas regardless of class, and per-subtask opacity/z-index is what dims
+    inactive layers today. Group canvases by class so the same CSS mechanism
+    gives per-class dimming and bring-to-front once classes share a subtask.
+  - Canvases now nest under a `div.class_canvasses` per class, and
+    `set_active_class_layer(subtask, class_id, inactive_opacity)` mirrors
+    `readjust_subtask_opacities` one level down. Verified: lint clean, 161
+    jest tests and 102 Chromium e2e tests pass.
+- [x] 8. `hidden_by` visibility map
+  - Mirror the keyed composition of `deprecated_by` for view filtering.
+    Separate from `deprecated`, which means "deleted" and is about to start
+    flowing back to Encord.
+  - Added `mark_hidden` plus a public `filter_annotations(hidden_by_key,
+    should_hide, subtask, redraw)`. Keys compose, so class/outcome/confidence
+    controls can be applied in any order. `hidden` gates drawing, edit
+    candidates, and annotation navigation, and also skips bulk polygon delete
+    so it can't remove something the user can't see. Export is untouched.
+- [x] 9. (model-registry) Rebuild subtask construction on the new model
+  - Three fixed subtasks, real class defs, `match_outcome` in
+    `annotation_meta`, Encord object hash carried on GT annotations, and class
+    chips driving filters rather than `set_subtask`.
+  - `buildViewSubtasks` replaces `buildClassSubtasks`/`buildDiffSubtasks`: all
+    three subtasks share one class list and one `allowed_modes` union derived
+    from the ontology, so the subtask shape no longer changes with the data.
+  - Diff layers by outcome instead of class, which class-keyed canvases alone
+    could not express. Added `annotation_canvas_group_resolver` to ULabel and
+    generalized `set_active_class_layer`'s `inactive_opacity` to accept a
+    per-key map, preserving the old fn 0.6 / fp 0.6 / tp 0.4 dim values.
+  - Sharing class ids across subtasks tripped ULabel's duplicate-id warning,
+    which checked the global `valid_class_ids`. Scoped the check to duplicates
+    within a subtask and made `valid_class_ids` a true set. Colors are written
+    idempotently and `findAllClassDefinitions` already de-duplicates by id, so
+    the confidence slider still shows one entry per class.
+  - Verified in browser on eval run #3: all three modes paint with no console
+    warnings; `canvasses__prediction` groups by class id (`0`/`1`/`2`) and
+    `canvasses__diff` by outcome (`tp`/`fp`/`fn`), with the selected layer at
+    opacity 1 / z-index 76 and the rest dimmed. GT shows polyline and bitmask
+    classes together in one subtask. Lint clean, 166 jest tests pass.
+
 
diff --git a/index.d.ts b/index.d.ts
index 01038817..75a6f2d8 100644
--- a/index.d.ts
+++ b/index.d.ts
@@ -45,6 +45,22 @@ export type DeprecatedBy = {
     confidence_slider?: boolean;
 };
 
+/**
+ * Valid keys for the HiddenBy type.
+ *
+ * Distinct from ValidDeprecatedBy on purpose: `deprecated` means the annotation
+ * has been deleted, while `hidden` only means it is filtered out of the current
+ * view. Conflating them loses data as soon as deprecation is persisted.
+ */
+export type ValidHiddenBy = "human" | "class_filter" | "outcome_filter" | "confidence_filter";
+
+export type HiddenBy = {
+    human?: boolean;
+    class_filter?: boolean;
+    outcome_filter?: boolean;
+    confidence_filter?: boolean;
+};
+
 /**
  * Info needed to filter distance from row without accessing the dom.
  * Primarily exists so that points can be filtered before the page loads.
@@ -301,6 +317,17 @@ export type ULabelConstructorArgs = {
     instructions_url?: string;
     toolbox_order?: AllowedToolboxItem[];
     auto_destroy_on_detach?: boolean;
+    /**
+     * Override the color of individual annotations. Return null to fall back to
+     * the annotation's class color.
+     */
+    annotation_color_resolver?: (annotation: ULabelAnnotation) => string | null;
+    /**
+     * Override which canvas layer an annotation is grouped onto. Return null to
+     * fall back to its class id. Grouping decides what `set_active_class_layer`
+     * can dim or raise.
+     */
+    annotation_canvas_group_resolver?: (annotation: ULabelAnnotation) => string | null;
     /** @deprecated Use top-level properties instead. */
     config_data?: object;
 };
@@ -334,6 +361,18 @@ export class ULabel {
 
     config: Configuration;
     toolbox: Toolbox;
+    drag_state: {
+        active_key: string | null;
+        release_button: number | null;
+    } & Record<
+        "annotation" | "brush" | "edit" | "pan" | "zoom" | "move" | "right",
+        {
+            mouse_start: [number, number] | null;
+            offset_start: [number, number] | null;
+            zoom_val_start: number | null;
+        }
+    >;
+
     color_info: { [key: number]: string };
     valid_class_ids: number[];
     toolbox_order?: number[];
@@ -380,6 +419,25 @@ export class ULabel {
     public get_current_subtask(): ULabelSubtask;
     public is_current_subtask_read_only(): boolean;
     public readjust_subtask_opacities(): void;
+    public set_active_class_layer(
+        subtask: string,
+        active_class_id: number | string | null,
+        inactive_opacity?: number | Record<string, number>,
+    ): void;
+    public get_annotation_canvas_group(annotation: ULabelAnnotation): string | null;
+    /**
+     * Hide or show annotations in a subtask under a named filter key. Filters
+     * compose, so independent controls can be applied in any order. Hiding is a
+     * view operation only and never marks an annotation deleted.
+     *
+     * @returns how many annotations changed visibility
+     */
+    public filter_annotations(
+        hidden_by_key: ValidHiddenBy,
+        should_hide: (annotation: ULabelAnnotation) => boolean,
+        subtask?: string | null,
+        redraw?: boolean,
+    ): number;
     public set_subtask(st_key: string): void;
     public switch_to_next_subtask(): void;
     /**
@@ -584,6 +642,7 @@ export class ULabel {
     ): void;
     public hide_global_edit_suggestion(): void;
     public hide_edit_suggestion(): void;
+    public hide_and_clear_action_candidates(): void;
 
     // Edit utils
     public get_with_access_string(
@@ -639,7 +698,10 @@ export class ULabel {
     public get_init_canvas_context_id(
         annotation_id: string,
         subtask?: string, // SUBTASK KEY
+        class_id?: number | string | null,
     ): string;
+    public get_canvas_class_key(class_id: number | string | null): string;
+    public get_class_canvasses_id(subtask: string, class_id: number | string | null): string;
 }
 
 declare global {
diff --git a/src/annotation.ts b/src/annotation.ts
index 8793b374..7fb4bf4d 100644
--- a/src/annotation.ts
+++ b/src/annotation.ts
@@ -1,6 +1,7 @@
 import type {
     DeprecatedBy,
     DistanceFromPolylineClasses,
+    HiddenBy,
     ULabelClassificationPayload,
     ULabelContainingBox,
     ULabelSpatialType,
@@ -70,6 +71,10 @@ export class ULabelAnnotation {
         // Lineage tracking fields
         public last_edited_by?: string,
         public last_edited_at?: string,
+        // View filtering. Unlike `deprecated`, these never mean "deleted", so they
+        // are left undefined unless something actually filters the annotation.
+        public hidden?: boolean,
+        public hidden_by?: HiddenBy,
     ) {}
 
     public ensure_compatible_classification_payloads(ulabel_class_ids: [number]) {
diff --git a/src/annotation_operators.ts b/src/annotation_operators.ts
index 287f80fb..315c07ab 100644
--- a/src/annotation_operators.ts
+++ b/src/annotation_operators.ts
@@ -4,7 +4,9 @@ import type {
     DeprecatedBy,
     DistanceFromPolylineClasses,
     FilterDistanceOverride,
+    HiddenBy,
     ValidDeprecatedBy,
+    ValidHiddenBy,
     ClassDefinition,
 } from "../index";
 import { ULabel } from "../src/index";
@@ -97,6 +99,32 @@ export function mark_deprecated(
     annotation.deprecated = false;
 }
 
+/**
+ * Takes in an annotation and marks it either hidden or not hidden by a given key.
+ *
+ * Hiding is a view concern and is kept separate from deprecation: a deprecated
+ * annotation has been deleted, while a hidden one is merely filtered out of the
+ * current view and must survive a round trip untouched.
+ *
+ * @param annotation ULabelAnnotation
+ * @param hidden boolean
+ * @param hidden_by_key which filter is hiding the annotation
+ */
+export function mark_hidden(
+    annotation: ULabelAnnotation,
+    hidden: boolean,
+    hidden_by_key: ValidHiddenBy = "human",
+) {
+    if (annotation.hidden_by === undefined) {
+        annotation.hidden_by = <HiddenBy> {};
+    }
+
+    annotation.hidden_by[hidden_by_key] = hidden;
+
+    // Hidden by any filter means hidden overall
+    annotation.hidden = Object.values(annotation.hidden_by).some((x) => x);
+}
+
 /**
  * If the value is less than the filter then return true, else return false.
  *
diff --git a/src/blobs.js b/src/blobs.js
index be942e96..ec3de6fd 100644
--- a/src/blobs.js
+++ b/src/blobs.js
@@ -1727,6 +1727,11 @@ div#${prntid} div.canvasses {
    padding-right: 100%;
    padding-bottom: 100%;
 }
+div#${prntid} div.class_canvasses {
+   position: absolute;
+   top: 0;
+   left: 0;
+}
 div#${prntid} canvas.canvas_cls {
    position: absolute;
    top: 0;
@@ -1862,7 +1867,8 @@ div#${prntid} div.dialogs_container {
    position: absolute;
    top: 0;
    left: 0;
-   z-index: ${BACK_Z_INDEX + 1};
+   /* Above the active class layer, which sits one above the resting canvases */
+   z-index: ${BACK_Z_INDEX + 2};
 }
 
 div.toolbox_inner_cls {
diff --git a/src/canvas_utils.ts b/src/canvas_utils.ts
index 6a51700e..4cf0b726 100644
--- a/src/canvas_utils.ts
+++ b/src/canvas_utils.ts
@@ -64,6 +64,7 @@ export function initialize_annotation_canvases(
             annotation["canvas_id"] = ulabel.get_init_canvas_context_id(
                 annotation_id,
                 subtask_key!,
+                ulabel.get_annotation_canvas_group(annotation),
             );
         }
     }
diff --git a/src/configuration.ts b/src/configuration.ts
index 066ad150..38c57a20 100644
--- a/src/configuration.ts
+++ b/src/configuration.ts
@@ -8,6 +8,7 @@ import type {
     ULabelSubmitButton,
     AnnoScalingMode,
     BrushOverlapMode,
+    ULabelAnnotation,
 } from "../index";
 import {
     ModeSelectionToolboxItem,
@@ -146,6 +147,14 @@ export class Configuration {
     public image_data: ImageData | null = null;
     public allow_soft_id: boolean = false;
     public default_annotation_color: string = "#fa9d2a";
+    // Override the color of individual annotations. Return null to fall back to
+    // the annotation's class color. Lets a subtask keep real class defs while
+    // rendering by some other property, e.g. a diff outcome.
+    public annotation_color_resolver: ((annotation: ULabelAnnotation) => string | null) | null = null;
+    // Override which canvas layer an annotation is grouped onto. Return null to
+    // fall back to its class id. Grouping decides what `set_active_class_layer`
+    // can dim or raise, so a diff subtask can layer by outcome instead of class.
+    public annotation_canvas_group_resolver: ((annotation: ULabelAnnotation) => string | null) | null = null;
     public username: string = "ULabelUser";
     public initial_line_size: number = 5;
 
diff --git a/src/index.js b/src/index.js
index 709754cc..c3d6bd68 100644
--- a/src/index.js
+++ b/src/index.js
@@ -23,6 +23,7 @@ import {
     get_annotation_confidence,
     get_point_and_line_annotations,
     mark_deprecated,
+    mark_hidden,
     update_distance_from_line_to_each_point,
 } from "../build/annotation_operators";
 import { AnnotationResizeItem, BrushToolboxItem } from "../build/toolbox";
@@ -307,8 +308,8 @@ export class ULabel {
                     // Only create an id if one wasn't provided
                     id = class_definition.id ?? ULabel.create_unused_class_id(ulabel);
 
-                    if (ulabel.valid_class_ids.includes(id)) {
-                        log_message(`Duplicate class id ${id} detected. This is not supported and may result in unintended side-effects.
+                    if (subtask.class_ids.includes(id)) {
+                        log_message(`Duplicate class id ${id} detected within subtask ${subtask_key}. This is not supported and may result in unintended side-effects.
                         This may be caused by mixing string and object class definitions, or by assigning the same id to two or more object class definitions.`,
                         LogLevel.WARNING);
                     }
@@ -334,8 +335,11 @@ export class ULabel {
             subtask.class_defs.push(modifed_class_definition);
             subtask.class_ids.push(modifed_class_definition.id);
 
-            // Also save the id and color_info on the ULabel object
-            ulabel.valid_class_ids.push(modifed_class_definition.id);
+            // Also save the id and color_info on the ULabel object. Subtasks may
+            // legitimately share a class, so this stays a set.
+            if (!ulabel.valid_class_ids.includes(modifed_class_definition.id)) {
+                ulabel.valid_class_ids.push(modifed_class_definition.id);
+            }
             ulabel.color_info[modifed_class_definition.id] = modifed_class_definition.color;
         }
 
@@ -348,7 +352,9 @@ export class ULabel {
                 color: COLORS[1],
                 keybind: null,
             });
-            ulabel.valid_class_ids.push(DELETE_CLASS_ID);
+            if (!ulabel.valid_class_ids.includes(DELETE_CLASS_ID)) {
+                ulabel.valid_class_ids.push(DELETE_CLASS_ID);
+            }
             ulabel.color_info[DELETE_CLASS_ID] = COLORS[1];
         }
     }
@@ -1058,6 +1064,73 @@ export class ULabel {
         }
     }
 
+    /**
+     * Dim every class in a subtask except the active one, and raise the active
+     * one above its siblings. The per-subtask equivalent of `set_subtask`, for
+     * when one subtask holds several classes.
+     *
+     * @param {string} subtask subtask key
+     * @param {number|string|null} active_class_id class to bring to front, or null for none
+     * @param {number|object} inactive_opacity opacity for the other classes, either one
+     *     value for all of them or a map keyed by class, mirroring how each
+     *     subtask carries its own `inactive_opacity`
+     */
+    set_active_class_layer(subtask, active_class_id, inactive_opacity = 0.4) {
+        const active_key = this.get_canvas_class_key(active_class_id);
+        // Dimmed classes stop being hover/edit targets, so what's interactive
+        // matches what's legible. Guarded because callers can race a pending
+        // `replace_subtasks` and name a subtask that doesn't exist yet.
+        const subtask_state = this.subtasks[subtask]?.["state"];
+        if (subtask_state) {
+            subtask_state["active_class_layer"] = active_class_id === null ? null : active_key;
+        }
+        $("#canvasses__" + subtask + " > div.class_canvasses").each((_, el) => {
+            const key = $(el).attr("data-class-key");
+            const is_active = active_class_id !== null && key === active_key;
+            let dim = inactive_opacity;
+            if (typeof inactive_opacity !== "number") {
+                dim = inactive_opacity?.[key] ?? 0.4;
+            }
+            $(el).css("opacity", is_active ? 1 : dim);
+            $(el).css("z-index", is_active ? BACK_Z_INDEX + 1 : BACK_Z_INDEX);
+        });
+    }
+
+    /**
+     * Hide or show annotations in a subtask based on a predicate, under a named
+     * filter key. Filters compose: an annotation stays hidden while any key
+     * hides it, so independent controls (class, outcome, confidence) can be
+     * applied in any order without clobbering each other.
+     *
+     * Hiding is purely a view operation. Unlike deprecation it never marks an
+     * annotation deleted, and it survives export untouched.
+     *
+     * @param {string} hidden_by_key which filter is being applied
+     * @param {function} should_hide predicate receiving each annotation
+     * @param {string|null} subtask subtask key, or null for the current subtask
+     * @param {boolean} redraw whether to redraw the subtask afterwards
+     * @returns {number} how many annotations changed visibility
+     */
+    filter_annotations(hidden_by_key, should_hide, subtask = null, redraw = true) {
+        if (subtask === null) {
+            subtask = this.get_current_subtask_key();
+        }
+
+        const annotations = this.subtasks[subtask]["annotations"]["access"];
+        let changed = 0;
+        for (const annotation_id in annotations) {
+            const annotation = annotations[annotation_id];
+            const was_hidden = annotation["hidden"] === true;
+            mark_hidden(annotation, should_hide(annotation) === true, hidden_by_key);
+            if (annotation["hidden"] !== was_hidden) changed++;
+        }
+
+        if (redraw && changed > 0) {
+            this.redraw_all_annotations(subtask);
+        }
+        return changed;
+    }
+
     set_subtask(st_key) {
         let old_st = this.get_current_subtask_key();
 
@@ -1437,32 +1510,62 @@ export class ULabel {
      * @param {string} subtask subtask name
      * @returns {string} The ID of an available canvas
      */
-    get_next_available_canvas_id(subtask = null) {
+    get_next_available_canvas_id(subtask = null, class_id = null) {
         if (subtask === null) {
             subtask = this.get_current_subtask_key();
         }
-        const canvas_ids = Object.keys(this.subtasks[subtask]["state"]["annotation_contexts"]);
+        const contexts = this.subtasks[subtask]["state"]["annotation_contexts"];
+        const canvas_ids = Object.keys(contexts);
+        const key = this.get_canvas_class_key(class_id);
         for (let i = 0; i < canvas_ids.length; i++) {
-            // If the canvas has less than n_annos_per_canvas annotations, return its ID
-            if (this.subtasks[subtask]["state"]["annotation_contexts"][canvas_ids[i]]["annotation_ids"].length < this.config.n_annos_per_canvas) {
+            const context = contexts[canvas_ids[i]];
+            // Annotations are grouped onto canvases by class so that a whole
+            // class can be dimmed or raised with one CSS write.
+            if (context["class_key"] !== key) continue;
+            if (context["annotation_ids"].length < this.config.n_annos_per_canvas) {
                 return canvas_ids[i];
             }
         }
         // If no canvas has less than n_annos_per_canvas annotations, create a new canvas
-        return this.create_annotation_canvas(subtask);
+        return this.create_annotation_canvas(subtask, class_id);
+    }
+
+    // Normalize a class id into the suffix used for per-class canvas grouping.
+    get_canvas_class_key(class_id) {
+        return class_id === null || class_id === undefined ? "none" : String(class_id);
+    }
+
+    // Which canvas layer an annotation belongs to. Defaults to its class.
+    get_annotation_canvas_group(annotation) {
+        const resolved = this.config.annotation_canvas_group_resolver?.(annotation);
+        return resolved != null ? resolved : get_annotation_class_id(annotation);
+    }
+
+    // The div grouping one class's annotation canvases within a subtask.
+    get_class_canvasses_id(subtask, class_id) {
+        return `canvasses__${subtask}__cls__${this.get_canvas_class_key(class_id)}`;
     }
 
     /**
      * Create a new canvas and return its ID
      *
      * @param {string} subtask name
+     * @param {number|string|null} class_id class to group the canvas under
      * @returns {string} The ID of a new canvas
      */
-    create_annotation_canvas(subtask) {
+    create_annotation_canvas(subtask, class_id = null) {
         const canvas_id = `canvas__${this.make_new_annotation_id()}`;
+        const class_key = this.get_canvas_class_key(class_id);
+        const group_id = this.get_class_canvasses_id(subtask, class_id);
+
+        if ($("#" + group_id).length === 0) {
+            $("#canvasses__" + subtask).append(
+                `<div id="${group_id}" class="class_canvasses" data-class-key="${class_key}"></div>`,
+            );
+            $("#" + group_id).css("z-index", BACK_Z_INDEX);
+        }
 
-        // Add canvas to the "canvasses__${subtask}" div
-        $("#canvasses__" + subtask).append(`
+        $("#" + group_id).append(`
             <canvas 
                 id="${canvas_id}" 
                 class="${this.config["canvas_class"]} ${this.config["imgsz_class"]} canvas_cls annotation_canvas" 
@@ -1478,6 +1581,7 @@ export class ULabel {
         // Add the canvas context to the state
         this.subtasks[subtask]["state"]["annotation_contexts"][canvas_id] = {
             annotation_ids: [],
+            class_key: class_key,
             context: document.getElementById(canvas_id).getContext("2d"),
         };
 
@@ -1489,14 +1593,15 @@ export class ULabel {
      *
      * @param {string} annotation_id annotation ID
      * @param {string} subtask subtask name
+     * @param {number|string|null} class_id class to group the annotation under
      * @returns {string} The ID of the canvas context
      */
-    get_init_canvas_context_id(annotation_id, subtask = null) {
+    get_init_canvas_context_id(annotation_id, subtask = null, class_id = null) {
         if (subtask === null) {
             subtask = this.get_current_subtask_key();
         }
         // Get the next available canvas id
-        const canvas_id = this.get_next_available_canvas_id(subtask);
+        const canvas_id = this.get_next_available_canvas_id(subtask, class_id);
         // Add the annotation id to the canvas context
         this.subtasks[subtask]["state"]["annotation_contexts"][canvas_id]["annotation_ids"].push(annotation_id);
 
@@ -1700,6 +1805,13 @@ export class ULabel {
     }
 
     get_annotation_color(annotation) {
+        const gradient_val = $("#gradient-slider").val() / 100;
+
+        const resolved = this.config.annotation_color_resolver?.(annotation);
+        if (resolved != null) {
+            return get_gradient(annotation, resolved, get_annotation_confidence, gradient_val);
+        }
+
         // Use the annotation's class id to get the color of the annotation
         const class_id = get_annotation_class_id(annotation);
         const color = this.color_info[class_id];
@@ -1712,7 +1824,7 @@ export class ULabel {
 
         // Return the color after applying a gradient to it based on its confidence
         // If gradients are disabled, get_gradient will return the passed in color
-        return get_gradient(annotation, color, get_annotation_confidence, $("#gradient-slider").val() / 100);
+        return get_gradient(annotation, color, get_annotation_confidence, gradient_val);
     }
 
     get_active_class_color() {
@@ -2383,8 +2495,8 @@ export class ULabel {
     draw_annotation(annotation_object, offset = null, subtask = null) {
         // DEBUG left here for refactor reference, but I don't think it's needed moving forward
         //    there may be a use case for drawing depreacted annotations
-        // Don't draw if deprecated
-        if (annotation_object["deprecated"]) return;
+        // Don't draw if deprecated or filtered out of the current view
+        if (annotation_object["deprecated"] || annotation_object["hidden"]) return;
 
         // Get actual context from context key and subtask
         let context = null;
@@ -3214,8 +3326,9 @@ export class ULabel {
         let modified_annotations = {};
         // Loop through all annotations
         for (let [annid, annotation] of Object.entries(annotations)) {
-            // Skip deprecated annotations
-            if (annotation["deprecated"]) {
+            // Skip deprecated annotations, and hidden ones so a bulk delete
+            // can never remove something the user can't see
+            if (annotation["deprecated"] || annotation["hidden"]) {
                 continue;
             }
             // Skip non-spatial annotations and 3D annotations
@@ -4279,7 +4392,11 @@ export class ULabel {
             init_id_payload = redo_payload.init_payload;
         }
 
-        let canvas_id = this.get_init_canvas_context_id(annotation_id, subtask_key);
+        let canvas_id = this.get_init_canvas_context_id(
+            annotation_id,
+            subtask_key,
+            get_annotation_class_id({ classification_payloads: init_id_payload }),
+        );
 
         // TODO(3d)
         if (NONSPATIAL_MODES.includes(annotation_mode)) {
@@ -4911,8 +5028,12 @@ export class ULabel {
         const subtask_key = this.get_current_subtask_key();
         const current_subtask = this.subtasks[subtask_key];
         const annotation_id = this.make_new_annotation_id();
-        const canvas_id = this.get_init_canvas_context_id(annotation_id, subtask_key);
         const init_id_payload = this.get_init_id_payload("bitmask");
+        const canvas_id = this.get_init_canvas_context_id(
+            annotation_id,
+            subtask_key,
+            get_annotation_class_id({ classification_payloads: init_id_payload }),
+        );
 
         current_subtask["annotations"]["access"][annotation_id] = {
             id: annotation_id,
@@ -6114,11 +6235,16 @@ export class ULabel {
         };
         let minsize = Infinity;
         let found_containing_annotation = false;
+        const active_class_layer = this.get_current_subtask()["state"]["active_class_layer"] ?? null;
         // TODO(3d)
         for (let edi = 0; edi < this.get_current_subtask()["annotations"]["ordering"].length; edi++) {
             const annotation_id = this.get_current_subtask()["annotations"]["ordering"][edi];
             let annotation = this.get_current_subtask()["annotations"]["access"][annotation_id];
-            if (annotation["deprecated"]) continue;
+            if (annotation["deprecated"] || annotation["hidden"]) continue;
+            if (
+                active_class_layer !== null &&
+                this.get_canvas_class_key(this.get_annotation_canvas_group(annotation)) !== active_class_layer
+            ) continue;
             let cbox = annotation["containing_box"];
             let frame = annotation["frame"];
             const spatial_type = annotation["spatial_type"];
@@ -7152,11 +7278,12 @@ export class ULabel {
     }
 
     fly_to_annotation(annotation, subtask_key = null, max_zoom = 10) {
-        // Handle null, deprecated, and non-spatial annotations
+        // Handle null, deprecated, hidden, and non-spatial annotations
         if (
             annotation === null ||
             annotation === undefined ||
             annotation["deprecated"] ||
+            annotation["hidden"] ||
             NONSPATIAL_MODES.includes(annotation["spatial_type"])
         ) {
             return false;
@@ -7224,7 +7351,7 @@ export class ULabel {
         let current_visible_idx = -1;
         for (let i = 0; i < ordering.length; i++) {
             const ann = current_subtask["annotations"]["access"][ordering[i]];
-            if (!ann["deprecated"]) {
+            if (!ann["deprecated"] && !ann["hidden"]) {
                 if (ordering[i] === annotation_id) {
                     current_visible_idx = visible_count;
                 }
@@ -7486,7 +7613,8 @@ export class ULabel {
         // Only remove per-annotation canvases. The subtask's front/back canvases and the
         // #dialogs__<subtask> container (which owns the brush circle, polygon ender, and
         // id dialogs) live in the same parent and MUST survive.
-        $("#canvasses__" + subtask + " > canvas.annotation_canvas").remove();
+        $("#canvasses__" + subtask + " canvas.annotation_canvas").remove();
+        $("#canvasses__" + subtask + " > div.class_canvasses").remove();
         this.subtasks[subtask]["state"]["annotation_contexts"] = {};
     }
 
diff --git a/src/listeners.ts b/src/listeners.ts
index 83d3ddf5..79401e33 100644
--- a/src/listeners.ts
+++ b/src/listeners.ts
@@ -699,6 +699,20 @@ export function create_ulabel_listeners(
         },
     );
 
+    $(document).on(
+        "mouseleave" + ULABEL_NAMESPACE,
+        "#" + ulabel.config["annbox_id"],
+        () => {
+            // The suggestion dialogs live inside the annbox, so moving onto one
+            // of them doesn't count as leaving.
+            if (ulabel.drag_state["active_key"] !== null) return;
+            const state = ulabel.get_current_subtask()["state"];
+            // A clicked-open id dialog is its own interaction; don't yank it away
+            if (state["idd_visible"] && !state["idd_thumbnail"]) return;
+            ulabel.hide_and_clear_action_candidates();
+        },
+    );
+
     // Button to toggle night mode
     $(document).on(
         "click" + ULABEL_NAMESPACE,
diff --git a/tests/annotation_operators.test.js b/tests/annotation_operators.test.js
index 4a6a5cf5..78dcc25b 100644
--- a/tests/annotation_operators.test.js
+++ b/tests/annotation_operators.test.js
@@ -4,6 +4,7 @@ const {
     get_spatial_annotations_with_confidence,
     findAllClassDefinitions,
     findAllPolylineClassDefinitions,
+    mark_hidden,
     CONFIDENCE_FILTERABLE_SPATIAL_TYPES,
 } = require("../build/annotation_operators");
 
@@ -150,3 +151,63 @@ describe("findAllClassDefinitions", () => {
         expect(ids).toEqual([1, 3]);
     });
 });
+
+describe("mark_hidden", () => {
+    test("hides and unhides under a single key", () => {
+        const annotation = make_annotation("bbox", [{ class_id: 1, confidence: 0.9 }]);
+
+        mark_hidden(annotation, true, "class_filter");
+        expect(annotation.hidden).toBe(true);
+
+        mark_hidden(annotation, false, "class_filter");
+        expect(annotation.hidden).toBe(false);
+    });
+
+    test("stays hidden while any key hides it", () => {
+        const annotation = make_annotation("bbox", [{ class_id: 1, confidence: 0.9 }]);
+
+        mark_hidden(annotation, true, "class_filter");
+        mark_hidden(annotation, true, "confidence_filter");
+        expect(annotation.hidden).toBe(true);
+
+        // Clearing one filter must not reveal an annotation the other still hides
+        mark_hidden(annotation, false, "class_filter");
+        expect(annotation.hidden).toBe(true);
+
+        mark_hidden(annotation, false, "confidence_filter");
+        expect(annotation.hidden).toBe(false);
+    });
+
+    test("keys are independent of each other's order", () => {
+        const a = make_annotation("bbox", [{ class_id: 1, confidence: 0.9 }]);
+        const b = make_annotation("bbox", [{ class_id: 1, confidence: 0.9 }]);
+
+        mark_hidden(a, true, "outcome_filter");
+        mark_hidden(a, false, "class_filter");
+
+        mark_hidden(b, false, "class_filter");
+        mark_hidden(b, true, "outcome_filter");
+
+        expect(a.hidden).toBe(true);
+        expect(b.hidden).toBe(true);
+        expect(a.hidden_by).toEqual(b.hidden_by);
+    });
+
+    test("does not deprecate the annotation", () => {
+        const annotation = make_annotation("bbox", [{ class_id: 1, confidence: 0.9 }]);
+
+        mark_hidden(annotation, true, "class_filter");
+
+        // Hiding is a view concern; deprecation means deleted and must be untouched
+        expect(annotation.deprecated).toBe(false);
+        expect(annotation.deprecated_by).toBeUndefined();
+    });
+
+    test("defaults to the human key", () => {
+        const annotation = make_annotation("bbox", [{ class_id: 1, confidence: 0.9 }]);
+
+        mark_hidden(annotation, true);
+
+        expect(annotation.hidden_by).toEqual({ human: true });
+    });
+});

From 0c4ee0f24628f60725322cc0f41d653d3d64cc39 Mon Sep 17 00:00:00 2001
From: Elliott Imhoff <contact@elliottimhoff.com>
Date: Wed, 2 Sep 2026 17:48:54 -0500
Subject: [PATCH 3/3] Fix hover bug

---
 .github/tasks.md       | 37 ++++++++++++++++++
 index.d.ts             |  5 +++
 src/configuration.ts   |  3 ++
 src/geometric_utils.ts | 42 +++++++++++++++++++++
 src/index.js           | 85 ++++++++++++++++++++++++++++++++++--------
 5 files changed, 157 insertions(+), 15 deletions(-)

diff --git a/.github/tasks.md b/.github/tasks.md
index 53124f4e..a9bedb72 100644
--- a/.github/tasks.md
+++ b/.github/tasks.md
@@ -98,3 +98,40 @@ both loaded together with class identity intact.
     classes together in one subtask. Lint clean, 166 jest tests pass.
 
 
+
+- [x] 10. Show every diff outcome at once, and keep the hover card off the annotation
+  - (model-registry) Dropped the TP/FP/FN layer picker: diff mode now calls
+    `set_active_class_layer(key, null, 1)` so all three outcome groups stay at
+    full opacity. A null active layer is also what makes them all hover
+    targets, since `get_edit_candidates` skips groups that aren't active.
+    Passing the opacity explicitly matters: with no active class every group
+    takes the `inactive_opacity` branch, so the default would dim all of them.
+    The sidebar "Diff Colors" rows are now a legend plus recolor.
+  - Added `annotation_display_name_resolver` to ULabel, alongside the existing
+    color and canvas-group resolvers, so the hover card can name the diff
+    outcome instead of the class. Every diff annotation carries the same class,
+    which made the old class name useless there.
+  - The hover card was anchored at the containing box's centre, so it covered
+    whatever was under the cursor. It now clears the box by half its on-screen
+    height plus a gap, flipping below only when there isn't room above.
+    Offsets are divided by the dialog container's CSS scale (0.5 / 0.66666
+    from `.global_edit_suggestion`), which otherwise halves them.
+  - Verified in browser on eval run #3 item 503: `canvasses__diff` holds `fn`,
+    `fp` and `tp` all at opacity 1, each is hover-targetable, the card reads
+    "True Positive" / "False Negative", and it sits a 10 px gap above the
+    hovered box in every sampled position. Lint clean in both repos.
+
+- [x] 11. Hover on the annotation boundary, not its containing box
+  - `get_edit_candidates` already hit-tests exactly (`get_pixel` for bitmasks,
+    point-in-polygon for polygons), so this cost nothing extra. The stray
+    hovers came from the fallback underneath: when nothing contains the
+    cursor, it still picked the smallest annotation whose *containing box*
+    was within `dst_thresh`. That fallback exists so you can grab an
+    annotation to edit it, which a read-only subtask never needs.
+  - Now skipped when the subtask is read-only and the spatial type has an
+    exact test. Types without one (polyline, tbar, contour) keep the box
+    fallback, so they stay hoverable.
+  - Verified on run #3 item 503: across six probes the hover card appeared if
+    and only if the cursor was over a painted mask pixel, comparing against
+    the coordinates ULabel itself received. GT polylines still hover and read
+    "Row". 166 unit tests pass.
diff --git a/index.d.ts b/index.d.ts
index 75a6f2d8..d5feb5ed 100644
--- a/index.d.ts
+++ b/index.d.ts
@@ -328,6 +328,11 @@ export type ULabelConstructorArgs = {
      * can dim or raise.
      */
     annotation_canvas_group_resolver?: (annotation: ULabelAnnotation) => string | null;
+    /**
+     * Override the name the hover card shows for an annotation. Return null to
+     * fall back to its class name.
+     */
+    annotation_display_name_resolver?: (annotation: ULabelAnnotation) => string | null;
     /** @deprecated Use top-level properties instead. */
     config_data?: object;
 };
diff --git a/src/configuration.ts b/src/configuration.ts
index 38c57a20..24235010 100644
--- a/src/configuration.ts
+++ b/src/configuration.ts
@@ -155,6 +155,9 @@ export class Configuration {
     // fall back to its class id. Grouping decides what `set_active_class_layer`
     // can dim or raise, so a diff subtask can layer by outcome instead of class.
     public annotation_canvas_group_resolver: ((annotation: ULabelAnnotation) => string | null) | null = null;
+    // Override the name the hover card shows for an annotation. Return null to
+    // fall back to its class name, so a diff subtask can label by outcome.
+    public annotation_display_name_resolver: ((annotation: ULabelAnnotation) => string | null) | null = null;
     public username: string = "ULabelUser";
     public initial_line_size: number = 5;
 
diff --git a/src/geometric_utils.ts b/src/geometric_utils.ts
index b9373846..5c178fb3 100644
--- a/src/geometric_utils.ts
+++ b/src/geometric_utils.ts
@@ -543,6 +543,48 @@ export class GeometricUtils {
         return false;
     }
 
+    /** Squared distance from a point to a line segment, endpoints included. */
+    public static point_segment_distance_squared(
+        point: Point2D,
+        kp1: Point2D,
+        kp2: Point2D,
+    ): number {
+        const dx: number = kp2[0] - kp1[0];
+        const dy: number = kp2[1] - kp1[1];
+        const len_sq: number = dx * dx + dy * dy;
+        let t: number = 0;
+        if (len_sq > 0) {
+            t = ((point[0] - kp1[0]) * dx + (point[1] - kp1[1]) * dy) / len_sq;
+            t = Math.max(0, Math.min(1, t));
+        }
+        const nx: number = kp1[0] + t * dx - point[0];
+        const ny: number = kp1[1] + t * dy - point[1];
+        return nx * nx + ny * ny;
+    }
+
+    /** Whether a point falls within `threshold` of a polyline's path. */
+    public static point_is_near_polyline(
+        point: Point2D,
+        polyline: ULabelSpatialPayload2D,
+        threshold: number,
+    ): boolean {
+        if (polyline.length === 0) return false;
+        const threshold_sq: number = threshold * threshold;
+        if (polyline.length === 1) {
+            const dx: number = polyline[0][0] - point[0];
+            const dy: number = polyline[0][1] - point[1];
+            return dx * dx + dy * dy <= threshold_sq;
+        }
+        for (let i = 0; i < polyline.length - 1; i++) {
+            if (
+                GeometricUtils.point_segment_distance_squared(point, polyline[i], polyline[i + 1]) <= threshold_sq
+            ) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     // Convert a bbox to a simple polygon by adding the last point
     public static bbox_to_simple_polygon(
         bbox: ULabelSpatialPayload2D,
diff --git a/src/index.js b/src/index.js
index c3d6bd68..5dd07d5c 100644
--- a/src/index.js
+++ b/src/index.js
@@ -3658,11 +3658,29 @@ export class ULabel {
             current_subtask["state"]["visible_dialogs"][esid]["top"] = new_top;
             // Decide confidence card position from the un-offset cbox so it stays stable during moves.
             // Account for annbox scroll: what matters is the visible position, not the image-space position.
-            const cbox_center_y_imwrap = ((cbox["tly"] + cbox["bry"]) / 2) * this.state["zoom_val"];
             const scroll_top = $("#" + this.config["annbox_id"]).scrollTop() || 0;
             const conf_id = `global_annotation_confidence__${subtask_key}`;
-            const flip_below = (cbox_center_y_imwrap - scroll_top) < 100;
-            $(`#${conf_id}`).css("margin-top", flip_below ? "-1em" : "-9.5em");
+            const conf_jq = $(`#${conf_id}`);
+            // The dialog container is CSS-scaled about the anchor, so a margin
+            // set here lands `scale` times as far. `outerHeight` is already in
+            // that local space; the cbox measurements are in screen px.
+            const es_el = esjq[0];
+            const scale = es_el.offsetWidth > 0 ?
+                es_el.getBoundingClientRect().width / es_el.offsetWidth :
+                1;
+            const card_height = conf_jq.outerHeight() || 0;
+            const gap = 10;
+            // The anchor is the box centre, so clear it by half the on-screen
+            // box height or the card covers whatever is being hovered.
+            const half_height = Math.abs(cbox["bry"] - cbox["tly"]) / 2 * this.state["zoom_val"];
+            const box_top_visible = (cbox["tly"] * this.state["zoom_val"]) - scroll_top;
+            const flip_below = box_top_visible - (card_height * scale) - gap < 0;
+            conf_jq.css(
+                "margin-top",
+                flip_below ?
+                    `${(half_height + gap) / scale}px` :
+                    `${-((half_height + gap) / scale + card_height)}px`,
+            );
             this.reposition_dialogs();
             idd_x = (cbox["tlx"] + cbox["brx"] + 2 * diffX) / 2;
             idd_y = (cbox["tly"] + cbox["bry"] + 2 * diffY) / 2;
@@ -6236,6 +6254,14 @@ export class ULabel {
         let minsize = Infinity;
         let found_containing_annotation = false;
         const active_class_layer = this.get_current_subtask()["state"]["active_class_layer"] ?? null;
+        // A read-only subtask has nothing to grab, so the near-miss fallback
+        // below is just noise: require a real hit wherever one can be tested.
+        const require_exact_hit = this.is_current_subtask_read_only();
+        // One cursor pixel spans several image pixels when zoomed out, so the
+        // exact tests get that much slack. Without it a thin mask or the edge
+        // of a large one is unhittable: the pixel under the cursor is empty
+        // even though the downscaled render looks filled there.
+        const slack = 2 / this.get_empirical_scale();
         // TODO(3d)
         for (let edi = 0; edi < this.get_current_subtask()["annotations"]["ordering"].length; edi++) {
             const annotation_id = this.get_current_subtask()["annotations"]["ordering"][edi];
@@ -6273,6 +6299,9 @@ export class ULabel {
                 (this.state["current_frame"] <= cbox["brz"])
             ) {
                 let is_a_containing_annotation = false;
+                // Whether this spatial type can be hit-tested against its real
+                // boundary rather than just its containing box.
+                let has_exact_test = true;
                 let boxsize = (cbox["brx"] - cbox["tlx"]) * (cbox["bry"] - cbox["tly"]);
                 switch (spatial_type) {
                     case "polygon":
@@ -6292,14 +6321,28 @@ export class ULabel {
                             is_a_containing_annotation = true;
                         }
                         break;
+                    case "polyline":
+                        // Within the drawn stroke of the line itself
+                        if (GeometricUtils.point_is_near_polyline(
+                            [gblx, gbly],
+                            annotation["spatial_payload"],
+                            (annotation["line_size"] ?? this.get_subtask_line_size()) / 2 + slack,
+                        )) {
+                            is_a_containing_annotation = true;
+                        }
+                        break;
                     case "bitmask":
                         // The mouse must be over a painted pixel of the mask
-                        if (this.get_bitmask(annotation).get_pixel(Math.round(gblx), Math.round(gbly))) {
+                        if (this.get_bitmask(annotation).has_foreground_in_circle(
+                            Math.round(gblx),
+                            Math.round(gbly),
+                            slack,
+                        )) {
                             is_a_containing_annotation = true;
                         }
                         break;
                     default:
-
+                        has_exact_test = false;
                         break;
                 }
 
@@ -6321,7 +6364,7 @@ export class ULabel {
                             };
                         }
                     }
-                } else if (boxsize < minsize) {
+                } else if (!(require_exact_hit && has_exact_test) && boxsize < minsize) {
                     ret["candidate_ids"].push(annotation_id);
                     minsize = boxsize;
                     ret["best"] = {
@@ -6452,18 +6495,31 @@ export class ULabel {
 
     // ================= Mouse event interpreters =================
 
+    /**
+     * Page coordinates of the annbox's content origin, which is where the image
+     * starts. `offset()` gives the border box, so any border on the annbox would
+     * otherwise shift every position by its width.
+     */
+    get_annbox_content_origin() {
+        const annbox = $("#" + this.config["annbox_id"]);
+        const offset = annbox.offset();
+        const el = annbox[0];
+        return {
+            left: offset.left + (el?.clientLeft ?? 0) - annbox.scrollLeft(),
+            top: offset.top + (el?.clientTop ?? 0) - annbox.scrollTop(),
+        };
+    }
+
     // Get the mouse position on the screen
     get_global_mouse_x(mouse_event) {
         const scale = this.get_empirical_scale();
-        const annbox = $("#" + this.config["annbox_id"]);
-        const raw = (mouse_event.pageX - annbox.offset().left + annbox.scrollLeft()) / scale;
+        const raw = (mouse_event.pageX - this.get_annbox_content_origin().left) / scale;
         return raw;
     }
 
     get_global_mouse_y(mouse_event) {
         const scale = this.get_empirical_scale();
-        const annbox = $("#" + this.config["annbox_id"]);
-        const raw = (mouse_event.pageY - annbox.offset().top + annbox.scrollTop()) / scale;
+        const raw = (mouse_event.pageY - this.get_annbox_content_origin().top) / scale;
         return raw;
     }
 
@@ -6491,16 +6547,14 @@ export class ULabel {
 
     get_global_element_center_x(jqel) {
         const scale = this.get_empirical_scale();
-        const annbox = $("#" + this.config["annbox_id"]);
-        const raw = (jqel.offset().left + jqel.width() / 2 - annbox.offset().left + annbox.scrollLeft()) / scale;
+        const raw = (jqel.offset().left + jqel.width() / 2 - this.get_annbox_content_origin().left) / scale;
         // return Math.round(raw);
         return raw;
     }
 
     get_global_element_center_y(jqel) {
         const scale = this.get_empirical_scale();
-        const annbox = $("#" + this.config["annbox_id"]);
-        const raw = (jqel.offset().top + jqel.height() / 2 - annbox.offset().top + annbox.scrollTop()) / scale;
+        const raw = (jqel.offset().top + jqel.height() / 2 - this.get_annbox_content_origin().top) / scale;
         // return Math.round();
         return raw;
     }
@@ -6829,10 +6883,11 @@ export class ULabel {
                 class_name = class_def.name;
             }
         }
+        const display_name = this.config.annotation_display_name_resolver?.(active_annotation) ?? class_name;
 
         // Update the display dialog
         const global_id = `global_annotation_confidence__${subtask_key}`;
-        $(`#${global_id} .annotation-confidence-classname`).text(class_name);
+        $(`#${global_id} .annotation-confidence-classname`).text(display_name);
         $(`#${global_id} .annotation-confidence-value`).text(`Confidence: ${confidence.toFixed(2)}`);
     }