diff --git a/client/dive-common/constants.ts b/client/dive-common/constants.ts index 38feece98..d94d5ff6f 100644 --- a/client/dive-common/constants.ts +++ b/client/dive-common/constants.ts @@ -90,6 +90,16 @@ const largeImageTypes = [ 'image/ntf', ]; +/** Extension-only formats for basic image sequences (aligned with server validImageFormats). */ +const basicImageFileExtensions = [ + 'png', + 'jpg', + 'jpeg', + 'sgi', + 'bmp', + 'pgm', +]; + /** Extension-only formats for large-image uploads (aligned with server validLargeImageFormats). */ const largeImageFileExtensions = [ 'nitf', @@ -251,6 +261,7 @@ export { getImageSequenceFileAccept, inputAnnotationTypes, largeImageTypes, + basicImageFileExtensions, largeImageFileExtensions, largeImageDesktopTypes, largeImageWebAccept, diff --git a/client/dive-common/frameMetadata/csvTokenizer.ts b/client/dive-common/frameMetadata/csvTokenizer.ts new file mode 100644 index 000000000..1515b916b --- /dev/null +++ b/client/dive-common/frameMetadata/csvTokenizer.ts @@ -0,0 +1,68 @@ +export type DelimitedTableDelimiter = ',' | '\t'; + +// Keep tokenization node-free because frame metadata parsing runs in both Electron and the browser +// renderer. The parser is lenient with bare quotes because field logs commonly contain units such +// as `5"`. Cells are trimmed here so every consumer sees one canonical cell value. +export function parseDelimitedRows(text: string, delimiter: DelimitedTableDelimiter): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ''; + let inQuotedField = false; + let atFieldStart = true; + let rowHasContent = false; + const endField = () => { + row.push(field.trim()); + field = ''; + atFieldStart = true; + }; + const endRow = () => { + endField(); + rows.push(row); + row = []; + rowHasContent = false; + }; + const { length } = text; + let i = 0; + while (i < length) { + const ch = text[i]; + if (inQuotedField) { + if (ch === '"') { + if (text[i + 1] === '"') { + field += '"'; + i += 2; + } else { + inQuotedField = false; + i += 1; + } + } else { + field += ch; + i += 1; + } + } else if (ch === '"' && atFieldStart) { + inQuotedField = true; + atFieldStart = false; + rowHasContent = true; + i += 1; + } else if (ch === delimiter) { + rowHasContent = true; + endField(); + i += 1; + } else if (ch === '\n' || ch === '\r') { + if (ch === '\r' && text[i + 1] === '\n') { + i += 1; + } + endRow(); + i += 1; + } else { + field += ch; + atFieldStart = false; + rowHasContent = true; + i += 1; + } + } + if (rowHasContent || field.length > 0 || row.length > 0) { + endField(); + rows.push(row); + } + return rows; +} diff --git a/client/dive-common/frameMetadata/join.spec.ts b/client/dive-common/frameMetadata/join.spec.ts new file mode 100644 index 000000000..0915d2012 --- /dev/null +++ b/client/dive-common/frameMetadata/join.spec.ts @@ -0,0 +1,275 @@ +/// + +import { extractCounter, resolveTableToFrames } from './join'; +import type { ResolvedCameraFrameMetadata } from './join'; +import { parseFrameMetadataTable } from './parser'; +import { buildFrameAlignmentIndex } from './resolve'; +import type { FrameMetadataFrameContext } from './resolve'; + +function imageFrameContext(mediaNames: string[]): FrameMetadataFrameContext { + return { mediaType: 'image-sequence', mediaNames }; +} + +function join(text: string, context: FrameMetadataFrameContext, sourceName?: string) { + const table = parseFrameMetadataTable(text); + if (table === null) { + throw new Error('expected a parsable table'); + } + return resolveTableToFrames(table, buildFrameAlignmentIndex(context), sourceName); +} + +/** Joined payload for an image sequence, or null when no tier matched. */ +function joinImages( + text: string, + mediaNames: string[], + sourceName?: string, +): ResolvedCameraFrameMetadata | null { + const result = join(text, imageFrameContext(mediaNames), sourceName); + return result.status === 'matched' ? result.parsed : null; +} + +describe('filename joins', () => { + it('emits DIVE frame numbers and preserves source column order', () => { + const parsed = joinImages( + 'filename,3,1,2\nimg002.png,c,a,b\nimg001.png,cc,aa,bb\n', + ['img001.png', 'img002.png'], + 'frame-metadata.csv', + ); + + expect(parsed?.sourceName).toBe('frame-metadata.csv'); + expect(parsed?.columns).toEqual(['filename', '3', '1', '2']); + expect(parsed?.records[0]).toEqual(['img001.png', 'cc', 'aa', 'bb']); + expect(parsed?.records[1]).toEqual(['img002.png', 'c', 'a', 'b']); + }); + + it('keeps the first row for a duplicate filename', () => { + const parsed = joinImages( + 'filename,depth\nimg001.png,10\nimg001.png,99\n', + ['img001.png'], + ); + + expect(parsed?.records[0]).toEqual(['img001.png', '10']); + }); + + it('matches media stored with a different image extension', () => { + const parsed = joinImages( + 'filename,depth\nimg001.gif,10\nimg002.avif,12\n', + ['img001.tif', 'img002.tif'], + ); + + expect(parsed?.records[0]).toEqual(['img001.gif', '10']); + expect(parsed?.records[1]).toEqual(['img002.avif', '12']); + }); + + it('selects a camera-local filename column from a dual-camera table', () => { + const text = [ + 'port_image,depth,starboard_image', + 'port001.tif,10,star001.tif', + 'port002.tif,12,star002.tif', + '', + ].join('\n'); + const port = joinImages(text, ['port001.tif', 'port002.tif']); + const starboard = joinImages(text, ['star002.tif', 'star001.tif']); + + expect(port?.records[0]).toEqual(['port001.tif', '10', 'star001.tif']); + expect(starboard?.records[0]).toEqual(['port002.tif', '12', 'star002.tif']); + expect(starboard?.records[1]).toEqual(['port001.tif', '10', 'star001.tif']); + }); + + it('joins on the leftmost matching column and keeps the rest as data', () => { + const parsed = joinImages( + 'primary,secondary,depth\na.png,a.png,1\nb.png,b.png,2\nc.png,qq.png,3\n', + ['a.png', 'b.png', 'c.png'], + ); + + expect(parsed?.columns).toEqual(['primary', 'secondary', 'depth']); + expect(parsed?.records[2]).toEqual(['c.png', 'qq.png', '3']); + }); + + it('takes the leftmost column when two columns both name this camera', () => { + // Only reachable when one dataset holds both cameras' media; a real multicamera dataset + // resolves each camera against its own media, where the sibling column matches nothing. + const parsed = joinImages( + 'left_image,right_image,depth\na.png,b.png,1\nb.png,a.png,2\n', + ['a.png', 'b.png'], + ); + + expect(parsed?.records[0]).toEqual(['a.png', 'b.png', '1']); + expect(parsed?.records[1]).toEqual(['b.png', 'a.png', '2']); + }); + + it('keeps filename precedence for a VIAME-shaped source with a different frame field', () => { + const parsed = joinImages('index,image,frame,depth\n1,img001.png,100,12.5\n', ['img001.png']); + + expect(parsed?.records[0]).toEqual(['1', 'img001.png', '100', '12.5']); + }); + + it('keeps both columns when a header name repeats', () => { + const parsed = joinImages('filename,depth,depth\nimg001.png,10,20\n', ['img001.png']); + + expect(parsed?.columns).toEqual(['filename', 'depth', 'depth']); + expect(parsed?.records[0]).toEqual(['img001.png', '10', '20']); + }); +}); + +describe('literal frame joins', () => { + it('accepts zero-based, sparse, out-of-order, and leading-zero values', () => { + const parsed = joinImages( + 'frame,depth\n03,30\n0,10\n2,20\n', + ['a.png', 'b.png', 'c.png', 'd.png'], + ); + + expect(parsed?.columns).toEqual(['frame', 'depth']); + expect(parsed?.records).toEqual({ + 0: ['0', '10'], + 2: ['2', '20'], + 3: ['03', '30'], + }); + }); + + it('skips malformed and out-of-range rows while keeping valid rows', () => { + const invalid = ['', '-1', '1.5', '+1', '9007199254740992', 'text', '3', '99']; + const text = [ + 'frame,value', + ...invalid.map((value, index) => `${value},bad-${index}`), + '1,good', + '', + ].join('\n'); + const parsed = joinImages(text, ['a.png', 'b.png', 'c.png']); + + expect(parsed?.records).toEqual({ 1: ['1', 'good'] }); + }); + + it('keeps the first row for a duplicate valid frame', () => { + const parsed = joinImages('frame,value\n1,first\n01,second\n', ['a.png', 'b.png']); + + expect(parsed?.records[1]).toEqual(['1', 'first']); + }); + + it('blocks a wholly invalid declaration without falling through to a counter', () => { + expect(join( + 'frame,source_count,value\nbad,1,a\n99,2,b\n', + imageFrameContext(['img_001.jpg', 'img_002.jpg']), + )).toEqual({ status: 'blocked', reason: 'invalid-declaration' }); + }); + + it('rejects a bare frame list', () => { + expect(joinImages('frame\n0\n1\n', ['a.png', 'b.png'])).toBeNull(); + }); + + it.each(['Frame', 'frame_index', 'frame_id', 'sample'])( + 'does not treat %s as the literal frame declaration', + (column) => { + expect(joinImages(`${column},value\n0,a\n1,b\n`, ['alpha.png', 'beta.png'])).toBeNull(); + }, + ); +}); + +describe('source image counter joins', () => { + it('extracts safe trailing decimal runs', () => { + expect(extractCounter('cam_00173')).toBe(173); + expect(extractCounter('20181101.155406.00082')).toBe(82); + expect(extractCounter('img12_34')).toBe(34); + expect(extractCounter('frame')).toBeUndefined(); + expect(extractCounter('h_1234567890123456789012')).toBeUndefined(); + }); + + it('resolves a uniquely best monotonic counter column to frame numbers', () => { + const parsed = joinImages( + 'frame_count,date,depth,pass\n173,145.00,10,1\n174,146.30,12,1\n175,147.10,14,1\n', + ['cam_00173.jpg', 'cam_00174.jpg', 'cam_00175.jpg'], + ); + + expect(parsed?.records[0]).toEqual(['173', '145.00', '10', '1']); + expect(parsed?.records[2][0]).toBe('175'); + }); + + it('accepts strictly descending frame matches', () => { + const parsed = joinImages( + 'count,value\n3,c\n2,b\n1,a\n', + ['cam_1.jpg', 'cam_2.jpg', 'cam_3.jpg'], + ); + + expect(parsed?.records[0][1]).toBe('a'); + expect(parsed?.records[2][1]).toBe('c'); + }); + + it('excludes ambiguous media counters', () => { + const parsed = joinImages( + 'count,value\n1,a\n2,b\n', + ['port_1.jpg', 'star_1.jpg', 'port_2.jpg'], + ); + + expect(parsed?.records[0]).toBeUndefined(); + expect(parsed?.records[1]).toBeUndefined(); + expect(parsed?.records[2]).toEqual(['2', 'b']); + }); + + it('disqualifies a column whose counters are claimed by two rows', () => { + expect(joinImages( + 'count,value\n1,a\n1,b\n2,c\n3,d\n', + ['cam_1.jpg', 'cam_2.jpg', 'cam_3.jpg'], + )).toBeNull(); + }); + + it('rejects a reset/reuse table while either monotonic segment resolves', () => { + const mediaNames = ['cam_1.jpg', 'cam_2.jpg', 'cam_3.jpg']; + + expect(joinImages('count,value\n1,a\n2,b\n3,c\n1,d\n2,e\n3,f\n', mediaNames)).toBeNull(); + expect(joinImages('count,value\n1,a\n2,b\n3,c\n', mediaNames)?.records[2][1]).toBe('c'); + expect(joinImages('count,value\n1,d\n2,e\n3,f\n', mediaNames)?.records[0][1]).toBe('d'); + }); + + it('blocks a non-monotonic candidate', () => { + expect(joinImages( + 'count,value\n1,a\n3,c\n2,b\n', + ['cam_1.jpg', 'cam_2.jpg', 'cam_3.jpg'], + )).toBeNull(); + }); + + it('takes the leftmost qualifying counter column', () => { + const parsed = joinImages( + 'left,right,value\n1,1,a\n2,2,b\n3,3,c\n', + ['cam_1.jpg', 'cam_2.jpg', 'cam_3.jpg'], + ); + + expect(parsed?.columns).toEqual(['left', 'right', 'value']); + expect(parsed?.records[0]).toEqual(['1', '1', 'a']); + expect(parsed?.records[2]).toEqual(['3', '3', 'c']); + }); + + it('finds frame_count in an independently authored 31-column hazard table', () => { + const hazardColumns = Array.from({ length: 26 }, (_, index) => `sensor_${index + 1}`); + const header = [ + 'date', 'pass', 'time_a', 'time_b', ...hazardColumns, 'frame_count', + ]; + const row = (counter: number, offset: number) => [ + String(20240708 + offset), + '1', + offset % 2 ? '12:00:00' : '12:00:01', + offset % 2 ? '12:00:01' : '12:00:00', + ...hazardColumns.map((_, index) => ( + index % 4 === 0 ? '' : `${6000 + offset * 100 + index}.5` + )), + String(counter), + ]; + const text = [ + header.join(','), + row(900, 0).join(','), + row(173, 1).join(','), + row(174, 2).join(','), + row(175, 3).join(','), + '', + ].join('\n'); + const parsed = joinImages(text, ['cam_00173.jpg', 'cam_00174.jpg', 'cam_00175.jpg']); + + expect(header).toHaveLength(31); + expect(parsed?.columns).toEqual(header); + expect(parsed?.records[0][30]).toBe('173'); + expect(parsed?.records[2][30]).toBe('175'); + }); + + it('still rejects a counter-only list with nothing to display', () => { + expect(joinImages('count\n1\n2\n', ['cam_1.jpg', 'cam_2.jpg'])).toBeNull(); + }); +}); diff --git a/client/dive-common/frameMetadata/join.ts b/client/dive-common/frameMetadata/join.ts new file mode 100644 index 000000000..fb223a601 --- /dev/null +++ b/client/dive-common/frameMetadata/join.ts @@ -0,0 +1,271 @@ +import { fileImageTypes } from 'dive-common/components/ImportMultiCamDialog/multicamSubfolderLayout'; +import { basicImageFileExtensions, largeImageFileExtensions } from 'dive-common/constants'; +import type { FrameMetadataRow, FrameMetadataTable } from './parser'; + +type FrameRowMatch = Map; + +/** One camera's joined frame metadata. */ +export interface ResolvedCameraFrameMetadata { + /** + * Payload column names in source order. A repeated name keeps its own column, so consumers + * address cells by index rather than by name. + */ + columns: string[]; + /** DIVE frame number -> cell values, positionally aligned with `columns`. */ + records: Record; + /** Attachment the rows came from, when the caller supplied one. */ + sourceName?: string; +} + +export interface FrameAlignmentIndex { + frameCount: number; + frameByAlignmentKey: Map; + frameByCounter?: Map; +} + +export type JoinBlockedReason = 'invalid-declaration'; + +type JoinAttempt = + | { status: 'not-applicable' } + | { status: 'matched'; parsed: ResolvedCameraFrameMetadata } + | { status: 'blocked'; reason: JoinBlockedReason }; + +// Alignment keys drop a media extension so a metadata cell like `img001.png` still matches an +// image stored as `img001.tif`. The allowlist is the union of what both platforms accept as image +// media: the web upload lists plus the desktop importer's set (which adds gif and avif). +const imageExtensions = new Set([ + ...basicImageFileExtensions, + ...largeImageFileExtensions, + ...fileImageTypes, +]); +const TRAILING_DIGITS = /(\d+)$/; +const INTEGER_CELL = /^\d+$/; + +// Split a basename like node path.extname: a leading dot is not an extension. +function splitExtension(basename: string): { stem: string; extension: string } { + const dot = basename.lastIndexOf('.'); + if (dot <= 0) { + return { stem: basename, extension: '' }; + } + return { stem: basename.slice(0, dot), extension: basename.slice(dot + 1).toLowerCase() }; +} + +export function normalizeAlignmentKey(value: string): string { + const basename = String(value).trim().split(/[\\/]/).pop() ?? ''; + const { stem, extension } = splitExtension(basename); + return imageExtensions.has(extension) ? stem : basename; +} + +export function extractCounter(stem: string): number | undefined { + const match = TRAILING_DIGITS.exec(stem); + if (match === null) { + return undefined; + } + const value = Number.parseInt(match[1], 10); + return Number.isSafeInteger(value) ? value : undefined; +} + +function cellCounter(cell: string): number | undefined { + if (!INTEGER_CELL.test(cell)) { + return undefined; + } + const value = Number.parseInt(cell, 10); + return Number.isSafeInteger(value) ? value : undefined; +} + +function columnIndices(header: string[]): number[] { + return header.map((_, index) => index); +} + +// First matching row wins for a frame; later rows claiming the same frame are ignored on every +// join tier. +function recordsFromMatches( + matches: Iterable, + rows: FrameMetadataRow[], + keptIndices: number[], +): Record { + const records: Record = {}; + Array.from(matches).forEach(([frame, rowIndex]) => { + if (!(frame in records)) { + records[frame] = keptIndices.map((columnIndex) => rows[rowIndex][columnIndex]); + } + }); + return records; +} + +function matchByFilename( + table: FrameMetadataTable, + index: FrameAlignmentIndex, + sourceName?: string, +): JoinAttempt { + const { header, rows } = table; + if (header.length === 1) { + // The lone column would be the filename itself, leaving nothing to display. + return { status: 'not-applicable' }; + } + const threshold = Math.min(2, rows.length, index.frameByAlignmentKey.size); + const hits = (columnIndex: number) => rows.reduce((total, row) => ( + row[columnIndex] && index.frameByAlignmentKey.has(normalizeAlignmentKey(row[columnIndex])) + ? total + 1 + : total + ), 0); + const qualifies = (columnIndex: number) => { + const score = hits(columnIndex); + return score >= threshold && score > 0; + }; + // Leftmost qualifying column wins. A sibling camera's filename column in a shared multicamera + // table scores nothing against this camera's media, so it never qualifies in the first place. + const winner = columnIndices(header).find(qualifies); + if (winner === undefined || columnIndices(header).every(qualifies)) { + // Either nothing names this camera's media, or every column does and there is no payload. + return { status: 'not-applicable' }; + } + + const keptIndices = columnIndices(header); + const matches = rows + .map((row, rowIndex): [number | undefined, number] => [ + index.frameByAlignmentKey.get(normalizeAlignmentKey(row[winner])), + rowIndex, + ]) + .filter((match): match is [number, number] => match[0] !== undefined); + return matches.length === 0 + ? { status: 'not-applicable' } + : { + status: 'matched', + parsed: { + columns: header, + records: recordsFromMatches(matches, rows, keptIndices), + sourceName, + }, + }; +} + +function matchByExplicitFrame( + table: FrameMetadataTable, + frameCount: number, + sourceName?: string, +): JoinAttempt { + const { header, rows } = table; + const frameColumn = header.indexOf('frame'); + if (frameColumn === -1) { + return { status: 'not-applicable' }; + } + if (header.length === 1) { + return { status: 'blocked', reason: 'invalid-declaration' }; + } + + const matches = rows + .map((row, rowIndex): [number | undefined, number] => { + const frame = cellCounter(row[frameColumn]); + return [frame !== undefined && frame < frameCount ? frame : undefined, rowIndex]; + }) + .filter((match): match is [number, number] => match[0] !== undefined); + return matches.length === 0 + ? { status: 'blocked', reason: 'invalid-declaration' } + : { + status: 'matched', + parsed: { + columns: header, + records: recordsFromMatches(matches, rows, columnIndices(header)), + sourceName, + }, + }; +} + +function isStrictlyMonotonic(frames: number[]): boolean { + if (frames.length < 2) { + return true; + } + const direction = Math.sign(frames[1] - frames[0]); + if (direction === 0) { + return false; + } + return frames.slice(1).every((frame, index) => ( + Math.sign(frame - frames[index]) === direction + )); +} + +/** + * The frame-to-row matches a counter column yields, or undefined when the column does not + * qualify as this camera's counter: too few matches, a counter claimed by two rows, or matches + * that do not advance in one direction as the rows do. + */ +function counterColumnMatches( + columnIndex: number, + rows: FrameMetadataRow[], + counterIndex: Map, + threshold: number, +): FrameRowMatch | undefined { + const matched: FrameRowMatch = new Map(); + const claimed = new Set(); + let duplicateMatch = false; + rows.forEach((row, rowIndex) => { + const counter = cellCounter(row[columnIndex]); + const frame = counter === undefined ? undefined : counterIndex.get(counter); + if (counter === undefined || frame === undefined) { + return; + } + if (claimed.has(counter)) { + duplicateMatch = true; + return; + } + claimed.add(counter); + matched.set(frame, rowIndex); + }); + if (duplicateMatch || matched.size < threshold) { + return undefined; + } + const framesInRowOrder = Array.from(matched, ([frame, rowIndex]) => ({ frame, rowIndex })) + .sort((a, b) => a.rowIndex - b.rowIndex) + .map(({ frame }) => frame); + return isStrictlyMonotonic(framesInRowOrder) ? matched : undefined; +} + +function matchByCounter( + table: FrameMetadataTable, + counterIndex: Map | undefined, + sourceName?: string, +): JoinAttempt { + const { header, rows } = table; + if (counterIndex === undefined || counterIndex.size === 0 || header.length === 1) { + // A lone counter column leaves nothing to display, so skip the scoring work entirely. + return { status: 'not-applicable' }; + } + const threshold = Math.min(2, rows.length, counterIndex.size); + // Leftmost qualifying column wins, matching the filename tier. + const matched = columnIndices(header).reduce( + (found, columnIndex) => ( + found ?? counterColumnMatches(columnIndex, rows, counterIndex, threshold) + ), + undefined, + ); + if (matched === undefined) { + return { status: 'not-applicable' }; + } + return { + status: 'matched', + parsed: { + columns: header, + records: recordsFromMatches(matched, rows, columnIndices(header)), + sourceName, + }, + }; +} + +export function resolveTableToFrames( + table: FrameMetadataTable, + frameContext: FrameAlignmentIndex, + sourceName?: string, +): JoinAttempt { + const filename = matchByFilename(table, frameContext, sourceName); + if (filename.status !== 'not-applicable') { + return filename; + } + + const explicitFrame = matchByExplicitFrame(table, frameContext.frameCount, sourceName); + if (explicitFrame.status !== 'not-applicable') { + return explicitFrame; + } + + return matchByCounter(table, frameContext.frameByCounter, sourceName); +} diff --git a/client/dive-common/frameMetadata/parser.spec.ts b/client/dive-common/frameMetadata/parser.spec.ts new file mode 100644 index 000000000..065131ad3 --- /dev/null +++ b/client/dive-common/frameMetadata/parser.spec.ts @@ -0,0 +1,134 @@ +/// + +import fs from 'fs'; +import path from 'path'; + +import isFrameMetadataSourceName from './naming'; +import { parseFrameMetadataTable } from './parser'; + +const SOURCE_NAMES_TRUTH_TABLE = path.resolve( + __dirname, + '../../../testutils/framemetadata.spec.json', +); + +describe('frame metadata table parser', () => { + it('parses comma, tab, and whitespace-delimited tables', () => { + [ + 'filename,depth\nimg001.png,10\n', + 'filename\tdepth\nimg001.png\t10\n', + 'filename depth\nimg001.png 10\n', + ].forEach((text) => { + expect(parseFrameMetadataTable(text)).toEqual({ + header: ['filename', 'depth'], + rows: [['img001.png', '10']], + }); + }); + }); + + it('trims a BOM, headers, and cells', () => { + expect(parseFrameMetadataTable(' filename , depth \n img001.png , 10 \n')).toEqual({ + header: ['filename', 'depth'], + rows: [['img001.png', '10']], + }); + }); + + it('skips a leading comment block and sniffs the first data line', () => { + expect(parseFrameMetadataTable( + '# Position (lat, lon) log\nfilename\tdepth\nimg001.png\t10\n', + )).toEqual({ + header: ['filename', 'depth'], + rows: [['img001.png', '10']], + }); + }); + + it('does not promote a comment to the header', () => { + expect(parseFrameMetadataTable( + '# filename,depth,heading\nimg001.png,10,180\n', + )).toBeNull(); + }); + + it('drops empty header cells and rows', () => { + expect(parseFrameMetadataTable( + ',,,\n,filename,depth,\n, img001.png ,10,\n,,,\n', + )).toEqual({ + header: ['filename', 'depth'], + rows: [['img001.png', '10']], + }); + }); + + it('keeps a bare double quote and Windows path text', () => { + expect(parseFrameMetadataTable( + 'filename,depth\nimages\\img001.png,5"\n', + )).toEqual({ + header: ['filename', 'depth'], + rows: [['images\\img001.png', '5"']], + }); + }); + + it('keeps a repeated header name as its own column', () => { + expect(parseFrameMetadataTable('filename,depth,depth\nimg001.png,10,20\n')).toEqual({ + header: ['filename', 'depth', 'depth'], + rows: [['img001.png', '10', '20']], + }); + }); + + it('rejects empty, comment-only, and NUL-poisoned input', () => { + expect(parseFrameMetadataTable('')).toBeNull(); + expect(parseFrameMetadataTable('# one\n# two\n')).toBeNull(); + expect(parseFrameMetadataTable('filename,alt\0\nimg001.png,42\0\n')).toBeNull(); + }); + + it('keeps a cell of any size', () => { + const value = 'x'.repeat(400000); + const table = parseFrameMetadataTable(`filename,notes\nimg001.png,${value}\n`); + + expect(table?.rows[0]).toEqual(['img001.png', value]); + }); + + it('keeps a header cell that names an Object prototype member', () => { + const table = parseFrameMetadataTable('filename,__proto__\nimg001.png,value\n'); + + expect(table?.header).toEqual(['filename', '__proto__']); + expect(table?.rows[0]).toEqual(['img001.png', 'value']); + }); +}); + +describe('delimiter sniffing', () => { + it('ignores commas inside quoted header cells of a tab-delimited table', () => { + expect(parseFrameMetadataTable( + '"Pos (lat, lon)"\t"Vel (x, y)"\tdepth\n"1, 2"\t"3, 4"\t10\n', + )).toEqual({ + header: ['Pos (lat, lon)', 'Vel (x, y)', 'depth'], + rows: [['1, 2', '3, 4', '10']], + }); + }); + + it('keeps commas inside quoted cells of a comma-delimited table', () => { + expect(parseFrameMetadataTable( + 'filename,"notes, more"\nimg001.png,"a, b"\n', + )).toEqual({ + header: ['filename', 'notes, more'], + rows: [['img001.png', 'a, b']], + }); + }); + + it('resolves an equal unquoted count in favour of tabs', () => { + expect(parseFrameMetadataTable('name\tnotes,extra\nimg001.png\t10,20\n')).toEqual({ + header: ['name', 'notes,extra'], + rows: [['img001.png', '10,20']], + }); + }); +}); + +describe('shared frame-metadata naming', () => { + it('matches the shared source-name predicate truth table', () => { + const truthTable = JSON.parse( + fs.readFileSync(SOURCE_NAMES_TRUTH_TABLE, 'utf-8'), + ) as Record; + + expect(Object.keys(truthTable).length).toBeGreaterThan(0); + Object.entries(truthTable).forEach(([name, expected]) => { + expect(isFrameMetadataSourceName(name)).toBe(expected); + }); + }); +}); diff --git a/client/dive-common/frameMetadata/parser.ts b/client/dive-common/frameMetadata/parser.ts new file mode 100644 index 000000000..488f2f448 --- /dev/null +++ b/client/dive-common/frameMetadata/parser.ts @@ -0,0 +1,134 @@ +import { parseDelimitedRows } from './csvTokenizer'; +import type { DelimitedTableDelimiter } from './csvTokenizer'; + +// Shared by the desktop backend and the web client. Keep this node-free so the same parser runs +// in Electron and in the browser renderer. + +/** One row of cells, positionally aligned with `FrameMetadataTable.header`. */ +export type FrameMetadataRow = string[]; + +export interface FrameMetadataTable { + /** + * Column names in file order. A repeated name is kept as its own column: rows are positional, + * so consumers must address cells by index rather than by name. + */ + header: string[]; + rows: FrameMetadataRow[]; +} + +function dropLeadingCommentRows(rawRows: string[][]): string[][] { + let index = 0; + while (index < rawRows.length && rawRows[index].length > 0 && rawRows[index][0].startsWith('#')) { + index += 1; + } + return rawRows.slice(index); +} + +function buildTable(headerCells: string[], dataRows: string[][]): FrameMetadataTable { + // Empty header cells usually come from pandas indexes or trailing commas. Short rows are padded + // to the header width here, so joins can address a cell by index without a fallback. + const keptIndices = headerCells + .map((cell, index) => (cell.length > 0 ? index : -1)) + .filter((index) => index >= 0); + if (keptIndices.length === 0) { + return { header: [], rows: [] }; + } + return { + header: keptIndices.map((index) => headerCells[index]), + rows: dataRows + .filter((row) => row.some((cell) => cell.length > 0)) + .map((row) => keptIndices.map((index) => row[index] ?? '')), + }; +} + +export function parseFrameMetadataTable(text: string): FrameMetadataTable | null { + if (text.includes('\0')) { + return null; + } + const content = text.charCodeAt(0) === 0xFEFF ? text.slice(1) : text; + const rawRows = readRows(content); + if (rawRows.length === 0) { + return null; + } + + const body = dropLeadingCommentRows(rawRows); + if (body.length === 0) { + return null; + } + const table = buildTable(body[0], body.slice(1)); + return table.header.length > 0 && table.rows.length > 0 ? table : null; +} + +function readRows(text: string): string[][] { + const sniff = sniffLine(text); + if (sniff === null) { + return []; + } + + const delimiter = sniffDelimiter(sniff); + if (delimiter === null) { + return text + .split(/\r?\n/) + .filter((line) => line.trim().length > 0) + .map((line) => line.trim().split(/\s+/)); + } + + // Drop delimiter-only rows before header selection; otherwise `,,,` becomes the header. + return parseDelimitedRows(text, delimiter).filter((row) => row.some((cell) => cell.length > 0)); +} + +// Ignore leading prose comments while sniffing so commas inside prose do not override TSV data. +function sniffLine(text: string): string | null { + const lines = text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + return lines.find((line) => !line.startsWith('#')) ?? lines[0] ?? null; +} + +// Only delimiters outside quoted cells vote: a quoted header cell such as `"Position (lat, lon)"` +// contains a comma but does not make the line comma-separated. Cell starts are delimiter-agnostic +// (start of line, or just past a comma or tab) because the delimiter is what we are sniffing for. +function countUnquotedDelimiters(line: string): { commas: number; tabs: number } { + let commas = 0; + let tabs = 0; + let inQuotedCell = false; + let atCellStart = true; + let i = 0; + while (i < line.length) { + const ch = line[i]; + if (inQuotedCell) { + if (ch === '"' && line[i + 1] === '"') { + i += 1; + } else if (ch === '"') { + inQuotedCell = false; + } + } else if (ch === '"' && atCellStart) { + inQuotedCell = true; + atCellStart = false; + } else if (ch === ',') { + commas += 1; + atCellStart = true; + } else if (ch === '\t') { + tabs += 1; + atCellStart = true; + } else { + atCellStart = false; + } + i += 1; + } + return { commas, tabs }; +} + +// Tabs win a tie because the two ways to lose are not equally likely, not because a stray tab is +// harmless: cells are only edge-trimmed, so an interior tab does survive tokenization. An unquoted +// TSV header routinely carries commas inside field names (`Pos (lat, lon)\tVel (x, y)\tdepth` +// ties 2-2 and must sniff as TSV), whereas a literal tab inside a CSV header name is vanishingly +// rare. `Poslon,depth` is the case this rule gets wrong, and it is the cheaper one to lose. +function sniffDelimiter(line: string): DelimitedTableDelimiter | null { + const { commas, tabs } = countUnquotedDelimiters(line); + if (tabs > 0 && tabs >= commas) { + return '\t'; + } + return commas > 0 ? ',' : null; +} diff --git a/client/dive-common/frameMetadata/resolve.spec.ts b/client/dive-common/frameMetadata/resolve.spec.ts new file mode 100644 index 000000000..cf16bd1ee --- /dev/null +++ b/client/dive-common/frameMetadata/resolve.spec.ts @@ -0,0 +1,154 @@ +/// + +import { buildFrameAlignmentIndex, resolveCameraAttachment } from './resolve'; +import type { FrameMetadataFrameContext } from './resolve'; + +function imageFrameContext(mediaNames: string[]): FrameMetadataFrameContext { + return { mediaType: 'image-sequence', mediaNames }; +} + +function resolveImages(text: string, mediaNames: string[], sourceName?: string) { + return resolveCameraAttachment( + text, + buildFrameAlignmentIndex(imageFrameContext(mediaNames)), + sourceName, + ); +} + +function resolvedMetadata(result: ReturnType) { + if (result.status !== 'resolved') { + throw new Error(`expected a resolved attachment, got ${result.status}`); + } + return result.metadata; +} + +describe('buildFrameAlignmentIndex', () => { + it('normalizes an ordered media list to frame numbers', () => { + const index = buildFrameAlignmentIndex(imageFrameContext(['img001.png', 'nested/img002.png'])); + + expect(index.frameCount).toBe(2); + expect(index.frameByAlignmentKey).toEqual(new Map([['img001', 0], ['img002', 1]])); + expect(index.frameByCounter).toEqual(new Map([[1, 0], [2, 1]])); + }); + + it('keeps later duplicate basenames but excludes repeated counters', () => { + const index = buildFrameAlignmentIndex(imageFrameContext([ + 'a/img001.png', 'b/img001.png', 'other001.png', 'img002.png', + ])); + + expect(index.frameByAlignmentKey.get('img001')).toBe(1); + expect(index.frameByAlignmentKey.get('img002')).toBe(3); + expect(index.frameByCounter?.has(1)).toBe(false); + expect(index.frameByCounter?.get(2)).toBe(3); + }); + + it('builds a frame-bound-only index for video', () => { + const index = buildFrameAlignmentIndex({ mediaType: 'video', frameCount: 3 }); + + expect(index).toEqual({ + frameCount: 3, + frameByAlignmentKey: new Map(), + }); + expect(index.frameByCounter).toBeUndefined(); + }); +}); + +describe('resolveCameraAttachment', () => { + it('distinguishes invalid input from an unmatched valid table', () => { + expect(resolveImages('', ['img001.png']).status).toBe('invalid'); + expect(resolveImages('station,depth\nA,10\n', ['img001.png']).status).toBe('unmatched'); + }); + + it('reports why a blocked table was rejected', () => { + expect(resolveImages( + 'frame,depth\nbad,1\n99,2\n', + ['a.png', 'b.png'], + )).toEqual({ status: 'unmatched', reason: 'invalid-declaration' }); + }); + + it('resolves filename rows into the compact payload', () => { + const metadata = resolvedMetadata(resolveImages( + 'filename,depth\nimg001.png,10\nimg002.png,12\n', + ['img001.png', 'img002.png'], + 'frame_metadata.csv', + )); + + expect(metadata.columns).toEqual(['filename', 'depth']); + expect(metadata.records[0]).toEqual(['img001.png', '10']); + expect(metadata.records[1]).toEqual(['img002.png', '12']); + expect(metadata.sourceName).toBe('frame_metadata.csv'); + }); + + it('keeps numeric-named columns in file order', () => { + const metadata = resolvedMetadata(resolveImages( + 'filename,3,1,2\nimg001.png,c,a,b\n', + ['img001.png'], + )); + + expect(metadata.columns).toEqual(['filename', '3', '1', '2']); + expect(metadata.records[0]).toEqual(['img001.png', 'c', 'a', 'b']); + }); + + it('bounds image-sequence literal frame rows by the media list length', () => { + const metadata = resolvedMetadata(resolveImages( + 'frame,depth\n2,12\n3,past-the-end\n0,10\n', + ['a.png', 'b.png', 'c.png'], + )); + + expect(metadata.records[0]).toEqual(['0', '10']); + expect(metadata.records[1]).toBeUndefined(); + expect(metadata.records[2]).toEqual(['2', '12']); + expect(metadata.records[3]).toBeUndefined(); + }); + + it('bounds video literal frame rows by the declared frame count', () => { + const metadata = resolvedMetadata(resolveCameraAttachment( + 'frame,value\n2,last\nbad,invalid\n0,first\n3,out\n2,duplicate\n', + buildFrameAlignmentIndex({ mediaType: 'video', frameCount: 3 }), + 'frame-metadata.csv', + )); + + expect(metadata.records).toEqual({ + 0: ['0', 'first'], + 2: ['2', 'last'], + }); + }); + + it.each(['filename', 'count', 'frame_index', 'frame_id', 'sample'])( + 'does not use video %s values as frame identity', + (column) => { + const result = resolveCameraAttachment( + `${column},value\n0,a\n2,b\n`, + buildFrameAlignmentIndex({ mediaType: 'video', frameCount: 3 }), + 'frame-metadata.csv', + ); + + expect(result.status).toBe('unmatched'); + }, + ); + + it('selects a per-camera filename column from a shared multicamera source', () => { + const text = [ + 'port_image,depth,starboard_image', + 'port001.tif,10,star001.tif', + 'port002.tif,12,star002.tif', + '', + ].join('\n'); + const left = resolvedMetadata(resolveImages(text, ['port001.tif', 'port002.tif'])); + const right = resolvedMetadata(resolveImages(text, ['star002.tif', 'star001.tif'])); + + expect(left.records[0]).toEqual(['port001.tif', '10', 'star001.tif']); + expect(right.records[0]).toEqual(['port002.tif', '12', 'star002.tif']); + expect(right.records[1]).toEqual(['port001.tif', '10', 'star001.tif']); + }); + + it('binds one shared counter log independently to each camera', () => { + const shared = 'frame_count,depth\n173,10\n174,12\n'; + const port = resolvedMetadata(resolveImages(shared, ['P_00173.jpg', 'P_00174.jpg'])); + const star = resolvedMetadata(resolveImages(shared, ['S_00173.jpg', 'S_00174.jpg'])); + + expect(port.records[0]).toEqual(['173', '10']); + expect(star.records[0]).toEqual(['173', '10']); + expect(star.records[1]).toEqual(['174', '12']); + }); +}); diff --git a/client/dive-common/frameMetadata/resolve.ts b/client/dive-common/frameMetadata/resolve.ts new file mode 100644 index 000000000..29b4c7c1f --- /dev/null +++ b/client/dive-common/frameMetadata/resolve.ts @@ -0,0 +1,89 @@ +import { + extractCounter, + normalizeAlignmentKey, + resolveTableToFrames, +} from './join'; +import type { + FrameAlignmentIndex, + JoinBlockedReason, + ResolvedCameraFrameMetadata, +} from './join'; +import { parseFrameMetadataTable } from './parser'; + +/** + * A camera's media defines its own frame bound: the ordered media list for an image sequence, + * the declared frame count for a video. + */ +export type FrameMetadataFrameContext = + | { + mediaType: 'image-sequence'; + mediaNames: string[]; + } + | { + mediaType: 'video'; + frameCount: number; + }; + +type CameraAttachmentResolution = + | { status: 'resolved'; metadata: ResolvedCameraFrameMetadata } + | { status: 'invalid' } + | { status: 'unmatched'; reason?: JoinBlockedReason }; + +// The read path must tolerate duplicate basenames because rejecting here would hide all metadata +// for the camera. Later media entries win for consistency with the ordered media list. +// +// Alongside the filename index, derive a frame-number-valued counter index. Repeated media counters +// are excluded because a trailing digit run is weaker evidence than a complete basename. +export function buildFrameAlignmentIndex(context: FrameMetadataFrameContext): FrameAlignmentIndex { + if (context.mediaType === 'video') { + return { + frameCount: context.frameCount, + frameByAlignmentKey: new Map(), + }; + } + const { mediaNames } = context; + const alignmentKeys = mediaNames.map(normalizeAlignmentKey); + const framesByCounter = new Map(); + alignmentKeys.forEach((key, frame) => { + const counter = extractCounter(key); + if (counter !== undefined) { + framesByCounter.set(counter, [...(framesByCounter.get(counter) ?? []), frame]); + } + }); + const frameByCounter = new Map(); + framesByCounter.forEach((frames, counter) => { + if (frames.length === 1) { + frameByCounter.set(counter, frames[0]); + } + }); + return { + frameCount: mediaNames.length, + frameByAlignmentKey: new Map(alignmentKeys.map((key, frame) => [key, frame])), + frameByCounter, + }; +} + +export function resolveCameraAttachment( + text: string, + index: FrameAlignmentIndex, + sourceName?: string, +): CameraAttachmentResolution { + const table = parseFrameMetadataTable(text); + if (table === null) { + return { status: 'invalid' }; + } + const result = resolveTableToFrames(table, index, sourceName); + if (result.status === 'matched') { + return { status: 'resolved', metadata: result.parsed }; + } + return result.status === 'blocked' + ? { status: 'unmatched', reason: result.reason } + : { status: 'unmatched' }; +} + +// Re-exported so a consumer of `resolveCameraAttachment` names the payload type from the same +// module it calls, rather than reaching past this one into `./join`. `dive-common/use/ +// useFrameMetadata` accumulates these per-camera results itself, keyed by camera: one camera at a +// time is the whole contract, because a shared multicamera source binds to each camera's own +// media independently. +export type { ResolvedCameraFrameMetadata }; diff --git a/client/dive-common/use/index.ts b/client/dive-common/use/index.ts index e35d1597e..3c6fa0349 100644 --- a/client/dive-common/use/index.ts +++ b/client/dive-common/use/index.ts @@ -2,10 +2,12 @@ import useModeManager from './useModeManager'; import useSave from './useSave'; import useRequest from './useRequest'; import { useLassoMode } from './useLassoMode'; +import { useFrameMetadata } from './useFrameMetadata'; export { useModeManager, useRequest, useSave, useLassoMode, + useFrameMetadata, }; diff --git a/client/dive-common/use/useFrameMetadata.spec.ts b/client/dive-common/use/useFrameMetadata.spec.ts new file mode 100644 index 000000000..457affc4e --- /dev/null +++ b/client/dive-common/use/useFrameMetadata.spec.ts @@ -0,0 +1,709 @@ +import { effectScope, nextTick, ref } from 'vue'; + +// eslint-disable-next-line import/no-extraneous-dependencies -- Vitest is only used in tests +import { + beforeEach, describe, expect, it, vi, +} from 'vitest'; + +import type { FrameMetadataSourcesResponse } from '../apispec'; +import type { FrameMetadataFrameContext } from '../frameMetadata/resolve'; +import { resetFrameMetadataSessionCache, useFrameMetadata } from './useFrameMetadata'; + +// Drain Vue's watcher scheduler and the promise/microtask + macrotask queues so a dataset switch, +// an async source load/resolve, and any deferred per-camera pass all settle before we assert. +async function settle() { + for (let i = 0; i < 4; i += 1) { + // eslint-disable-next-line no-await-in-loop + await nextTick(); + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, 0); }); + } +} + +/** + * Stand in for the component that owns the composable in production: `onScopeDispose` binds to + * whatever effect scope is active, which for a real panel is its `setup()`. Every instance here + * gets one so the composable never has to make an allowance for its tests. The scope is left + * running -- a test that exercises teardown stops its own. + */ +function mountFrameMetadata(options: Parameters[0]) { + const metadata = effectScope().run(() => useFrameMetadata(options)); + if (metadata === undefined) { + throw new Error('effect scope did not run'); + } + return metadata; +} + +function frameContextFromMediaNames( + getMediaNames: (camera: string) => string[] | undefined, +): (camera: string) => FrameMetadataFrameContext | undefined { + return (camera: string) => { + const mediaNames = getMediaNames(camera); + if (mediaNames === undefined || mediaNames.length === 0) { + return undefined; + } + return { mediaType: 'image-sequence', mediaNames }; + }; +} + +describe('useFrameMetadata', () => { + // The module-level session cache is intentionally global; reset it so no dataset id or + // resolved payload from one test's mocks leaks into the next. + beforeEach(() => { + resetFrameMetadataSessionCache(); + }); + + it('discards a stale response after the dataset switches (stale-response token)', async () => { + let resolveFirst: (payload: FrameMetadataSourcesResponse) => void = () => {}; + const first = new Promise((resolve) => { resolveFirst = resolve; }); + const second: FrameMetadataSourcesResponse = { + shared: { + name: 'frame_metadata.csv', + text: 'filename,label\nimg001.png,second\n', + }, + cameras: {}, + }; + const loadFrameMetadata = vi.fn() + .mockReturnValueOnce(first) + .mockResolvedValueOnce(second); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? ['img001.png'] : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = mountFrameMetadata({ + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.loading.value).toBe(true); + + datasetId.value = 'dataset-b'; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(2); + expect(metadata.loading.value).toBe(false); + expect(metadata.currentEntries.value).toEqual([['filename', 'img001.png'], ['label', 'second']]); + + // The stale dataset-a response arrives late; its token no longer matches, so it is ignored. + resolveFirst({ + shared: { + name: 'frame-metadata.txt', + text: 'filename,label\nimg001.png,stale\n', + }, + cameras: {}, + }); + await settle(); + expect(metadata.currentEntries.value).toEqual([['filename', 'img001.png'], ['label', 'second']]); + expect(metadata.resolvedSourceName.value).toBe('frame_metadata.csv'); + expect(metadata.error.value).toBeNull(); + }); + + it('recovers after a failed load instead of stranding the panel on error', async () => { + const good: FrameMetadataSourcesResponse = { + cameras: { + port: { name: 'nav.csv', text: 'filename,depth\nport001.png,10\n' }, + starboard: { name: 'nav.csv', text: 'filename,depth\nstar001.png,20\n' }, + }, + }; + // The first load rejects (transient network blip); the next succeeds. + const loadFrameMetadata = vi.fn() + .mockRejectedValueOnce(new Error('network blip')) + .mockResolvedValueOnce(good); + const getCameraMediaNames = vi.fn((camera: string) => ({ + port: ['port001.png'], + starboard: ['star001.png'], + }[camera])); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('port'); + const metadata = mountFrameMetadata({ + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + // The initial load rejected: the panel is in an error state, not silently "loaded". + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.error.value).not.toBeNull(); + expect(metadata.attachmentState.value).toBe('none'); + + // A camera change re-runs ensure(). A failed load must not have committed the dataset as + // loaded, or this would short-circuit and leave the panel stuck forever; instead it retries. + selectedCamera.value = 'starboard'; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(2); + expect(metadata.error.value).toBeNull(); + expect(metadata.attachmentState.value).toBe('resolved'); + expect(metadata.currentEntries.value).toEqual([['filename', 'star001.png'], ['depth', '20']]); + }); + + it('negative-caches an empty source listing and never refetches until a dataset switch', async () => { + const loadFrameMetadata = vi.fn(async () => ({ cameras: {} })); + const getCameraMediaNames = vi.fn(() => [] as string[]); + + const datasetId = ref('dataset-a'); + const frame = ref(10); + const selectedCamera = ref('singleCam'); + const metadata = mountFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.attachmentState.value).toBe('none'); + expect(metadata.currentEntries.value).toEqual([]); + + // Scrubbing frames and switching cameras on the same dataset must not refetch. + frame.value = 500; + await settle(); + frame.value = 5000; + await settle(); + selectedCamera.value = 'starboard'; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + + // A dataset switch re-runs source loading. + datasetId.value = 'dataset-b'; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(2); + expect(loadFrameMetadata).toHaveBeenLastCalledWith('dataset-b'); + }); + + it('negative-caches an empty source listing across a panel remount', async () => { + const loadFrameMetadata = vi.fn(async () => ({ cameras: {} })); + const options = { + datasetId: ref('dataset-a'), + frame: ref(0), + selectedCamera: ref('singleCam'), + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(() => ['img001.png']), + }; + + const first = mountFrameMetadata(options); + await settle(); + expect(first.attachmentState.value).toBe('none'); + + // A dataset with no attachment is the cheapest thing to remember and the most common: closing + // and reopening the panel must read it back from the session cache, not re-list the sources. + const second = mountFrameMetadata(options); + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(second.attachmentState.value).toBe('none'); + }); + + it('resolves attachments against the media list and exposes the current frame row', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'frame_metadata.csv', + text: 'filename,depth\nimg001.png,10\nimg002.png,12\n', + }, + cameras: {}, + })); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? ['img001.png', 'img002.png'] : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = mountFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.attachmentState.value).toBe('resolved'); + expect(metadata.resolvedSourceName.value).toBe('frame_metadata.csv'); + expect(metadata.currentEntries.value).toEqual([['filename', 'img001.png'], ['depth', '10']]); + + // Scrubbing re-materializes the row lazily from held data, with no refetch. + frame.value = 1; + await nextTick(); + expect(metadata.currentEntries.value).toEqual([['filename', 'img002.png'], ['depth', '12']]); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + + // A frame with no matching row shows nothing (empty-state), not blank columns. + frame.value = 2; + await nextTick(); + expect(metadata.currentEntries.value).toEqual([]); + }); + + it('resolves a multicam camera lazily when its media list loads after selection', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + cameras: { + port: { + name: 'frame_metadata.csv', + text: 'filename,depth\nport001.png,10\n', + }, + starboard: { + name: 'frame-metadata.txt', + text: 'filename,depth\nstar001.png,20\n', + }, + }, + })); + // starboard's ordered media list is not available until it is selected. + const media: Record = { + port: ['port001.png'], + starboard: undefined, + }; + const getCameraMediaNames = vi.fn((camera: string) => media[camera]); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('port'); + const metadata = mountFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + // port resolved eagerly; starboard deferred because its media list was not yet available. + expect(metadata.currentEntries.value).toEqual([['filename', 'port001.png'], ['depth', '10']]); + + // The media list arrives and the user selects starboard: it resolves on selection. + media.starboard = ['star001.png']; + selectedCamera.value = 'starboard'; + await settle(); + expect(metadata.attachmentState.value).toBe('resolved'); + expect(metadata.resolvedSourceName.value).toBe('frame-metadata.txt'); + expect(metadata.currentEntries.value).toEqual([['filename', 'star001.png'], ['depth', '20']]); + // No source refetch across the whole interaction. + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + }); + + it('follows the active camera for the resolved source and its rows', async () => { + const loadFrameMetadata = vi.fn(async (): Promise => ({ + cameras: { + port: { name: 'frame_metadata.csv', text: 'filename,latitude\nport001.png,58.10\n' }, + starboard: { name: 'frame_metadata.csv', text: 'filename,latitude\nstar001.png,59.10\n' }, + }, + })); + const getCameraMediaNames = vi.fn((camera: string) => ({ + port: ['port001.png'], + starboard: ['star001.png'], + }[camera])); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('port'); + const metadata = mountFrameMetadata({ + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + expect(metadata.resolvedSourceName.value).toBe('frame_metadata.csv'); + expect(metadata.attachmentState.value).toBe('resolved'); + expect(metadata.currentEntries.value).toEqual([['filename', 'port001.png'], ['latitude', '58.10']]); + + selectedCamera.value = 'starboard'; + await nextTick(); + expect(metadata.resolvedSourceName.value).toBe('frame_metadata.csv'); + expect(metadata.attachmentState.value).toBe('resolved'); + expect(metadata.currentEntries.value).toEqual([['filename', 'star001.png'], ['latitude', '59.10']]); + + selectedCamera.value = 'stern'; + await nextTick(); + expect(metadata.resolvedSourceName.value).toBeUndefined(); + expect(metadata.attachmentState.value).toBe('none'); + expect(metadata.currentEntries.value).toEqual([]); + }); + + it('selects a camera-local attachment as a whole without falling through to shared content', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'shared.csv', + text: 'filename,depth\nport001.png,10\nstar001.png,20\n', + }, + cameras: { + port: { + name: 'local.csv', + text: 'filename,depth\nother001.png,99\n', + }, + }, + })); + const getCameraMediaNames = vi.fn((camera: string) => ({ + port: ['port001.png'], + starboard: ['star001.png'], + }[camera])); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('port'); + const metadata = mountFrameMetadata({ + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + expect(metadata.attachmentName.value).toBe('local.csv'); + expect(metadata.attachmentState.value).toBe('unmatched'); + expect(metadata.currentEntries.value).toEqual([]); + + selectedCamera.value = 'starboard'; + await settle(); + expect(metadata.attachmentName.value).toBe('shared.csv'); + expect(metadata.resolvedSourceName.value).toBe('shared.csv'); + expect(metadata.currentEntries.value).toEqual([['filename', 'star001.png'], ['depth', '20']]); + }); + + it('keeps an unavailable attachment distinct from an unmatched attachment', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'missing.csv', + error: 'Metadata attachment is unavailable.', + }, + cameras: {}, + })); + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = mountFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(() => ['img001.png']), + }); + + await settle(); + expect(metadata.attachmentName.value).toBe('missing.csv'); + expect(metadata.attachmentState.value).toBe('unavailable'); + // `error` is the source-load channel only; an unreadable attachment is a state, not a failure. + expect(metadata.error.value).toBeNull(); + }); + + it.each([ + { + name: 'opaque JSON', + attachment: { name: 'pipeline-input.json' }, + expected: 'opaque', + }, + { + name: 'invalid CSV', + attachment: { name: 'broken.csv', text: '' }, + expected: 'invalid', + }, + ])('classifies an $name attachment distinctly', async ({ attachment, expected }) => { + const metadata = mountFrameMetadata({ + datasetId: ref('dataset-a'), + frame: ref(0), + selectedCamera: ref('singleCam'), + loadFrameMetadata: vi.fn(async () => ({ shared: attachment, cameras: {} })), + getCameraFrameContext: frameContextFromMediaNames(() => ['img001.png']), + }); + + await settle(); + expect(metadata.attachmentState.value).toBe(expected); + expect(metadata.currentEntries.value).toEqual([]); + }); + + it('keeps an unavailable attachment visible after a panel remount', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'missing.csv', + error: 'Metadata attachment is unavailable.', + }, + cameras: {}, + })); + const options = { + datasetId: ref('dataset-a'), + frame: ref(0), + selectedCamera: ref('singleCam'), + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(() => ['img001.png']), + }; + + const first = mountFrameMetadata(options); + await settle(); + expect(first.attachmentState.value).toBe('unavailable'); + + const second = mountFrameMetadata(options); + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(second.attachmentState.value).toBe('unavailable'); + expect(second.attachmentName.value).toBe('missing.csv'); + }); + + it('shares one in-flight load across camera switches instead of refetching', async () => { + const response: FrameMetadataSourcesResponse = { + cameras: { + port: { name: 'port.csv', text: 'filename,depth\nport001.png,10\n' }, + starboard: { name: 'star.csv', text: 'filename,depth\nstar001.png,20\n' }, + }, + }; + let releaseSecondLoad = () => {}; + const loadFrameMetadata = vi.fn() + .mockResolvedValueOnce(response) + .mockImplementationOnce(() => new Promise((resolve) => { + releaseSecondLoad = () => resolve(response); + })); + const media: Record = { + port: ['port001.png'], + starboard: undefined, + }; + const getCameraFrameContext = frameContextFromMediaNames((camera) => media[camera]); + const datasetId = ref('dataset-a'); + const frame = ref(0); + + // The first panel resolves port and parks starboard in `pending`: its media list is missing. + mountFrameMetadata({ + datasetId, frame, selectedCamera: ref('port'), loadFrameMetadata, getCameraFrameContext, + }); + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + + // A remount hydrates from the session cache, which holds attachment names but no bytes, so + // the camera left pending has to refetch them. Camera switches during that download must + // await the same request instead of downloading every camera's attachment again. + media.starboard = ['star001.png']; + const selectedCamera = ref('starboard'); + const second = mountFrameMetadata({ + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraFrameContext, + }); + await nextTick(); + selectedCamera.value = 'port'; + await nextTick(); + selectedCamera.value = 'starboard'; + await nextTick(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(2); + + releaseSecondLoad(); + await settle(); + expect(second.currentEntries.value).toEqual([['filename', 'star001.png'], ['depth', '20']]); + expect(loadFrameMetadata).toHaveBeenCalledTimes(2); + }); + + it('exposes the attachment name for a present attachment whose rows match no media filename', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'frame_metadata.csv', + text: 'filename,depth\nother001.png,10\n', + }, + cameras: {}, + })); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? ['img001.png'] : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = mountFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + expect(metadata.attachmentName.value).toBe('frame_metadata.csv'); + expect(metadata.attachmentState.value).toBe('unmatched'); + expect(metadata.currentEntries.value).toEqual([]); + }); + + it('resolves the single camera once its initially-empty media list populates (reactive retry)', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'frame_metadata.csv', + text: 'filename,depth\nimg001.png,10\n', + }, + cameras: {}, + })); + // Mirrors Viewer.vue: imageData (and so getCameraMediaNames) starts at `[]`, not `undefined`. + const media = ref([]); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? media.value : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const metadata = mountFrameMetadata({ + datasetId, + frame, + selectedCamera, + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }); + + await settle(); + // The attachment is loaded, but an empty media list must defer, not claim-and-drop it. + expect(metadata.attachmentState.value).toBe('pending'); + expect(metadata.attachmentName.value).toBe('frame_metadata.csv'); + + // The media list populates later, with no dataset/camera change; the reactive retry resolves. + media.value = ['img001.png']; + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(metadata.attachmentState.value).toBe('resolved'); + expect(metadata.currentEntries.value).toEqual([['filename', 'img001.png'], ['depth', '10']]); + }); + + it('keeps video text pending until its frame bound is ready, then resolves without refetching', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'frame_metadata.csv', + text: 'frame,depth\n2,30\n0,10\n', + }, + cameras: {}, + })); + const ready = ref(false); + const maxFrame = ref(0); + const frame = ref(0); + const metadata = mountFrameMetadata({ + datasetId: ref('dataset-a'), + frame, + selectedCamera: ref('singleCam'), + loadFrameMetadata, + getCameraFrameContext: () => ( + ready.value + ? { mediaType: 'video', frameCount: maxFrame.value + 1 } + : undefined + ), + }); + + await settle(); + expect(metadata.attachmentState.value).toBe('pending'); + + maxFrame.value = 2; + ready.value = true; + await settle(); + expect(metadata.currentEntries.value).toEqual([['frame', '0'], ['depth', '10']]); + expect(metadata.attachmentState.value).toBe('resolved'); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + + frame.value = 2; + await settle(); + expect(metadata.currentEntries.value).toEqual([['frame', '2'], ['depth', '30']]); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + }); + + it('resolves shared video rows against each camera bound and keeps a local override whole', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'shared.csv', + text: 'frame,depth\n0,shared-start\n2,shared-long\n', + }, + cameras: { + port: { + name: 'local.csv', + text: 'frame,depth\n0,local-start\n', + }, + }, + })); + const selectedCamera = ref('port'); + const frame = ref(0); + const frameCounts: Record = { port: 1, starboard: 3 }; + const metadata = mountFrameMetadata({ + datasetId: ref('dataset-a'), + frame, + selectedCamera, + loadFrameMetadata, + getCameraFrameContext: (camera) => ({ + mediaType: 'video', + frameCount: frameCounts[camera], + }), + }); + + await settle(); + expect(metadata.resolvedSourceName.value).toBe('local.csv'); + expect(metadata.currentEntries.value).toEqual([['frame', '0'], ['depth', 'local-start']]); + + selectedCamera.value = 'starboard'; + frame.value = 2; + await settle(); + expect(metadata.resolvedSourceName.value).toBe('shared.csv'); + expect(metadata.currentEntries.value).toEqual([['frame', '2'], ['depth', 'shared-long']]); + + selectedCamera.value = 'port'; + await settle(); + expect(metadata.currentEntries.value).toEqual([]); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + }); + + it('reuses the module-level session cache across composable instances (panel remount)', async () => { + const loadFrameMetadata = vi.fn(async () => ({ + shared: { + name: 'frame_metadata.csv', + text: 'filename,depth\nimg001.png,10\n', + }, + cameras: {}, + })); + const getCameraMediaNames = vi.fn((camera: string) => ( + camera === 'singleCam' ? ['img001.png'] : undefined + )); + + const datasetId = ref('dataset-a'); + const frame = ref(0); + const selectedCamera = ref('singleCam'); + const options = { + datasetId, frame, selectedCamera, loadFrameMetadata, getCameraFrameContext: frameContextFromMediaNames(getCameraMediaNames), + }; + + const first = mountFrameMetadata(options); + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(first.attachmentState.value).toBe('resolved'); + + // SidebarContext.vue mounts DatasetInfo with v-if: closing/reopening the panel destroys and + // re-creates this composable. A second instance for the same dataset must hydrate from the + // module cache rather than reloading the attachment. + const second = mountFrameMetadata(options); + await settle(); + + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + expect(second.attachmentState.value).toBe('resolved'); + expect(second.currentEntries.value).toEqual([['filename', 'img001.png'], ['depth', '10']]); + }); + + it('drops a response that lands after the panel is destroyed', async () => { + let releaseLoad = () => {}; + const loadFrameMetadata = vi.fn(() => new Promise((resolve) => { + releaseLoad = () => resolve({ + shared: { name: 'frame_metadata.csv', text: 'filename,depth\nimg001.png,10\n' }, + cameras: {}, + }); + })); + const options = { + datasetId: ref('dataset-a'), + frame: ref(0), + selectedCamera: ref('singleCam'), + loadFrameMetadata, + getCameraFrameContext: frameContextFromMediaNames(() => ['img001.png']), + }; + + const scope = effectScope(); + const metadata = scope.run(() => useFrameMetadata(options)); + if (metadata === undefined) { + throw new Error('effect scope did not run'); + } + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(1); + + // Closing the panel stops the scope while the request is still open. + scope.stop(); + releaseLoad(); + await settle(); + expect(metadata.attachmentState.value).toBe('none'); + + // The destroyed panel wrote nothing to the shared session cache, so the next one refetches. + const second = mountFrameMetadata(options); + await settle(); + expect(loadFrameMetadata).toHaveBeenCalledTimes(2); + expect(second.attachmentState.value).toBe('none'); + }); +}); diff --git a/client/dive-common/use/useFrameMetadata.ts b/client/dive-common/use/useFrameMetadata.ts new file mode 100644 index 000000000..1c2452840 --- /dev/null +++ b/client/dive-common/use/useFrameMetadata.ts @@ -0,0 +1,327 @@ +import { + computed, onScopeDispose, readonly, ref, watch, +} from 'vue'; +import type { Ref } from 'vue'; +import { getResponseError } from 'vue-media-annotator/utils'; + +import type { + FrameMetadataAttachmentText, + FrameMetadataSourcesResponse, +} from 'dive-common/apispec'; +import { + buildFrameAlignmentIndex, + resolveCameraAttachment, +} from 'dive-common/frameMetadata/resolve'; +import type { + FrameMetadataFrameContext, + ResolvedCameraFrameMetadata, +} from 'dive-common/frameMetadata/resolve'; + +export interface UseFrameMetadataOptions { + /** Current dataset id (parent-root id for multicam). */ + datasetId: Readonly>; + /** Current playhead frame number. Drives `currentEntries`; never triggers a fetch. */ + frame: Readonly>; + /** Active camera key (`singleCam` for single-camera datasets). */ + selectedCamera: Readonly>; + /** Ready media context for a camera, including its usable DIVE frame bound. */ + getCameraFrameContext: (camera: string) => FrameMetadataFrameContext | undefined; + loadFrameMetadata: (datasetId: string) => Promise; +} + +/** + * What the active camera's attachment currently is. This enum is the whole taxonomy the panel + * renders from: no consumer may re-derive a state from the payload. + */ +export type FrameMetadataAttachmentState = + | 'none' + | 'opaque' + | 'unavailable' + | 'invalid' + | 'unmatched' + | 'pending' + | 'resolved'; + +/** + * The session-cacheable slice of a resolved dataset. Attachment text is stripped: a remount + * re-renders resolved rows straight from `cameras`, and a camera still waiting on its media + * refetches the bytes it needs. + */ +interface FrameMetadataCacheEntry { + cameras: Record; + sourceResponse: FrameMetadataSourcesResponse; + attachmentStatesByCamera: Record; + sourcesLoaded: boolean; +} + +// Cache the last resolved dataset so a panel remount can render without refetching attachments. +let sessionCache: { datasetId: string; entry: FrameMetadataCacheEntry } | null = null; + +export function resetFrameMetadataSessionCache() { + sessionCache = null; +} + +function withoutText(response: FrameMetadataSourcesResponse): FrameMetadataSourcesResponse { + const strip = ({ name, error }: FrameMetadataAttachmentText) => ({ name, error }); + return { + shared: response.shared === undefined ? undefined : strip(response.shared), + cameras: Object.fromEntries( + Object.entries(response.cameras).map(([camera, attachment]) => [camera, strip(attachment)]), + ), + }; +} + +export function useFrameMetadata({ + datasetId, + frame, + selectedCamera, + getCameraFrameContext, + loadFrameMetadata, +}: UseFrameMetadataOptions) { + const cameras = ref>({}); + const sourceResponse = ref({ cameras: {} }); + const attachmentStatesByCamera = ref>({}); + const loading = ref(false); + const error = ref(null); + + // One generation counter is the whole concurrency story: it is bumped on every dataset switch, + // and a response carrying an older token is dropped. + let token = 0; + let loadedDatasetId: string | null = null; + let sourcesLoaded = false; + // The whole attachment set loads eagerly, by design: a camera switch renders from bytes already + // in hand. Holding the in-flight promise keeps switches during the download from starting a + // second full load. (Per-camera lazy download is a separate, deliberately deferred change.) + let pendingLoad: { token: number; promise: Promise } | null = null; + + function setAttachmentState(camera: string, state: FrameMetadataAttachmentState) { + attachmentStatesByCamera.value = { + ...attachmentStatesByCamera.value, + [camera]: state, + }; + } + + function reset() { + cameras.value = {}; + sourceResponse.value = { cameras: {} }; + attachmentStatesByCamera.value = {}; + error.value = null; + loading.value = false; + sourcesLoaded = false; + } + + function syncSessionCache() { + if (loadedDatasetId === null) { + return; + } + sessionCache = { + datasetId: loadedDatasetId, + entry: { + cameras: { ...cameras.value }, + sourceResponse: withoutText(sourceResponse.value), + attachmentStatesByCamera: { ...attachmentStatesByCamera.value }, + sourcesLoaded, + }, + }; + } + + function hydrateFromCache(entry: FrameMetadataCacheEntry) { + cameras.value = { ...entry.cameras }; + sourceResponse.value = entry.sourceResponse; + attachmentStatesByCamera.value = { ...entry.attachmentStatesByCamera }; + sourcesLoaded = entry.sourcesLoaded; + } + + /** The one owner of camera-local-over-shared precedence. */ + function attachmentForCamera(camera: string): FrameMetadataAttachmentText | undefined { + return sourceResponse.value.cameras[camera] ?? sourceResponse.value.shared; + } + + // A camera is settled once its attachment has been classified against its own media. Only + // `pending` (media not ready yet) and `none` (nothing declared yet) can still change. + function isSettled(camera: string) { + const state = attachmentStatesByCamera.value[camera]; + return state !== undefined && state !== 'pending' && state !== 'none'; + } + + /** Classify one camera's attachment, recording its rows when they resolve. */ + function classifyCamera(camera: string): FrameMetadataAttachmentState { + const attachment = attachmentForCamera(camera); + if (attachment === undefined) { + return 'none'; + } + if (attachment.error !== undefined) { + return 'unavailable'; + } + if (attachment.text === undefined) { + return 'opaque'; + } + const frameContext = getCameraFrameContext(camera); + if (frameContext === undefined) { + return 'pending'; + } + const resolution = resolveCameraAttachment( + attachment.text, + buildFrameAlignmentIndex(frameContext), + attachment.name, + ); + if (resolution.status === 'resolved') { + cameras.value = { ...cameras.value, [camera]: resolution.metadata }; + } + return resolution.status; + } + + function resolveCamera(camera: string) { + if (isSettled(camera)) { + return; + } + setAttachmentState(camera, classifyCamera(camera)); + // Every classification syncs, including `none`: a dataset that declares no attachment is the + // case the session cache most needs to remember, or each panel remount refetches the listing. + syncSessionCache(); + } + + async function fetchSources(id: string, requestToken: number) { + loading.value = true; + try { + const response = await loadFrameMetadata(id); + if (requestToken !== token) { + return; + } + sourceResponse.value = response; + sourcesLoaded = true; + // Only the active camera is joined here: nothing outside can observe another camera's + // rows, and selecting one resolves it from the text this response already carries. + resolveCamera(selectedCamera.value); + } catch (err) { + if (requestToken === token) { + error.value = getResponseError(err); + // A failed load must not leave loadedDatasetId committed: ensure()'s same-dataset + // short-circuit would then treat the panel as loaded and never retry, stranding it + // on the error. Clearing it lets the next ensure() re-attempt the fetch. + loadedDatasetId = null; + } + } finally { + if (requestToken === token) { + loading.value = false; + } + if (pendingLoad?.token === requestToken) { + pendingLoad = null; + } + } + } + + function loadSources(id: string, requestToken: number): Promise { + if (pendingLoad?.token === requestToken) { + return pendingLoad.promise; + } + const promise = fetchSources(id, requestToken); + pendingLoad = { token: requestToken, promise }; + return promise; + } + + // A hydrated cache carries attachment names but no bytes, so the active camera needs a refetch + // exactly when it is still unclassified and has no text in hand. + function activeCameraNeedsText() { + const attachment = attachmentForCamera(selectedCamera.value); + return attachment !== undefined + && attachment.text === undefined + && !isSettled(selectedCamera.value); + } + + async function resolveActiveCamera(id: string) { + if (activeCameraNeedsText()) { + await loadSources(id, token); + } else if (sourcesLoaded) { + resolveCamera(selectedCamera.value); + } + } + + async function ensure() { + const id = datasetId.value; + if (!id) { + if (loadedDatasetId !== null) { + token += 1; + loadedDatasetId = null; + reset(); + } + return; + } + if (id !== loadedDatasetId) { + token += 1; + loadedDatasetId = id; + reset(); + const cached = sessionCache !== null && sessionCache.datasetId === id + ? sessionCache.entry + : null; + if (cached === null) { + await loadSources(id, token); + return; + } + hydrateFromCache(cached); + } + await resolveActiveCamera(id); + } + + watch( + [datasetId, selectedCamera], + () => { ensure(); }, + { immediate: true }, + ); + + // SidebarContext mounts the panel behind a v-if, so an in-flight load can outlive the + // composable. Retiring the token on teardown makes that response stale, which is the same + // mechanism a dataset switch uses -- a destroyed panel must not write the shared session cache. + onScopeDispose(() => { token += 1; }); + + const activeCameraFrameContextSignature = computed(() => { + const context = getCameraFrameContext(selectedCamera.value); + if (context === undefined) { + return undefined; + } + return context.mediaType === 'video' + ? `video:${context.frameCount}` + : `image-sequence:${context.mediaNames.length}`; + }); + // A camera's media list (or video frame bound) can arrive after its attachment: retry the + // camera parked in `pending` as soon as its frame context exists. + watch(activeCameraFrameContextSignature, (signature, previous) => { + if (signature !== undefined && previous === undefined) { + ensure(); + } + }); + + const currentEntries = computed<[string, string][]>(() => { + const resolved = cameras.value[selectedCamera.value]; + const row = resolved?.records[frame.value]; + if (resolved === undefined || row === undefined) { + return []; + } + return resolved.columns.map((column, i): [string, string] => [column, row[i] ?? '']); + }); + + /** Attachment declared for the active camera, whatever state it is in. */ + const attachmentName = computed(() => attachmentForCamera(selectedCamera.value)?.name); + /** + * Why the backend could not hand over the attachment's bytes; set exactly when the state is + * `unavailable`. The state alone cannot tell an unreadable attachment from several competing + * reserved-name ones, so the reason the backend authored is what the panel must show. + */ + const attachmentError = computed(() => attachmentForCamera(selectedCamera.value)?.error); + /** Attachment the displayed rows came from; undefined until the rows resolve. */ + const resolvedSourceName = computed(() => cameras.value[selectedCamera.value]?.sourceName); + const attachmentState = computed(() => ( + attachmentStatesByCamera.value[selectedCamera.value] + ?? (attachmentName.value === undefined ? 'none' : 'pending') + )); + + return { + currentEntries, + attachmentName, + attachmentError, + resolvedSourceName, + attachmentState, + loading: readonly(loading), + error: readonly(error), + }; +} diff --git a/client/src/components/annotators/ImageAnnotator.vue b/client/src/components/annotators/ImageAnnotator.vue index 7ff1bfd48..70bde6986 100644 --- a/client/src/components/annotators/ImageAnnotator.vue +++ b/client/src/components/annotators/ImageAnnotator.vue @@ -2,6 +2,7 @@ import { defineComponent, ref, onUnmounted, PropType, toRef, watch, } from 'vue'; +import { map } from 'lodash'; import { ImageEnhancementOutputs } from 'vue-media-annotator/use/useImageEnhancements'; import { SetTimeFunc } from '../../use/useTimeObserver'; import AnnotatorImageCursor from './AnnotatorImageCursor.vue'; @@ -77,7 +78,7 @@ export default defineComponent({ initializeViewer, mediaController, externallyDriven, - } = cameraInitializer(props.camera, { + } = cameraInitializer(props.camera, 'image-sequence', { // allow hoisting for these functions to pass a reference before defining them. // eslint-disable-next-line @typescript-eslint/no-use-before-define seek, pause, play, setVolume: unimplemented, setSpeed: unimplemented, @@ -88,6 +89,7 @@ export default defineComponent({ toRef(data, 'imageCursorEditing'), ); data.maxFrame = props.imageData.length - 1; + data.filenames = map(props.imageData, 'filename'); // Below are configuration settings we can set until we decide on good numbers to utilize. let local = { playCache: 1, // seconds required to be fully cached before playback @@ -389,6 +391,7 @@ export default defineComponent({ } function init() { data.maxFrame = props.imageData.length - 1; + data.filenames = map(props.imageData, 'filename'); // When the viewer already exists, an imageData change is a URL swap for // the same camera (e.g. the percentile-stretch display remap). Rebuild // only the image cache and redraw the current frame in place: calling diff --git a/client/src/components/annotators/LargeImageAnnotator.vue b/client/src/components/annotators/LargeImageAnnotator.vue index f11f44622..06687ad4a 100644 --- a/client/src/components/annotators/LargeImageAnnotator.vue +++ b/client/src/components/annotators/LargeImageAnnotator.vue @@ -136,7 +136,7 @@ export default defineComponent({ container, initializeViewer, mediaController, - } = cameraInitializer(props.camera, { + } = cameraInitializer(props.camera, 'large-image', { // allow hoisting for these functions to pass a reference before defining them. // eslint-disable-next-line @typescript-eslint/no-use-before-define seek, pause, play, setVolume: unimplemented, setSpeed: unimplemented, diff --git a/client/src/components/annotators/VideoAnnotator.vue b/client/src/components/annotators/VideoAnnotator.vue index 05a3628fa..0e067bb12 100644 --- a/client/src/components/annotators/VideoAnnotator.vue +++ b/client/src/components/annotators/VideoAnnotator.vue @@ -122,7 +122,7 @@ export default defineComponent({ initializeViewer, mediaController, externallyDriven, - } = cameraInitializer(props.camera, { + } = cameraInitializer(props.camera, 'video', { // allow hoisting for these functions. // eslint-disable-next-line @typescript-eslint/no-use-before-define seek, diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index 8ed06212c..183bfbd2b 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -1,6 +1,8 @@ import type { Ref } from 'vue'; import type { CameraImage } from '../../layers/cameraImage'; +export type MediaControllerKind = 'image-sequence' | 'video' | 'large-image'; + /** * Supplied by Viewer.vue when every camera in a multicam dataset has a * timestamp on every frame (see dive-common/alignedTimeline.ts). Translates @@ -89,9 +91,12 @@ export interface AggregateMediaController { * functions to control an individual camera */ export interface MediaController extends AggregateMediaController { + readonly mediaKind: MediaControllerKind; + ready: Readonly>; cameraName: Readonly>; duration: Readonly>; filename: Readonly>; + filenames: Readonly>; flick: Readonly>; // eslint-disable-next-line @typescript-eslint/no-explicit-any geoViewerRef: Readonly>; diff --git a/client/src/components/annotators/useMediaController.spec.ts b/client/src/components/annotators/useMediaController.spec.ts index dd6d7f3a4..047f2caeb 100644 --- a/client/src/components/annotators/useMediaController.spec.ts +++ b/client/src/components/annotators/useMediaController.spec.ts @@ -26,10 +26,10 @@ function mountMediaController() { const Host = defineComponent({ setup() { composable = useMediaController(); - composable.initialize('A', { + composable.initialize('A', 'image-sequence', { seek: seekA, play: playA, pause: pauseA, setVolume: noop, setSpeed: noop, }); - composable.initialize('B', { + composable.initialize('B', 'image-sequence', { seek: seekB, play: playB, pause: pauseB, setVolume: noop, setSpeed: noop, }); return {}; @@ -98,6 +98,18 @@ function makeShiftedResolver(): AlignedFrameResolver { } describe('useMediaController', () => { + it('exposes each registered camera kind and the existing readiness ref', () => { + const { composable } = mountMediaController(); + const controller = composable.aggregateController.value.getController('A'); + const keyA = Object.keys(composable.state) + .find((key) => composable.state[key].cameraName === 'A'); + + expect(controller.mediaKind).toBe('image-sequence'); + expect(controller.ready.value).toBe(false); + composable.state[keyA as string].ready = true; + expect(controller.ready.value).toBe(true); + }); + it('without a resolver, seek broadcasts the identical frame to every camera', () => { const { composable, mocks } = mountMediaController(); composable.aggregateController.value.seek(5); @@ -281,7 +293,7 @@ describe('useMediaController', () => { // the resolver) self-seeks to its own local frame 0 during init; the // roster watcher must re-seek it onto the current slot afterwards. const seekC = vi.fn(); - composable.initialize('C', { + composable.initialize('C', 'image-sequence', { seek: seekC, play: noop, pause: noop, setVolume: noop, setSpeed: noop, }); await wrapper.vm.$nextTick(); diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 669a0d3bf..48e88c86d 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -9,7 +9,12 @@ import Vue, { import { map, over } from 'lodash'; import { use } from '../../provides'; -import type { AggregateMediaController, AlignedFrameResolver, MediaController } from './mediaControllerType'; +import type { + AggregateMediaController, + AlignedFrameResolver, + MediaController, + MediaControllerKind, +} from './mediaControllerType'; import type { CameraImage } from '../../layers/cameraImage'; const AggregateControllerSymbol = Symbol('aggregate-controller'); @@ -29,6 +34,7 @@ interface MediaControllerReactiveData { volume: number; speed: number; maxFrame: number; + filenames: string[]; syncedFrame: number; /** False when an aligned-timeline slot has no frame for this camera; pane should blank. */ hasFrame: boolean; @@ -79,7 +85,7 @@ interface CameraInitializerReturn { externallyDriven: Readonly>; } -type CameraInitializerFunc = (cameraName: string, { +type CameraInitializerFunc = (cameraName: string, mediaKind: MediaControllerKind, { seek, play, pause, setVolume, setSpeed, }: { seek(frame: number | undefined): void; @@ -332,7 +338,7 @@ export function useMediaController() { * chicken-and-egg problem, allowing the function consumer to use * the state above to construct the dependencies for the methods below. */ - function initialize(cameraName: string, { + function initialize(cameraName: string, mediaKind: MediaControllerKind, { seek: _seek, play: _play, pause: _pause, setVolume: _setVolume, setSpeed: _setSpeed, }: { seek(frame: number | undefined): void; @@ -378,6 +384,7 @@ export function useMediaController() { volume: 0, speed: 1.0, maxFrame: 0, + filenames: [], syncedFrame: 0, hasFrame: true, imageRevision: 0, @@ -610,6 +617,8 @@ export function useMediaController() { }; const mediaController: MediaController = { + mediaKind, + ready: toRef(state[camera], 'ready'), geoViewerRef: geoViewers[camera], cameraName: toRef(state[camera], 'cameraName'), cameras: ref([]), @@ -618,6 +627,7 @@ export function useMediaController() { frame: toRef(state[camera], 'frame'), flick: toRef(state[camera], 'flick'), filename: toRef(state[camera], 'filename'), + filenames: toRef(state[camera], 'filenames'), duration: toRef(state[camera], 'duration'), volume: toRef(state[camera], 'volume'), maxFrame: toRef(state[camera], 'maxFrame'),