diff --git a/packages/cli/src/commands/timeline.ts b/packages/cli/src/commands/timeline.ts index 7224ca15a0..f025a08496 100644 --- a/packages/cli/src/commands/timeline.ts +++ b/packages/cli/src/commands/timeline.ts @@ -23,7 +23,7 @@ export default defineCommand({ async run({ args }) { const project = resolveProject(args.dir); ensureDOMParser(); - const timeline = describeProject(project.indexPath); + const timeline = await describeProject(project.indexPath); console.log( args.json ? JSON.stringify(withMeta({ timeline }), null, 2) : formatTimeline(timeline), ); diff --git a/packages/cli/src/timeline/describeProject.test.ts b/packages/cli/src/timeline/describeProject.test.ts index 69a7ff7026..f9de3321a7 100644 --- a/packages/cli/src/timeline/describeProject.test.ts +++ b/packages/cli/src/timeline/describeProject.test.ts @@ -1,13 +1,29 @@ -import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; +import { MEDIA_DURATION_FIXTURES } from "@hyperframes/parsers/media-duration-fixtures"; import { fileURLToPath } from "node:url"; import { afterEach, beforeAll, describe, expect, it } from "vitest"; import { ensureDOMParser } from "../utils/dom.js"; -import { describeProject } from "./describeProject.js"; +import { createProbeGate, describeProject, type MeasureMedia } from "./describeProject.js"; import { formatTimeline } from "./formatTimeline.js"; +const REAL_AUDIO = fileURLToPath( + new URL("../../../../skills/media-use/audio/assets/sfx/pop.mp3", import.meta.url), +); +const hasFfprobe = spawnSync("ffprobe", ["-version"]).status === 0; +/** Recorded ffprobe answer for pop.mp3 (0.72 s), so the resolver path runs on runners without ffprobe. */ +const POP_SECONDS = async () => 0.72; + const INDEX = `
@@ -21,6 +37,25 @@ const INDEX = ` const TITLE = ``; let dir = ""; +const TINY_PNG = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + "base64", +); + +const rowsOf = async ( + html: string, + withSting = false, + setup?: (root: string) => void, + measure?: MeasureMedia, +) => { + const index = project(); + writeFileSync(index, html); + if (withSting) copyFileSync(REAL_AUDIO, join(dir, "sting.mp3")); + setup?.(dir); + const timeline = await describeProject(index, measure); + return { rows: timeline.tracks.flatMap((t) => t.rows), text: formatTimeline(timeline) }; +}; + const project = () => { dir = mkdtempSync(join(tmpdir(), "hf-timeline-")); mkdirSync(join(dir, "compositions")); @@ -33,8 +68,8 @@ beforeAll(ensureDOMParser); afterEach(() => rmSync(dir, { recursive: true, force: true })); describe("describeProject", () => { - it("groups rows into tracks by kind with resolved timing and clip facts", () => { - const timeline = describeProject(project()); + it("groups rows into tracks by kind with resolved timing and clip facts", async () => { + const timeline = await describeProject(project()); expect(timeline.duration).toBe(10); expect(timeline.tracks.map((t) => [t.kind, t.rows.map((r) => r.id)])).toEqual([ ["video", ["a-roll"]], @@ -58,9 +93,9 @@ describe("describeProject", () => { ]); }); - it("nests a sub-composition's clips one level down with local times", () => { - const title = describeProject(project()) - .tracks.flatMap((t) => t.rows) + it("nests a sub-composition's clips one level down with local times", async () => { + const title = (await describeProject(project())).tracks + .flatMap((t) => t.rows) .find((r) => r.id === "title")!; expect(title.children.map((c) => [c.id, c.start, c.end])).toEqual([ ["t1", 0, 2], @@ -68,23 +103,23 @@ describe("describeProject", () => { ]); }); - it("reports unreadable automation instead of showing no lanes", () => { - const bad = describeProject(project()) - .tracks.flatMap((t) => t.rows) + it("reports unreadable automation instead of showing no lanes", async () => { + const bad = (await describeProject(project())).tracks + .flatMap((t) => t.rows) .find((r) => r.id === "bad")!; expect(bad.lanes).toEqual([]); expect(bad.laneError).toMatch(/not valid JSON/); - expect(formatTimeline(describeProject(project()))).toContain("lanes unreadable:"); + expect(formatTimeline(await describeProject(project()))).toContain("lanes unreadable:"); }); - it("does not read a sub-composition outside the project or a directory", () => { + it("does not read a sub-composition outside the project or a directory", async () => { const index = project(); writeFileSync(join(dir, "..", "hf-outside.html"), TITLE); writeFileSync( index, `
`, ); - const rows = describeProject(index).tracks.flatMap((t) => t.rows); + const rows = (await describeProject(index)).tracks.flatMap((t) => t.rows); expect(rows.map((r) => [r.id, r.children.length])).toEqual([ ["o", 0], ["d", 0], @@ -92,7 +127,7 @@ describe("describeProject", () => { rmSync(join(dir, "..", "hf-outside.html")); }); - it("reads a sub-composition whose folder name starts with two dots", () => { + it("reads a sub-composition whose folder name starts with two dots", async () => { const index = project(); mkdirSync(join(dir, "..scenes")); writeFileSync(join(dir, "..scenes", "s.html"), TITLE); @@ -100,11 +135,11 @@ describe("describeProject", () => { index, `
`, ); - const [row] = describeProject(index).tracks.flatMap((t) => t.rows); + const [row] = (await describeProject(index)).tracks.flatMap((t) => t.rows); expect(row!.children.length).toBe(2); }); - it("does not follow a symlink out of the project", () => { + it("does not follow a symlink out of the project", async () => { const index = project(); const outside = mkdtempSync(join(tmpdir(), "hf-outside-")); writeFileSync(join(outside, "secret.html"), TITLE); @@ -113,20 +148,165 @@ describe("describeProject", () => { index, `
`, ); - const [row] = describeProject(index).tracks.flatMap((t) => t.rows); + const [row] = (await describeProject(index)).tracks.flatMap((t) => t.rows); expect(row!.children).toEqual([]); rmSync(outside, { recursive: true, force: true }); }); - it("marks a clip without an authored duration instead of reporting 0 as a fact", () => { - const logo = describeProject(project()) - .tracks.flatMap((t) => t.rows) + it("claims no duration source for a leaf with nothing authored and no children", async () => { + const index = project(); + writeFileSync(index, `
`); + const [row] = (await describeProject(index)).tracks.flatMap((t) => t.rows); + expect(row).toMatchObject({ durationSource: null, duration: 0 }); + expect(formatTimeline(await describeProject(index))).not.toMatch(/duration=|pending/); + }); + + it("does not probe a remote, absolute or parent-relative src and says why", async () => { + const index = project(); + const outside = mkdtempSync(join(tmpdir(), "hf-outside-media-")); + copyFileSync(REAL_AUDIO, join(outside, "out.mp3")); + copyFileSync(REAL_AUDIO, join(dir, "..", "hf-parent-media.mp3")); + writeFileSync( + index, + `
+ + + +
`, + ); + const rows = (await describeProject(index)).tracks.flatMap((t) => t.rows); + const byId = (id: string) => rows.find((r) => r.id === id)!; + expect(byId("remote")).toMatchObject({ + durationSource: "pending", + pendingReason: "remote source not probed", + }); + for (const id of ["abs", "up"]) { + expect(byId(id)).toMatchObject({ + durationSource: "pending", + pendingReason: "source file not found", + duration: 0, + }); + } + rmSync(outside, { recursive: true, force: true }); + rmSync(join(dir, "..", "hf-parent-media.mp3")); + }); + + it("reports a still image used as a video source as pending, not a measured zero", async () => { + const { rows, text } = await rowsOf( + `
`, + false, + (root) => writeFileSync(join(root, "still.png"), TINY_PNG), + ); + expect(rows[0]).toMatchObject({ + durationSource: "pending", + pendingReason: "source reports no duration", + duration: 0, + }); + expect(text).toContain("pending: source reports no duration"); + }); + + it("does not probe a media src that is a symlink out of the project", async () => { + const outside = mkdtempSync(join(tmpdir(), "hf-outside-")); + try { + copyFileSync(REAL_AUDIO, join(outside, "secret.mp3")); + const { rows } = await rowsOf( + `
`, + false, + (root) => symlinkSync(join(outside, "secret.mp3"), join(root, "link.mp3")), + ); + expect(rows[0]).toMatchObject({ + durationSource: "pending", + pendingReason: "source file not found", + duration: 0, + }); + } finally { + rmSync(outside, { recursive: true, force: true }); + } + }); + + it("gives an image with no authored duration the resolver's default length", async () => { + const logo = (await describeProject(project())).tracks + .flatMap((t) => t.rows) .find((r) => r.id === "logo")!; expect(logo.durationAuthored).toBe(false); - expect(formatTimeline(describeProject(project()))).toContain( - "logo 0-0s src=logo.png duration=unauthored", + expect(logo).toMatchObject({ durationSource: "default", duration: 3, pendingReason: null }); + expect(formatTimeline(await describeProject(project()))).toContain( + "logo 0-3s src=logo.png duration=default", ); }); + + it("infers a composition host's duration from its children", async () => { + const { + rows: [host], + text, + } = await rowsOf( + `
`, + ); + expect(host).toMatchObject({ durationSource: "inner", duration: 3.5 }); + expect(text).toContain("duration=inferred"); + }); + + it("applies the media offset and playback rate through the resolver", async () => { + const { + rows: [row], + } = await rowsOf( + `
`, + true, + undefined, + POP_SECONDS, + ); + // (0.72 - 0.2) / 2, hand-computed + expect(row!.duration).toBeCloseTo(0.26, 1); + }); + + it("takes a media leaf's duration from the probe when none is authored", async () => { + const probed: string[] = []; + const { + rows: [row], + text, + } = await rowsOf( + `
`, + true, + undefined, + async (file, tag) => { + probed.push(`${tag}:${basename(file)}`); + return POP_SECONDS(); + }, + ); + expect(probed).toEqual(["audio:sting.mp3"]); + expect(row).toMatchObject({ durationAuthored: false, durationSource: "media" }); + expect(row!.pendingReason).toBeNull(); + expect(row!.duration).toBeCloseTo(0.72, 1); + expect(row!.end).toBeCloseTo(0.72, 1); + expect(text).toContain("duration=media"); + }); + + it.skipIf(!hasFfprobe)("measures a real audio file with ffprobe by default", async () => { + const { + rows: [row], + } = await rowsOf( + `
`, + true, + ); + expect(row).toMatchObject({ durationSource: "media" }); + expect(row!.duration).toBeCloseTo(0.72, 1); + }); + + it("reports pending with a reason instead of guessing when the source file is missing", async () => { + const { + rows: [row], + text, + } = await rowsOf( + `
`, + ); + expect(row).toMatchObject({ + durationAuthored: false, + durationSource: "pending", + pendingReason: "source file not found", + duration: 0, + }); + expect(text).toContain("pending: source file not found"); + }); }); // A direct clip declared with a big data-start (20) reads as "the later one", @@ -147,8 +327,8 @@ const inversionProject = () => { }; describe("absolute main-timeline time", () => { - it("gives a nested clip an absolute start smaller than a later-declared direct clip's, plus the owning file", () => { - const rows = describeProject(inversionProject()).tracks.flatMap((t) => t.rows); + it("gives a nested clip an absolute start smaller than a later-declared direct clip's, plus the owning file", async () => { + const rows = (await describeProject(inversionProject())).tracks.flatMap((t) => t.rows); const direct = rows.find((r) => r.id === "direct")!; const host = rows.find((r) => r.id === "host")!; const nested = host.children.find((c) => c.id === "nested")!; @@ -173,7 +353,7 @@ describe("absolute main-timeline time", () => { expect(nested.absStart).toBeLessThan(direct.absStart); }); - it("places a nested media clip with a negative start where the runtime plays it", () => { + it("places a nested media clip with a negative start where the runtime plays it", async () => { dir = mkdtempSync(join(tmpdir(), "hf-timeline-neg-")); mkdirSync(join(dir, "compositions")); writeFileSync(join(dir, "index.html"), INVERSION_INDEX); @@ -181,14 +361,14 @@ describe("absolute main-timeline time", () => { join(dir, "compositions", "scene.html"), INVERSION_SCENE.replace('data-start="1"', 'data-start="-3"'), ); - const host = describeProject(join(dir, "index.html")) - .tracks.flatMap((t) => t.rows) + const host = (await describeProject(join(dir, "index.html"))).tracks + .flatMap((t) => t.rows) .find((r) => r.id === "host")!; // Host starts at 5 and the runtime adds the raw -3: it plays at 2, not at the clamped 5. expect(host.children.find((c) => c.id === "nested")).toMatchObject({ absStart: 2, absEnd: 4 }); }); - it("resolves a media start given as an expression like any other clip, not as a literal", () => { + it("resolves a media start given as an expression like any other clip, not as a literal", async () => { dir = mkdtempSync(join(tmpdir(), "hf-timeline-expr-")); mkdirSync(join(dir, "compositions")); writeFileSync(join(dir, "index.html"), INVERSION_INDEX); @@ -199,39 +379,97 @@ describe("absolute main-timeline time", () => { ``, ), ); - const host = describeProject(join(dir, "index.html")) - .tracks.flatMap((t) => t.rows) + const host = (await describeProject(join(dir, "index.html"))).tracks + .flatMap((t) => t.rows) .find((r) => r.id === "host")!; // nested ends at local 3, so "nested + 1" is local 4; the host at 5 puts it at 9 on the main timeline. expect(host.children.find((c) => c.id === "after")).toMatchObject({ start: 4, absStart: 9 }); }); - it("clamps a negative media start to 0 when the host starts at 0, as the runtime does", () => { + it("clamps a negative media start to 0 when the host starts at 0, as the runtime does", async () => { dir = mkdtempSync(join(tmpdir(), "hf-timeline-neg0-")); writeFileSync( join(dir, "index.html"), `
`, ); - const v = describeProject(join(dir, "index.html")).tracks.flatMap((t) => t.rows)[0]; + const v = (await describeProject(join(dir, "index.html"))).tracks.flatMap((t) => t.rows)[0]; expect(v).toMatchObject({ id: "v", absStart: 0, absEnd: 2 }); }); - it("prints the absolute time first and the local time in parentheses for a nested row", () => { - const text = formatTimeline(describeProject(inversionProject())); + // Reproduces an eval miss (pr-to-video-launch, task A): an sfx clip local to + // its own sub-composition starts at 5.2s there, but that sub-composition is + // hosted at main-timeline 5.2s too, and a video in an EARLIER host ends at + // 5.27s — a 0.07s overlap only visible once both are on the same clock. + it("carries enough absolute time to detect a sub-second overlap across two different sub-compositions", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-timeline-overlap-")); + mkdirSync(join(dir, "compositions")); + writeFileSync( + join(dir, "index.html"), + `
+
+
+
`, + ); + writeFileSync( + join(dir, "compositions", "a.html"), + ``, + ); + writeFileSync( + join(dir, "compositions", "b.html"), + ``, + ); + const rows = (await describeProject(join(dir, "index.html"))).tracks.flatMap((t) => t.rows); + const clip = rows.find((r) => r.id === "a")!.children[0]!; + const sfx = rows.find((r) => r.id === "b")!.children[0]!; + expect(clip).toMatchObject({ absStart: 0, absEnd: 5.27 }); + expect(sfx).toMatchObject({ absStart: 5.2, absEnd: 5.6 }); + expect(sfx.absStart).toBeLessThan(clip.absEnd); + }); + + // Reproduces an eval miss (cloud-render-launch, task B): "the second video + // clip" has to be found by comparing videos nested in DIFFERENT + // sub-compositions, each printed with its own local 0-based start. + it("orders videos nested in different sub-compositions by absolute start, not local start", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-timeline-order-")); + mkdirSync(join(dir, "compositions")); + writeFileSync( + join(dir, "index.html"), + `
+
+
+
`, + ); + writeFileSync( + join(dir, "compositions", "first.html"), + ``, + ); + writeFileSync( + join(dir, "compositions", "second.html"), + ``, + ); + const rows = (await describeProject(join(dir, "index.html"))).tracks.flatMap((t) => t.rows); + const videos = rows.flatMap((r) => r.children).filter((c) => c.kind === "video"); + const byAbsStart = [...videos].sort((a, b) => a.absStart - b.absStart); + expect(byAbsStart.map((v) => v.id)).toEqual(["v1", "v2"]); + expect(byAbsStart[1]).toMatchObject({ id: "v2", absStart: 6 }); + }); + + it("prints the absolute time first and the local time in parentheses for a nested row", async () => { + const text = formatTimeline(await describeProject(inversionProject())); expect(text).toContain("direct 20-25s"); expect(text).toContain("nested 6-8s (local 1-3s) in compositions/scene.html"); }); }); describe("formatTimeline", () => { - it("treats playback rate 1 as unset and does not expand a host nested inside a sub-composition", () => { + it("treats playback rate 1 as unset and does not expand a host nested inside a sub-composition", async () => { const index = project(); writeFileSync( join(dir, "compositions", "title.html"), ``, ); - const title = describeProject(index) - .tracks.flatMap((t) => t.rows) + const title = (await describeProject(index)).tracks + .flatMap((t) => t.rows) .find((r) => r.id === "title")!; expect(title.children.map((c) => [c.id, c.playbackRate, c.children.length])).toEqual([ ["v", null, 0], @@ -239,19 +477,19 @@ describe("formatTimeline", () => { ]); }); - it("prints a small volume unrounded to two decimals", () => { + it("prints a small volume unrounded to two decimals", async () => { const index = project(); writeFileSync( index, `
`, ); - const text = formatTimeline(describeProject(index)); + const text = formatTimeline(await describeProject(index)); expect(text).toContain("vol=0.009772"); expect(text).toContain("2.317-3.317s"); }); - it("prints one bar per row under its track heading", () => { - const text = formatTimeline(describeProject(project())); + it("prints one bar per row under its track heading", async () => { + const text = formatTimeline(await describeProject(project())); expect(text).toMatch(/^timeline 10s\n\nvideo \(1\)\n \|█{16}/); expect(text).toContain("audio (2)"); expect(text).toContain("vol=0.5 group=vo volume[0:0.2 2:1]"); @@ -280,17 +518,23 @@ const hasJq = (() => { } })(); -const runOneLiner = (line: string): string => - execFileSync("bash", ["-c", line], { - env: { ...process.env, TL: JSON.stringify({ timeline: describeProject(inversionProject()) }) }, - encoding: "utf8", - }); +const runOneLiner = async (line: string): Promise => { + const own = inversionProject(); + try { + return execFileSync("bash", ["-c", line], { + env: { ...process.env, TL: JSON.stringify({ timeline: await describeProject(own) }) }, + encoding: "utf8", + }); + } finally { + rmSync(dirname(own), { recursive: true, force: true }); + } +}; describe("skill query one-liners", () => { - it("documents four node one-liners, each answering from the fixture", () => { + it("documents four node one-liners, each answering from the fixture", async () => { const lines = oneLiners("node -e"); expect(lines).toHaveLength(4); - const [at = "", find, track, gaps] = lines.map(runOneLiner); + const [at = "", find, track, gaps] = await Promise.all(lines.map(runOneLiner)); expect(at.split("\n").filter(Boolean)).toEqual([ "host index.html", "nested compositions/scene.html", @@ -300,11 +544,11 @@ describe("skill query one-liners", () => { expect(gaps).toBe(""); }); - it.skipIf(!hasJq)("documents four jq one-liners that agree with the node ones", () => { + it.skipIf(!hasJq)("documents four jq one-liners that agree with the node ones", async () => { const lines = oneLiners("jq"); expect(lines).toHaveLength(4); - const [at, find, track, gaps] = lines.map((l) => - JSON.parse(`[${runOneLiner(l).replace(/}\s*{/g, "},{")}]`), + const [at, find, track, gaps] = (await Promise.all(lines.map(runOneLiner))).map((out) => + JSON.parse(`[${out.replace(/}\s*{/g, "},{")}]`), ); expect(at[0].map((r: { id: string }) => r.id)).toEqual(["host", "nested"]); expect(find).toEqual([ @@ -314,3 +558,51 @@ describe("skill query one-liners", () => { expect(gaps[0]).toEqual([]); }); }); + +describe("shared media-duration fixtures", () => { + const probeFree = MEDIA_DURATION_FIXTURES.filter( + (f) => f.tag === "img" || f.sourceDurationSeconds === null || f.expected.source === "authored", + ); + + it.each(probeFree)("$name", async ({ tag, attrs, expected }) => { + const attrText = Object.entries(attrs) + .map(([k, v]) => `${k}="${v}"`) + .join(" "); + const { rows } = await rowsOf( + `
<${tag} id="x" src="absent.bin" ${attrText}>
`, + ); + expect(rows[0]).toMatchObject({ + durationSource: expected.source, + duration: expected.seconds ?? 0, + }); + }); +}); + +describe("createProbeGate", () => { + it("never runs more than its limit at once, even for jobs arriving while slots are held", async () => { + const gate = createProbeGate(4); + const releases: Array<() => void> = []; + let running = 0; + let peak = 0; + const job = () => + gate(async () => { + running += 1; + peak = Math.max(peak, running); + await new Promise((done) => releases.push(done)); + running -= 1; + }); + const tick = () => new Promise((r) => setTimeout(r, 0)); + const jobs = Array.from({ length: 8 }, job); + await tick(); + releases.shift()!(); + await tick(); + jobs.push(...Array.from({ length: 8 }, job)); + await tick(); + while (releases.length > 0) { + releases.shift()!(); + await tick(); + } + await Promise.all(jobs); + expect(peak).toBe(4); + }); +}); diff --git a/packages/cli/src/timeline/describeProject.ts b/packages/cli/src/timeline/describeProject.ts index eab1d4dba9..0438eb6a9e 100644 --- a/packages/cli/src/timeline/describeProject.ts +++ b/packages/cli/src/timeline/describeProject.ts @@ -9,6 +9,13 @@ import { HF_AUDIO_FX_ATTR, parseAudioFxChain } from "@hyperframes/core/audio-fx" import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups"; import { byStart, type ClipFact, type ClipLane } from "@hyperframes/core/clip-facts"; import { parseNumeric } from "@hyperframes/core"; +import { + readMediaOffsetSeconds, + readPlaybackRate, + resolveMediaDuration, + type MediaDurationSource, + type MediaTag, +} from "@hyperframes/parsers/media-duration"; import { topLevelElements, trackKindOf, @@ -16,12 +23,24 @@ import { type TrackKind, } from "@hyperframes/parsers"; import { resolveMediaStartSeconds } from "@hyperframes/core/media-timing"; -import { resolveReferencedDuration, resolveReferencedStart } from "@hyperframes/engine"; +import { + extractAudioMetadata, + extractMediaMetadata, + resolveReferencedDuration, + resolveReferencedStart, +} from "@hyperframes/engine"; + +/** How `duration` was determined: the parsers resolver's names for media, "inner" for a composition host. */ +export type DurationSource = MediaDurationSource | "inner"; export interface TimelineRow extends ClipFact { trackKind: TrackKind; /** False when the source does not author a duration (media length is only known at render). */ durationAuthored: boolean; + /** Where `duration` came from; `null` for a non-media row with nothing authored and no children to sum. */ + durationSource: DurationSource | null; + /** Why no duration could be resolved; `null` unless `durationSource` is "pending". */ + pendingReason: string | null; /** Why `data-automation` / `data-fx-chain` could not be read; `null` when fine or absent. */ laneError: string | null; /** Start and end on the main timeline, in seconds. `start`/`end` are local to the owning file's composition. */ @@ -91,6 +110,113 @@ interface DocScope { origin: number; /** Project-relative path of this document, with `/` separators. */ file: string; + /** Bounds concurrent ffprobe spawns for the whole run. Shared reference, not new per document. */ + withProbeSlot: (fn: () => Promise) => Promise; + measure: MeasureMedia; +} + +/** Source length in seconds of a media file. ffprobe in production; tests pass a recorded fake. */ +export type MeasureMedia = (file: string, tag: MediaTag) => Promise; + +const measureWithFfprobe: MeasureMedia = async (file, tag) => + (tag === "audio" ? await extractAudioMetadata(file) : await extractMediaMetadata(file)) + .durationSeconds; + +const MEDIA_TAG = /^(video|audio|img)$/; +const PROBE_CONCURRENCY = 4; + +/** ponytail: a 4-line gate beats importing producer's Semaphore, which would pull its whole + * dependency tree into the lightweight `timeline` command just to cap ffprobe spawns. */ +export function createProbeGate(max: number) { + let active = 0; + const waiting: Array<() => void> = []; + return async function withProbeSlot(fn: () => Promise): Promise { + if (active >= max) await new Promise((wake) => waiting.push(wake)); + else active += 1; + try { + return await fn(); + } finally { + const next = waiting.shift(); + if (next) next(); + else active -= 1; + } + }; +} + +type ProbeResult = { ok: true; seconds: number } | { ok: false; reason: string }; + +/** ffprobe length of a media source. `extractMediaMetadata` and `extractAudioMetadata` already + * memoize per resolved file path for the process lifetime. */ +async function probeSource(scope: DocScope, el: Element, tag: MediaTag): Promise { + const src = el.getAttribute("src"); + if (!src) return { ok: false, reason: "no src attribute" }; + if (/^https?:\/\//i.test(src)) return { ok: false, reason: "remote source not probed" }; + const file = realFileInside(scope.projectDir, resolve(scope.dir, src)); + if (!file) return { ok: false, reason: "source file not found" }; + return scope.withProbeSlot(async () => { + try { + return { ok: true, seconds: await scope.measure(file, tag) } as const; + } catch (err) { + return { ok: false, reason: err instanceof Error ? err.message : String(err) } as const; + } + }); +} + +interface DurationResolution { + duration: number; + durationSource: DurationSource | null; + pendingReason: string | null; +} + +function resolveContainerDuration( + authored: number | null, + children: readonly TimelineRow[], +): DurationResolution { + if (authored !== null) + return { duration: authored, durationSource: "authored", pendingReason: null }; + if (children.length === 0) return { duration: 0, durationSource: null, pendingReason: null }; + const inner = children.reduce((max, c) => Math.max(max, c.end), 0); + return { duration: inner, durationSource: "inner", pendingReason: null }; +} + +/** Media rows go through the parsers resolver; only a row it cannot settle without the file is probed. */ +async function resolveMediaRowDuration( + scope: DocScope, + el: Element, + tag: MediaTag, + authored: number | null, +): Promise { + const getAttr = (name: string) => el.getAttribute(name); + const input = { + tag, + authoredDurationSeconds: authored, + mediaStartSeconds: readMediaOffsetSeconds(getAttr), + playbackRate: readPlaybackRate(getAttr), + }; + const unprobed = resolveMediaDuration({ ...input, sourceDurationSeconds: null }); + if (unprobed.source !== "pending") { + return { + duration: unprobed.seconds ?? 0, + durationSource: unprobed.source, + pendingReason: null, + }; + } + const probe = await probeSource(scope, el, tag); + const measured = probe.ok && probe.seconds > 0 ? probe.seconds : null; + const result = resolveMediaDuration({ ...input, sourceDurationSeconds: measured }); + return { + duration: result.seconds ?? 0, + durationSource: result.source, + pendingReason: result.source === "pending" ? pendingReason(probe, result.reason) : null, + }; +} + +/** Why a media row is pending: the probe's failure, or that it opened but reported no length. */ +function pendingReason(probe: ProbeResult, resolverReason: string | undefined): string { + if (!probe.ok) return probe.reason; + return probe.seconds > 0 + ? (resolverReason ?? "source duration unavailable") + : "source reports no duration"; } const roundMs = (v: number) => Math.round(v * 1000) / 1000; @@ -108,18 +234,19 @@ function mainTimelineStart(scope: DocScope, el: Element, start: number): number }); } -function describeRow(scope: DocScope, node: DomNode, depth: number): TimelineRow { +async function describeRow(scope: DocScope, node: DomNode, depth: number): Promise { const { el } = node; const { doc, startCache } = scope; const start = resolveReferencedStart(doc, el, startCache, new Set()); const authored = resolveReferencedDuration(doc, el, startCache, new Set()); const host = el.getAttribute("data-composition-src"); const absStart = mainTimelineStart(scope, el, start); - const children = host && depth === 0 ? readSubComposition(host, scope, absStart) : []; - const inner = children.reduce((max, c) => Math.max(max, c.end), 0); - const duration = authored ?? inner; - const rate = parseNumeric(el.getAttribute("data-playback-rate")); + const children = host && depth === 0 ? await readSubComposition(host, scope, absStart) : []; const kind = el.tagName.toLowerCase(); + const { duration, durationSource, pendingReason } = MEDIA_TAG.test(kind) + ? await resolveMediaRowDuration(scope, el, kind as MediaTag, authored) + : resolveContainerDuration(authored, children); + const rate = parseNumeric(el.getAttribute("data-playback-rate")); return { id: el.id || el.getAttribute("data-composition-id") || kind, label: null, @@ -140,11 +267,17 @@ function describeRow(scope: DocScope, node: DomNode, depth: number): TimelineRow audioGroup: el.getAttribute(HF_AUDIO_GROUP_ATTR), role: null, durationAuthored: authored !== null, + durationSource, + pendingReason, children, }; } -function readSubComposition(src: string, parent: DocScope, origin: number): TimelineRow[] { +async function readSubComposition( + src: string, + parent: DocScope, + origin: number, +): Promise { const authored = resolve(parent.dir, src); const file = realFileInside(parent.projectDir, authored); if (!file) return []; @@ -159,10 +292,13 @@ function readSubComposition(src: string, parent: DocScope, origin: number): Time projectDir: parent.projectDir, origin, file: relative(parent.projectDir, authored).split(sep).join("/"), + withProbeSlot: parent.withProbeSlot, + measure: parent.measure, }; - return topLevelElements(toNode(root)) - .map((node) => describeRow(scope, node, 1)) - .sort(byStart); + const rows = await Promise.all( + topLevelElements(toNode(root)).map((node) => describeRow(scope, node, 1)), + ); + return rows.sort(byStart); } /** The file's real path when it is a regular file inside the project (symlinks resolved), else null. */ @@ -178,7 +314,10 @@ function realFileInside(projectDir: string, path: string): string | null { } /** Needs a global DOMParser (`ensureDOMParser`). Reads `index.html` and one level of sub-compositions. */ -export function describeProject(indexPath: string): ProjectTimeline { +export async function describeProject( + indexPath: string, + measure: MeasureMedia = measureWithFfprobe, +): Promise { const doc = new DOMParser().parseFromString(readFileSync(indexPath, "utf-8"), "text/html"); const root = doc.querySelector("[data-composition-id]") ?? doc.body; const dir = dirname(indexPath); @@ -189,10 +328,12 @@ export function describeProject(indexPath: string): ProjectTimeline { projectDir: dir, origin: 0, file: basename(indexPath), + withProbeSlot: createProbeGate(PROBE_CONCURRENCY), + measure, }; - const rows = topLevelElements(toNode(root)) - .map((node) => describeRow(scope, node, 0)) - .sort(byStart); + const rows = ( + await Promise.all(topLevelElements(toNode(root)).map((node) => describeRow(scope, node, 0))) + ).sort(byStart); const declared = parseNumeric(root.getAttribute("data-duration")); return { duration: declared ?? rows.reduce((max, r) => Math.max(max, r.end), 0), diff --git a/packages/cli/src/timeline/formatTimeline.ts b/packages/cli/src/timeline/formatTimeline.ts index 11969e1745..6bff8ef0ca 100644 --- a/packages/cli/src/timeline/formatTimeline.ts +++ b/packages/cli/src/timeline/formatTimeline.ts @@ -13,6 +13,22 @@ function bar(row: TimelineRow, total: number): string { return " ".repeat(from) + fill.repeat(width) + " ".repeat(BAR_WIDTH - from - width); } +/** Never "unauthored": says either the resolved length's source or why one is pending. */ +function durationNote(row: TimelineRow): string | false { + switch (row.durationSource) { + case "media": + case "default": + return `duration=${row.durationSource}`; + case "inner": + return "duration=inferred"; + case "pending": + return `pending: ${row.pendingReason}`; + case "authored": + case null: + return false; + } +} + function details(row: TimelineRow): string { const lanes = row.lanes.map( (l) => `${l.target}[${l.points.map((p) => `${n(p.t)}:${n(p.v)}`).join(" ")}]`, @@ -22,7 +38,7 @@ function details(row: TimelineRow): string { row.volume !== null && `vol=${row.volume}`, row.playbackRate !== null && `rate=${n(row.playbackRate)}`, row.audioGroup && `group=${row.audioGroup}`, - !row.durationAuthored && "duration=unauthored", + durationNote(row), row.sourceFile && !row.children.length && "children=unread", row.laneError && `lanes unreadable: ${row.laneError}`, ...lanes, diff --git a/packages/parsers/src/mediaDuration.ts b/packages/parsers/src/mediaDuration.ts index a1a5c14136..02c62fe682 100644 --- a/packages/parsers/src/mediaDuration.ts +++ b/packages/parsers/src/mediaDuration.ts @@ -125,5 +125,4 @@ export const PENDING_MEDIA_DURATION_READERS = [ "engine/src/services/audioMixer.ts", "parsers/src/htmlParser.ts (defaults to 5s, reads only data-media-start)", "studio (timelineDOM.ts, timelineElementHelpers.ts, useTimelineSyncCallbacks.ts)", - "cli/src/timeline/describeProject.ts describeRow (#4138)", ] as const; diff --git a/skills-manifest.json b/skills-manifest.json index 0f9e21c193..a04ab456c6 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -30,7 +30,7 @@ "files": 7 }, "hyperframes-cli": { - "hash": "8fbf0e81f55bbf39", + "hash": "3fa86bc183a976c3", "files": 11 }, "hyperframes-core": { diff --git a/skills/hyperframes-cli/references/upgrade-info-misc.md b/skills/hyperframes-cli/references/upgrade-info-misc.md index 38d5233685..e7f9559cf0 100644 --- a/skills/hyperframes-cli/references/upgrade-info-misc.md +++ b/skills/hyperframes-cli/references/upgrade-info-misc.md @@ -47,10 +47,10 @@ audio (1) - The bar is 40 columns over the whole timeline. Times are seconds. - `src=`, `vol=`, `rate=` (playback rate, only when not 1), `group=` (audio group), and `[t:v ...]` (automation lane points, `t` in seconds from the clip start) appear only when the clip has them. - Clips of a sub-composition are indented one level. Their time is printed as the absolute main-timeline start-end first, then `(local -s)` (time inside that sub-composition), then `in ` (the file that declares the clip). Deeper nesting is not expanded (`children=unread` on that row). -- `duration=unauthored` (dotted bar) means the element has no `data-duration`/`data-end`, so its length is only known at render time (typically media). Add a `data-duration` if the length matters. +- With no `data-duration`/`data-end`, a media row still gets a resolved length and says where it came from: `duration=media` means ffprobe measured the source (with playback start and rate applied); `duration=default` means an `img` got the 3s default; `duration=inferred` means a composition host summed its children; `pending: ` (dotted bar, `duration` 0) means the source could not be probed (missing file, remote `src`, ffprobe error). A non-media leaf with nothing to resolve prints no source. - `lanes unreadable: ...` means the clip's `data-automation` or `data-fx-chain` did not parse; fix the attribute. -`--json` prints `{ timeline: { duration, tracks: [{ kind, rows: [...] }] } }`. Each row has `id`, `kind` (tag), `trackKind`, `start`, `duration`, `end` (local to the row's own file), **`absStart`, `absEnd`, `file`** (main-timeline time and the project-relative file that declares the clip — use these, not `start`/`end`, to compare clips across nesting), `trackIndex`, `src`, `sourceFile`, `volume`, `lanes`, `playbackRate`, `audioGroup`, `durationAuthored`, `laneError`, and `children` (the sub-composition's rows, one level). +`--json` prints `{ timeline: { duration, tracks: [{ kind, rows: [...] }] } }`. Each row has `id`, `kind` (tag), `trackKind`, `start`, `duration`, `end` (local to the row's own file), **`absStart`, `absEnd`, `file`** (main-timeline time and the project-relative file that declares the clip — use these, not `start`/`end`, to compare clips across nesting), `trackIndex`, `src`, `sourceFile`, `volume`, `lanes`, `playbackRate`, `audioGroup`, `durationAuthored`, **`durationSource`** (`"authored" | "media" | "default" | "inner" | "pending"`, or `null` for a non-media leaf with nothing to resolve), **`pendingReason`** (why nothing resolved; `null` unless `durationSource` is `"pending"`), `laneError`, and `children` (the sub-composition's rows, one level). ### Query one-liners (jq, node fallback if jq is absent)