feat(engine): render speed ramps with the same source-time mapping as the preview

packages/core/src/compiler/timingCompiler.test.ts

@@ -468,3 +468,29 @@ describe("shouldClampResolvedMediaDuration", () => {
     expect(shouldClampResolvedMediaDuration("audio", 5, 1)).toBe(true);
   });
 });
+
+describe("rate lane in compiled media", () => {
+  const lane = {
+    version: 1,
+    lanes: [
+      {
+        target: "rate",
+        points: [
+          { t: 0, v: 1 },
+          { t: 2, v: 3 },
+        ],
+      },
+    ],
+  };
+  const encodings = [
+    ["single-quoted JSON", `data-automation='${JSON.stringify(lane)}'`],
+    ["entity-encoded JSON", `data-automation="${JSON.stringify(lane).replace(/"/g, """)}"`],
+  ];
+
+  it.each(encodings)("hands a %s rate lane to the unresolved element", (_name, attr) => {
+    const { unresolved } = compileTimingAttrs(
+      `<video id="v" src="a.mp4" data-start="0" ${attr}></video>`,
+    );
+    expect(unresolved[0]?.playbackRate).toMatchObject({ target: "rate" });
+  });
+});

packages/core/src/compiler/timingCompiler.ts

@@ -21,9 +21,10 @@
 import { parseNumeric } from "@hyperframes/parsers/composition-contract";
 import {
   parseStrictFiniteTimingNumber,
-  readElementPlaybackRate,
+  readElementRateSpec,
   readMediaStart,
 } from "../runtime/playbackRate.js";
+import type { RateSpec } from "../speedRamp.js";
 // ── Types ────────────────────────────────────────────────────────────────
 
 export interface UnresolvedElement {
@@ -34,7 +35,7 @@ export interface UnresolvedElement {
   end?: number;
   duration?: number;
   mediaStart: number;
-  playbackRate: number;
+  playbackRate: RateSpec;
   compositionSrc?: string;
 }
 
@@ -50,7 +51,7 @@ export interface ResolvedMediaElement {
   start: number;
   duration: number;
   mediaStart: number;
-  playbackRate: number;
+  playbackRate: RateSpec;
   loop: boolean;
 }
 
@@ -91,8 +92,25 @@ function getAttr(tag: string, attr: string): string | null {
   // made compileTag believe a Studio-stamped `data-hf-id`-only element already
   // had an `id`, so it skipped its `hf-video-N` injection — leaving the element
   // with no real `el.id`, which the render pipeline keys off of (blank wash).
-  const match = tag.match(new RegExp(`(?<![\\w-])${attr}=["']([^"']+)["']`));
-  return match ? (match[1] ?? null) : null;
+  const match = tag.match(new RegExp(`(?<![\\w-])${attr}=(?:"([^"]+)"|'([^']+)')`));
+  return match ? (match[1] ?? match[2] ?? null) : null;
+}
+
+/** An attribute reader over tag source that decodes the entities the DOM would, for JSON-valued attributes. */
+function tagAttrReader(tag: string): Pick<Element, "getAttribute"> {
+  return {
+    getAttribute: (name) => {
+      const raw = getAttr(tag, name);
+      return raw && name === "data-automation"
+        ? raw
+            .replace(/&quot;/g, '"')
+            .replace(/&#39;/g, "'")
+            .replace(/&lt;/g, "<")
+            .replace(/&gt;/g, ">")
+            .replace(/&amp;/g, "&")
+        : raw;
+    },
+  };
 }
 
 function hasAttr(tag: string, attr: string): boolean {
@@ -228,9 +246,9 @@ function compileTag(
     startStr = "0";
   }
   const start = parseNumeric(startStr);
-  const attrReader = { getAttribute: (name: string) => getAttr(result, name) };
+  const attrReader = tagAttrReader(result);
   const mediaStart = readMediaStart(attrReader);
-  const playbackRate = readElementPlaybackRate(attrReader);
+  const playbackRate = readElementRateSpec(attrReader);
 
   // 1. Compute data-end from data-start + data-duration. Skip relative id-refs.
   if (!hasAttr(result, "data-end")) {
@@ -374,7 +392,7 @@ export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
 
     const isVideo = /^<video/i.test(tag);
     const startStr = getAttr(tag, "data-start");
-    const attrReader = { getAttribute: (name: string) => getAttr(tag, name) };
+    const attrReader = tagAttrReader(tag);
 
     resolved.push({
       id,
@@ -383,7 +401,7 @@ export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
       start: parseNumeric(startStr) ?? 0,
       duration,
       mediaStart: readMediaStart(attrReader),
-      playbackRate: readElementPlaybackRate(attrReader),
+      playbackRate: readElementRateSpec(attrReader),
       loop: hasAttr(tag, "loop"),
     });
   }

packages/core/src/index.ts

@@ -321,12 +321,15 @@ export {
 export { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
 export {
   normalizePlaybackRate,
+  normalizeRateSpec,
   parseStrictFiniteTimingNumber,
   readElementPlaybackRate,
+  readElementRateSpec,
   readMediaStart,
   resolveNaturalMediaTimelineDuration,
   resolveNaturalMediaTimelineDurationFromValues,
 } from "./runtime/playbackRate.js";
+export { shiftRateLane, sourceTimeAt, timeAtSourceTime, type RateSpec } from "./speedRamp.js";
 
 // Variable validation (CLI / tooling-side)
 export {

packages/core/src/runtime/media.test.ts

@@ -380,7 +380,6 @@ describe("syncRuntimeMedia", () => {
       expect(clip.el.currentTime).toBeCloseTo(3.641, 2);
       syncRuntimeMedia({ clips: [clip], timeSeconds: 3, playing: true, playbackRate: 1 });
       expect(clip.el.playbackRate).toBeCloseTo(3, 5);
-      expect(clip.el.preservesPitch).toBe(true);
     });
   });
 

packages/core/src/runtime/media.ts

@@ -2,13 +2,7 @@ import { swallow } from "./diagnostics";
 import { interpolateVolumeGain, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
 import { elementVolumeLaneGain } from "./audioAutomationVolume.js";
 import { readElementPlaybackRate, readElementRateSpec, readMediaStart } from "./playbackRate.js";
-import {
-  rateAt,
-  readPreservePitch,
-  sourceTimeAt,
-  timeAtSourceTime,
-  type RateSpec,
-} from "../speedRamp.js";
+import { rateAt, sourceTimeAt, timeAtSourceTime, type RateSpec } from "../speedRamp.js";
 import { clampAudioGain } from "../audioGain.js";
 import { isMemberGroupHidden } from "../audioGroups.js";
 import { findInjectedRenderFrame } from "./renderFrameSibling.js";
@@ -386,7 +380,6 @@ export function syncRuntimeMedia(params: {
       try {
         // Per-element rate × global transport rate
         el.playbackRate = rateAt(clipRate, params.timeSeconds - clip.start) * params.playbackRate;
-        el.preservesPitch = readPreservePitch(el);
       } catch (err) {
         // ignore unsupported playbackRate
         swallow("runtime.media.site1", err);

packages/core/src/runtime/playbackRate.ts

@@ -24,6 +24,11 @@ export function readElementPlaybackRate(el: Pick<Element, "getAttribute">): numb
   return normalizePlaybackRate(raw);
 }
 
+/** A constant rate clamped to the shared range; a lane is already normalised by its parser. */
+export function normalizeRateSpec(spec: RateSpec | undefined): RateSpec {
+  return typeof spec === "object" ? spec : normalizePlaybackRate(spec ?? 1);
+}
+
 /** The clip's rate: its `rate` lane when present, otherwise the constant rate. */
 export function readElementRateSpec(el: Pick<Element, "getAttribute">): RateSpec {
   return resolveRateSpec(el.getAttribute("data-automation"), readElementPlaybackRate(el));

packages/core/src/speedRamp.test.ts

@@ -4,8 +4,8 @@ import {
   SPEED_PRESETS,
   parseRateLane,
   rateAt,
-  readPreservePitch,
   resolveRateSpec,
+  shiftRateLane,
   sourceTimeAt,
   speedPresetLane,
   timeAtSourceTime,
@@ -71,6 +71,31 @@ describe("rateAt", () => {
   });
 });
 
+describe("shiftRateLane", () => {
+  it("sees the lane from dt seconds in: integrating from 1s of a 4s 1x to 3x ramp", () => {
+    const lane = ramp([
+      [0, 1],
+      [4, 3],
+    ]);
+    // source(3) - source(1) = 4.6586 - 1.1508
+    expect(sourceTimeAt(shiftRateLane(lane, 1), 2)).toBeCloseTo(3.5078, 2);
+  });
+});
+
+describe("shiftRateLane with a shaped segment", () => {
+  it("keeps the curve of a segment cut in the middle: shifted integral equals the original's difference", () => {
+    const lane: HfAutomationLane = {
+      target: RATE_TARGET,
+      points: [
+        { t: 0, v: 1, curve: 0.8 },
+        { t: 4, v: 3 },
+      ],
+    };
+    const expected = sourceTimeAt(lane, 4) - sourceTimeAt(lane, 1);
+    expect(sourceTimeAt(shiftRateLane(lane, 1), 3)).toBeCloseTo(expected, 2);
+  });
+});
+
 describe("lane parsing", () => {
   const attr = JSON.stringify({
     version: 1,
@@ -88,18 +113,12 @@ describe("lane parsing", () => {
   });
 });
 
-describe("presets and pitch", () => {
+describe("presets", () => {
   it("stretches every preset over the clip and keeps it in range", () => {
     for (const { id } of SPEED_PRESETS) {
       const lane = speedPresetLane(id, 8);
       expect(lane.points[lane.points.length - 1]!.t).toBe(8);
       for (const p of lane.points) expect(p.v).toBeGreaterThanOrEqual(0.1);
     }
   });
-
-  it("preserves pitch unless the clip opts out", () => {
-    const el = (v: string | null) => ({ getAttribute: () => v });
-    expect(readPreservePitch(el(null))).toBe(true);
-    expect(readPreservePitch(el("false"))).toBe(false);
-  });
 });

packages/core/src/speedRamp.ts

@@ -7,7 +7,25 @@ import { MAX_PLAYBACK_RATE, MIN_PLAYBACK_RATE } from "./playbackRateBounds.js";
 /** A constant multiplier, or a lane whose `v` is the multiplier over clip-local time. */
 export type RateSpec = number | HfAutomationLane;
 
-const PRESERVE_PITCH_ATTR = "data-preserve-pitch";
+const SHIFT_RESAMPLES = 16;
+
+/**
+ * The lane as seen from `dt` seconds into the clip: what a renderer sees after it trims the clip's start.
+ * A shaped segment (`curve`, `viaX`) cut in the middle is resampled, since its shape belongs to its left point.
+ */
+export function shiftRateLane(spec: RateSpec, dt: number): RateSpec {
+  if (typeof spec === "number" || dt === 0) return spec;
+  const next = spec.points.findIndex((p) => p.t > dt);
+  const later = next < 0 ? [] : spec.points.slice(next).map((p) => ({ ...p, t: p.t - dt }));
+  const left = next > 0 ? spec.points[next - 1] : undefined;
+  const shaped = left !== undefined && (left.curve || left.viaX !== undefined);
+  const cut = shaped && next > 0 ? spec.points[next]!.t - dt : 0;
+  const samples = Array.from({ length: shaped ? SHIFT_RESAMPLES - 1 : 0 }, (_, k) => {
+    const t = ((k + 1) * cut) / SHIFT_RESAMPLES;
+    return { t, v: rateAt(spec, dt + t) };
+  });
+  return { ...spec, points: [{ t: 0, v: rateAt(spec, dt) }, ...samples, ...later] };
+}
 
 const CELLS_PER_SEGMENT = 48;
 
@@ -114,11 +132,6 @@ export function resolveRateSpec(
   return parseRateLane(automationAttr) ?? constant;
 }
 
-/** Pitch is preserved unless the clip opts out with `data-preserve-pitch="false"`. */
-export function readPreservePitch(el: Pick<Element, "getAttribute">): boolean {
-  return el.getAttribute(PRESERVE_PITCH_ATTR) !== "false";
-}
-
 interface RatePreset {
   id: string;
   label: string;

packages/engine/src/services/audioMixer.test.ts

@@ -390,6 +390,95 @@ describe("processCompositionAudio", () => {
     expect(runFfmpegMock.mock.calls[0]?.[0]).toEqual(expect.arrayContaining(["-af", filter]));
   });
 
+  it("bakes a rate lane as concatenated source slices whose tempo is the mean rate over each slice", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+    writeFileSync(join(baseDir, "timecode.wav"), "stub");
+    const rate = {
+      target: "rate",
+      points: [
+        { t: 0, v: 2 },
+        { t: 1, v: 2 },
+      ],
+    };
+
+    await processCompositionAudio(
+      [
+        {
+          id: "timecode",
+          src: "timecode.wav",
+          start: 0,
+          end: 1,
+          mediaStart: 0,
+          playbackRate: rate,
+          layer: 0,
+          volume: 1,
+          type: "audio",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      1,
+    );
+
+    const args: string[] = runFfmpegMock.mock.calls[0]?.[0] ?? [];
+    const graph = args[args.indexOf("-filter_complex") + 1] ?? "";
+    // a constant 2x lane: 1s of composition consumes 2s of source, in 4 quarter-second slices
+    expect(args).toEqual(expect.arrayContaining(["-t", "2", "-map", "[out]"]));
+    expect(graph).toContain("asplit=4");
+    expect(graph).toContain(
+      "[s0]atrim=start=0:end=0.5,asetpts=PTS-STARTPTS,atempo=2,apad,asetpts=N/SR/TB,atrim=0:0.25[t0]",
+    );
+    expect(graph).toContain(
+      "[s3]atrim=start=1.5:end=2,asetpts=PTS-STARTPTS,atempo=2,apad,asetpts=N/SR/TB,atrim=0:0.25[t3]",
+    );
+    expect(graph).toContain("concat=n=4:v=0:a=1");
+  });
+
+  it("stretches each slice of a non-constant lane by its mean rate, and trims a video's audio to the consumed span", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+    writeFileSync(join(baseDir, "clip.mp4"), "stub");
+    const rate = {
+      target: "rate",
+      points: [
+        { t: 0, v: 1 },
+        { t: 1, v: 3 },
+      ],
+    };
+
+    await processCompositionAudio(
+      [
+        {
+          id: "clip",
+          src: "clip.mp4",
+          start: 0,
+          end: 1,
+          mediaStart: 0,
+          playbackRate: rate,
+          layer: 0,
+          volume: 1,
+          type: "video",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      1,
+    );
+
+    const args: string[] = runFfmpegMock.mock.calls[0]?.[0] ?? [];
+    const graph = args[args.indexOf("-filter_complex") + 1] ?? "";
+    // geometric 1x to 3x over 1s consumes (3-1)/ln 3 = 1.8205 source seconds; quarter-second 0 has mean tempo 1.1508
+    expect(Number(args[args.indexOf("-t") + 1])).toBeCloseTo(1.8205, 2);
+    expect(
+      Number(graph.match(/\[s0\]atrim=[^,]*,asetpts=PTS-STARTPTS,atempo=([\d.]+)/)?.[1]),
+    ).toBeCloseTo(1.1508, 2);
+  });
+
   it("keeps automation on authored timeline time after constant retiming", async () => {
     const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
     const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));