From 18497e333251c3e992f14912df0720359075c3c6 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 15:47:59 -0500 Subject: [PATCH 001/251] chore: open 0.6.0-alpha.1 on develop --- plugins/dw/.claude-plugin/plugin.json | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/dw/.claude-plugin/plugin.json b/plugins/dw/.claude-plugin/plugin.json index bc87b5b5..550f41d3 100644 --- a/plugins/dw/.claude-plugin/plugin.json +++ b/plugins/dw/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "dw", "description": "Compose MiniMax H3 video, MiniMax Music 3 and LTX-2.5 workflows over a dw MCP server, and cut a multi-episode series from them: which template fits which shape, the hard rules, cost, and how to judge the output. Prompt format comes from the vendors' own guides.", - "version": "0.5.0", + "version": "0.6.0-alpha.1", "author": { "name": "Don Kackman" }, diff --git a/pyproject.toml b/pyproject.toml index 145b2741..365dfb44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ name = "diffusers-workflow" # runtime (static in TOML because importing dw at build time would drag # torch into the build environment). Release tags must match it - see # docs/RELEASING.md -version = "0.5.0" +version = "0.6.0-alpha.1" description = "Declarative workflow engine and web UI for Hugging Face Diffusers" readme = "README.md" requires-python = ">=3.10" From ff055249204bf4212764506db80943d568af4c75 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 17:30:29 -0500 Subject: [PATCH 002/251] fix(pipeline-and-docs): #511 - name the video/image mismatch behind the image_crf error, document for_each's sequential-dependency split Diffusers' image_crf re-compression error suggests image_crf=0 for a still-expected argument that actually received an AudioVideo - advice that does not fix the mismatch. Pipeline.run now rewrites that specific error to name the offending argument(s) and point at get_last_frame / video_frames. Also documents, in the for_each section of WORKFLOW_GUIDE.md, that conditioning list entry N on a derived still of entry N-1 can't be expressed inside one for_each list, and how to split the list at the dependency instead. Co-Authored-By: Claude Sonnet 5 --- docs/WORKFLOW_GUIDE.md | 17 +++++++++ dw/pipeline_processors/pipeline.py | 33 +++++++++++++++++ tests/test_pipeline_components.py | 58 ++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index f92e3c58..07168773 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -580,6 +580,23 @@ came from. Spell the member out: `{"reference_type": "…", fails the run with a precise error naming the members that do exist, but the guide says it here first. +A member may reference an *earlier* member of its own list directly +(`"image": "previous_result:shot@accuse"` inside `shot@deflect`), but it +cannot interpose a derivation on that reference — "shot 2 starts where shot +1 ended," a still pulled from shot 1's last frame with `get_last_frame`. +There is no step between two members of the same `for_each` list: a step +meant to derive that still (say, named `last`) would itself have to sit +after `shot` in the file to read `shot@accuse`'s result, which makes it a +step *after* the whole list, and referencing it from inside the list is a +`previous_result:` naming no earlier step — refused at validate. Split the +list at the dependency instead: run the members up to and including the one +being derived from as their own `for_each` (or, for a single entry, an +ordinary step), add an ordinary `get_last_frame` step after it reading +`previous_result:shot@accuse`, and run the remaining members as a second +`for_each` whose dependent entry takes that derived still — as a variable +value or, over the API/MCP, folded into the entry itself — rather than a +bare `previous_result:` reference to the video. + Limits: a list has at most 32 entries, and an empty list is a validation error — the step would run nothing. Validation realizes a `constant:` default before checking it, so a list defaulted to a constant validates the diff --git a/dw/pipeline_processors/pipeline.py b/dw/pipeline_processors/pipeline.py index f63662f5..30b695da 100644 --- a/dw/pipeline_processors/pipeline.py +++ b/dw/pipeline_processors/pipeline.py @@ -493,6 +493,10 @@ def run(self, arguments, previous_pipelines={}): logger.info("Pipeline run cancelled") raise except Exception as e: + diagnosed = _diagnose_image_crf_error(e, arguments) + if diagnosed is not None: + logger.error(f"{type(e).__name__} running pipeline: {e}", exc_info=True) + raise diagnosed from e # One log line with the full traceback - every error class was # logged and re-raised identically logger.error(f"{type(e).__name__} running pipeline: {e}", exc_info=True) @@ -1971,6 +1975,35 @@ def load_component( raise +def _diagnose_image_crf_error(error, arguments): + """Rewrite diffusers' `image_crf` re-compression error into one that + names the actual mismatch, when it is one: an argument the pipeline + expects as a still image (`PIL.Image.Image`) instead holds an + `AudioVideo` - a video where an image was required. Diffusers' own + message suggests `image_crf=0`, which does not fix anything, since the + argument still receives a video; the caller needs to derive a still + first. Returns None when the error is something else, so the caller + re-raises unchanged. + """ + if not isinstance(error, (ValueError, TypeError)): + return None + if "image_crf" not in str(error) or "re-compression requires" not in str(error): + return None + + from ..result import AudioVideo + + offending = [name for name, value in arguments.items() if isinstance(value, AudioVideo)] + if not offending: + return None + + names = ", ".join(sorted(offending)) + return ValueError( + f"Argument(s) {names} expected an image but received a video. Derive a " + f"still from it first - `get_last_frame` or `video_frames` - and pass " + f"that instead." + ) + + def _hub_auth_status(error): """The 401/403 status an exception (or anything in its cause/context chain) carries from the Hub, else None.""" diff --git a/tests/test_pipeline_components.py b/tests/test_pipeline_components.py index 35e92bdb..dece2481 100644 --- a/tests/test_pipeline_components.py +++ b/tests/test_pipeline_components.py @@ -759,3 +759,61 @@ def test_a_failed_load_unpublishes_what_it_had_shared(self, monkeypatch): pipeline.load(shared) assert shared == {} + + +class TestImageCrfMismatchDiagnosis: + """#511: diffusers' `image_crf` re-compression error names a re-encoding + knob that does not fix an argument holding a video where an image was + required - the caller needs a still, not a different crf.""" + + def _definition(self): + return { + "configuration": {"component_type": "{MockPipeline}"}, + "from_pretrained_arguments": {"model_name": "some/repo"}, + "arguments": {"image": "variable:image"}, + } + + def test_an_audiovideo_argument_is_named_in_the_rewritten_error(self, monkeypatch): + from dw.result import AudioVideo + + def _raise(self, arguments): + raise ValueError("re-compression requires you to set `image_crf`") + + monkeypatch.setattr(Pipeline, "_run_once", _raise) + + pipeline = Pipeline(self._definition(), 42, "cpu") + pipeline.pipeline = MagicMock() + + video = AudioVideo(frames=[], audio=None, sample_rate=None) + with pytest.raises(ValueError, match="expected an image but received a video"): + pipeline.run({"image": video}) + + def test_names_every_offending_argument(self): + from dw.result import AudioVideo + from dw.pipeline_processors.pipeline import _diagnose_image_crf_error + + video = AudioVideo(frames=[], audio=None, sample_rate=None) + error = ValueError("re-compression requires you to set `image_crf`") + + diagnosed = _diagnose_image_crf_error( + error, {"image": video, "conditioning_image": video, "prompt": "a cat"} + ) + + assert diagnosed is not None + assert "conditioning_image" in str(diagnosed) + assert "image" in str(diagnosed) + assert "get_last_frame" in str(diagnosed) + assert "video_frames" in str(diagnosed) + + def test_a_different_value_error_is_left_alone(self): + from dw.pipeline_processors.pipeline import _diagnose_image_crf_error + + error = ValueError("some unrelated failure") + assert _diagnose_image_crf_error(error, {"image": "not a video"}) is None + + def test_the_same_message_with_no_video_argument_is_left_alone(self): + """The text alone isn't enough - it has to actually be this mismatch.""" + from dw.pipeline_processors.pipeline import _diagnose_image_crf_error + + error = ValueError("re-compression requires you to set `image_crf`") + assert _diagnose_image_crf_error(error, {"image": "a/path.png"}) is None From fbeaacb407306e5b4e9719f7d7ecd2d53f032bf6 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 17:33:07 -0500 Subject: [PATCH 003/251] fix(pipeline): #511 - satisfy ruff format on _diagnose_image_crf_error Co-Authored-By: Claude Sonnet 5 --- dw/pipeline_processors/pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dw/pipeline_processors/pipeline.py b/dw/pipeline_processors/pipeline.py index 30b695da..cf754a7d 100644 --- a/dw/pipeline_processors/pipeline.py +++ b/dw/pipeline_processors/pipeline.py @@ -1992,7 +1992,9 @@ def _diagnose_image_crf_error(error, arguments): from ..result import AudioVideo - offending = [name for name, value in arguments.items() if isinstance(value, AudioVideo)] + offending = [ + name for name, value in arguments.items() if isinstance(value, AudioVideo) + ] if not offending: return None From 798fba53a00a62cc9857a60aeac41e48c857c733 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 17:34:14 -0500 Subject: [PATCH 004/251] fix(engine): #516 - scope vram_estimate projection to the pipeline it was measured for _projections used to project a workflow-level vram_estimate onto every step that loaded any pipeline, including image/music steps sharing only some of the estimate's voxel_variables (e.g. width/height without num_frames) - the same mistake vram_inheritance.py already avoids for inherited ceilings. _estimate_identity infers which pipeline identity the estimate describes: the one shared by every step that names *all* of the voxel_variables in its own pipeline arguments. When that is ambiguous (no qualifying step, disagreement, or no identity metadata at all - true of every workflow before #516 and most existing fixtures), projection falls back to every pipeline step, unchanged. No new schema surface - purely inferential, per the triage direction on #516. Co-Authored-By: Claude Sonnet 5 --- dw/vram_estimate.py | 59 ++++++++++++++++++++++++++++++++++++- tests/test_vram_estimate.py | 59 +++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/dw/vram_estimate.py b/dw/vram_estimate.py index 8fdb9772..635b7165 100644 --- a/dw/vram_estimate.py +++ b/dw/vram_estimate.py @@ -26,6 +26,19 @@ pass; one whose media resolved null is dropped before the pipeline sees it (#478) and costs nothing. +A workflow-level estimate was measured against one pipeline, not against +every step that happens to load one (#516) - a music-video template's image +step at a caller-chosen resolution has no business being judged by the H3 +video estimate beside it just because both name `width`/`height`. Which +identity the estimate describes is inferred rather than declared twice: the +one every step that names *all* of the estimate's `voxel_variables` in its +own pipeline arguments shares (`_estimate_identity`, using +`vram_inheritance.pipeline_identity`). When that comes out ambiguous - no +step owns every voxel variable directly, more than one identity does, or the +steps carry no pipeline identity metadata at all, as every workflow before +#516 and most of this module's own tests do - projection falls back to every +pipeline step, unchanged from before. + The entries checked are the serving device's own (see `_entries_for`); `bytes_per_voxel` was calibrated on CUDA, so a check against a Mac's capacity is an estimate of an estimate. @@ -124,6 +137,42 @@ def _entries_for(cost, device_type, capacity_gb): return entries +def _estimate_identity(steps, variables, names): + """The pipeline identity a workflow-level estimate describes, or None + when that is ambiguous. + + The identity every step that names *all* of `names` in its own pipeline + arguments shares - a step only partly naming the voxel variables (an + image step with `width`/`height` but no `num_frames`) is not a candidate, + so it cannot pull an unrelated estimate onto itself. None when no step + qualifies, when the qualifying steps disagree, or when the identity found + carries no real pipeline metadata (`pipeline_identity` returns None) - + each of those means "cannot tell", not "no pipeline", and the caller + falls back to projecting every step as it did before #516. + """ + if not names: + return None + from .vram_inheritance import pipeline_identity + + identities = set() + for step in steps: + if not isinstance(step, dict): + continue + pipeline = step.get("pipeline") + if not isinstance(pipeline, dict): + continue + step_arguments = pipeline.get("arguments") + step_arguments = step_arguments if isinstance(step_arguments, dict) else {} + if not all(name in step_arguments for name in names): + continue + identities.add(pipeline_identity(step, variables)) + if len(identities) == 1: + found = next(iter(identities)) + if found is not None: + return found + return None + + def _projections(definition, estimate, arguments): """(step index, step, values, references, projected GB) for every step the estimate can project. @@ -133,7 +182,10 @@ def _projections(definition, estimate, arguments): substituted, so a for_each member carries its entry's `num_frames` - and from the workflow's variables (the caller's arguments over them) when the step does not name it. A definition with no steps at all is projected - from its variables alone, as a single step. + from its variables alone, as a single step. When the estimate's own + pipeline identity can be determined (`_estimate_identity`), a step whose + identity does not match it is not projected at all (#516) - the estimate + was never measured for that pipeline. """ variables = {**(definition.get("variables") or {}), **(arguments or {})} steps = definition.get("steps") @@ -143,10 +195,15 @@ def _projections(definition, estimate, arguments): yield None, None, variables, 0, projected return names = estimate.get("voxel_variables", []) + identity = _estimate_identity(steps, variables, names) + if identity is not None: + from .vram_inheritance import pipeline_identity as _pipeline_identity for index, step in enumerate(steps): pipeline = step.get("pipeline") if isinstance(step, dict) else None if not isinstance(pipeline, dict): continue + if identity is not None and _pipeline_identity(step, variables) != identity: + continue step_arguments = pipeline.get("arguments") if not isinstance(step_arguments, dict): step_arguments = {} diff --git a/tests/test_vram_estimate.py b/tests/test_vram_estimate.py index e538a084..3d4d165c 100644 --- a/tests/test_vram_estimate.py +++ b/tests/test_vram_estimate.py @@ -324,6 +324,65 @@ def test_a_literal_for_each_list_names_the_steps_path(): assert "shot@b" in errors[0]["message"] +# --- A workflow-level estimate is scoped to the pipeline it was measured +# --- for, not projected onto every step that shares a voxel variable (#516) + + +def music_video_shaped_definition(image_width=768, image_height=768): + """An H3 ref2va estimate beside an unrelated image step, shaped like + music-video.json: both name width/height, only the H3 step also names + num_frames - which is what should keep the estimate off the image step.""" + return { + "cost": [{"device": "cuda", "name": "RTX 3090", "vram_gb": 24, "minutes": 1}], + "vram_estimate": { + "base_gb": 16.0, + "bytes_per_voxel": 28.71, + "gb_per_reference": 1.0, + "voxel_variables": ["width", "height", "num_frames"], + }, + "steps": [ + { + "name": "draw_singer", + "pipeline": { + "configuration": {"component_type": "ZImagePipeline"}, + "from_pretrained_arguments": { + "model_name": "Tongyi-MAI/Z-Image-Turbo" + }, + "arguments": {"width": image_width, "height": image_height}, + }, + "result": {"content_type": "image/png"}, + }, + { + "name": "shot", + "pipeline": { + "configuration": {"component_type": "ModularPipeline"}, + "from_pretrained_arguments": { + "model_name": "MiniMaxAI/MiniMax-H3", + "workflow": "ref2va", + }, + "arguments": {"width": 960, "height": 544, "num_frames": 124}, + }, + "result": {"content_type": "video/mp4"}, + }, + ], + } + + +def test_an_image_step_sharing_only_some_voxel_variables_is_not_projected(): + # 2048x2048 would project to ~29.9 GB under the H3 formula, but the + # image step never holds H3's memory - it should not be checked at all. + d = music_video_shaped_definition(image_width=2048, image_height=2048) + assert vram_estimate_errors(d) == [] + + +def test_the_matching_identity_still_projects_and_refuses(): + d = music_video_shaped_definition() + d["steps"][1]["pipeline"]["arguments"]["num_frames"] = 600 + errors = vram_estimate_errors(d) + assert len(errors) == 1 + assert "RTX 3090" in errors[0]["message"] + + # --- Run-time backstop ----------------------------------------------------- From deca88fc2808c487e1f2bb098f66a37575650308 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 17:44:52 -0500 Subject: [PATCH 005/251] fix(engine): #517 - write manifest.json atomically via temp file + os.replace write_manifest truncated manifest.json in place before writing, so a worker killed mid-dump (the case #480's per-step rewrite targets) left a truncated or empty file. _read_manifest then dropped the run from history entirely, even though the previous step's complete manifest had already been on disk. Write to a temp file in the run directory and os.replace() it over the manifest instead, so a killed write leaves the prior manifest untouched. Co-Authored-By: Claude Sonnet 5 --- dw/runs.py | 11 +++++++++-- tests/test_runs.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/dw/runs.py b/dw/runs.py index 19334bce..1d15e931 100644 --- a/dw/runs.py +++ b/dw/runs.py @@ -26,6 +26,7 @@ import logging import os import re +import tempfile from datetime import datetime, timezone logger = logging.getLogger("dw") @@ -605,8 +606,14 @@ def write_manifest(run_dir, manifest): path = os.path.join(run_dir, MANIFEST_FILE_NAME) try: os.makedirs(run_dir, exist_ok=True) - with open(path, "w") as file: - json.dump(manifest, file, indent=2, default=str) + fd, tmp_path = tempfile.mkstemp(dir=run_dir, prefix=f".{MANIFEST_FILE_NAME}-") + try: + with os.fdopen(fd, "w") as file: + json.dump(manifest, file, indent=2, default=str) + os.replace(tmp_path, path) + except OSError: + os.unlink(tmp_path) + raise except OSError as e: logger.warning(f"Could not write {path}: {e}") return None diff --git a/tests/test_runs.py b/tests/test_runs.py index 6dbb6fb5..c02669a6 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -10,6 +10,7 @@ from dw.runs import ( FLAT_LAYOUT, + MANIFEST_FILE_NAME, assign_run_version, record_run_versions, OUTPUT_LAYOUT_ENV_VAR, @@ -23,6 +24,7 @@ split_run_path, strip_run_id, workflow_identity, + write_manifest, ) @@ -227,6 +229,39 @@ def mock_load(self, shared_components): yield +class TestManifestAtomicity: + """#517: a worker killed mid-write must never leave manifest.json + truncated - the previous, complete manifest has to survive a failed + rewrite.""" + + def test_a_dump_failure_leaves_the_previous_manifest_intact(self, tmp_path): + run_dir = str(tmp_path) + write_manifest(run_dir, {"status": "running", "steps": ["gen0"]}) + + with patch("json.dump", side_effect=OSError("simulated kill mid-write")): + result = write_manifest( + run_dir, {"status": "running", "steps": ["gen0", "gen1"]} + ) + + assert result is None + manifest_path = tmp_path / MANIFEST_FILE_NAME + assert json.loads(manifest_path.read_text()) == { + "status": "running", + "steps": ["gen0"], + } + # no leftover temp file beside it + assert os.listdir(run_dir) == [MANIFEST_FILE_NAME] + + def test_a_successful_write_replaces_the_manifest(self, tmp_path): + run_dir = str(tmp_path) + write_manifest(run_dir, {"status": "running"}) + write_manifest(run_dir, {"status": "completed"}) + + manifest_path = tmp_path / MANIFEST_FILE_NAME + assert json.loads(manifest_path.read_text()) == {"status": "completed"} + assert os.listdir(run_dir) == [MANIFEST_FILE_NAME] + + class TestRunDirectories: def test_each_run_writes_its_own_directory(self, tmp_path, fake_pipeline): from dw.workflow import Workflow From d87b06bc08e69d0ecff126291d618d558c156151 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 17:52:27 -0500 Subject: [PATCH 006/251] fix(engine-and-docs): #518, #520, #522 - unwrap {"location"} before video size check, fix TASKS.md tables #518: video_size_errors unwraps a {"location": ...} dict's inner path before resolve_probe_path, so concat_videos/dissolve_videos entries in that form (#510) get the same validate-time frame-size check a plain asset:/output:/ literal path already gets, instead of failing only at run time. #520: move the attribute_voices `separate`/`device` rows into the argument table, above the prose paragraph that had swallowed them as text. #522: document the {"location": ...} entry form on both concat_videos and dissolve_videos `videos` rows, and note it is now size-checked at validate like any other statically-resolvable entry. Co-Authored-By: Claude Sonnet 5 --- docs/TASKS.md | 8 ++++---- dw/video_size_errors.py | 16 ++++++++++------ tests/test_video_size_errors.py | 26 ++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 5fb402dc..47cbe467 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -212,7 +212,7 @@ video generation" in the workflow guide): | Argument | Required | Description | | -------- | -------- | ----------- | -| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, which is read with the audio muxed into it. Every video must be the same frame size - unlike a sample-rate mismatch, there is no reconciliation for a size mismatch, so a statically-resolvable (`asset:`/`output:`/literal path) size disagreement is refused at validate; one only known at run time still fails there (#504) | +| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, which is read with the audio muxed into it; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, there is no reconciliation for a size mismatch, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518) | | `trim_frames` | No | Frames dropped from the head of every video after the first (default: 0) | | `crossfade_ms` | No | Equal-power crossfade at each audio seam, drawn from the trimmed material - no effect when `trim_frames` is 0, and validation warns when one is written there (default: 75) | | `audio_bleed_ms` | No | How long the outgoing video's tail rings on over the head of the next one, at seams with nothing trimmed to crossfade (default: 0, off) | @@ -358,7 +358,7 @@ montage cut to a score wants: | Argument | Required | Description | | -------- | -------- | ----------- | -| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, one entry per video as with `concat_videos`. Every video must be the same frame size - unlike a sample-rate mismatch, there is no reconciliation for a size mismatch, so a statically-resolvable (`asset:`/`output:`/literal path) size disagreement is refused at validate; one only known at run time still fails there (#504) | +| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, one entry per video as with `concat_videos`; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, there is no reconciliation for a size mismatch, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518) | | `dissolve_frames` | No | Frames of overlap at each seam, blended linearly (default: 12). 0 is a hard cut | | `fade_in_frames` | No | Frames over which the first video rises out of `fade_color` (default: 0) | | `fade_out_frames` | No | Frames over which the last video sinks into it (default: 0) | @@ -1097,6 +1097,8 @@ voiced, so a weak answer is visible as weak rather than silently accepted. | `windows` | No | Named spans to roll lines up into, `{name, start, end}`. Omitted: mirrors the fixed windows when `lines` is also omitted, otherwise none | | `window_seconds` | No | Length of the fixed windows used without `lines` (default `2.0`) | | `min_reference_seconds` | No | Least total reference length per voice; a shorter one is refused by name (default `3.0`) | +| `separate` | No | Isolate the vocal stem with htdemucs before embedding (default `true`); `false` for audio that is already a dry vocal | +| `device` | No | Where the models run | A literal `voices` is checked at validation as well as on the step: fewer than two voices, a bad name, a malformed span, or a span-list reference under @@ -1104,8 +1106,6 @@ than two voices, a bad name, a malformed span, or a span-list reference under bare path as a voice meets the same location policy as `audio`. What needs the song itself - a span past its end, a clip's voiced length - is the step's to refuse. -| `separate` | No | Isolate the vocal stem with htdemucs before embedding (default `true`); `false` for audio that is already a dry vocal | -| `device` | No | Where the models run | The result: `voices` (the names), `separated`, `duration_seconds`, `lines[]` (`start`, `end`, `text`, `scores`, `voice`, `margin`, `voiced_seconds`, diff --git a/dw/video_size_errors.py b/dw/video_size_errors.py index 3910807a..30c3fd42 100644 --- a/dw/video_size_errors.py +++ b/dw/video_size_errors.py @@ -9,13 +9,15 @@ only thing to move earlier is the refusal itself. Moved here, into `validation_errors`, for exactly the cases a size is -knowable without running anything: an `asset:`/`output:` reference, or a +knowable without running anything: an `asset:`/`output:` reference, a literal file path inside the directories the run may read -(`dw/probe_paths.py`). A `previous_result:` (or any reference `expand_for_each` -left unresolved), a `variable:`/`item:`/`gather:` reference, or a -`{"location": ...}` dict, names no size yet and is left to the existing -run-time check - silence there is correct, not a gap, since the size is not -known until the step that produces it runs. +(`dw/probe_paths.py`), or a `{"location": ...}` dict wrapping either (the +same idiom `load_audio_video` accepts, #510) - unwrapped before +`resolve_probe_path` sees it, since that resolver only takes a string. A +`previous_result:` (or any reference `expand_for_each` left unresolved) or a +`variable:`/`item:`/`gather:` reference names no size yet and is left to the +existing run-time check - silence there is correct, not a gap, since the +size is not known until the step that produces it runs. """ from .for_each import MEMBER_SEPARATOR, render_path @@ -65,6 +67,8 @@ def video_size_errors(workflow_definition, source_indices=None, base_dir=None): sizes = {} for video_index, video in enumerate(videos): + if isinstance(video, dict): + video = video.get("location") path = resolve_probe_path(video, base_dir, "a video argument") if path is None: continue diff --git a/tests/test_video_size_errors.py b/tests/test_video_size_errors.py index 5381e6f7..5a35a271 100644 --- a/tests/test_video_size_errors.py +++ b/tests/test_video_size_errors.py @@ -117,6 +117,32 @@ def test_a_single_video_needs_no_size_check(self, monkeypatch): assert video_size_errors(definition, base_dir=base_dir) == [] + def test_a_location_dict_wrapping_a_mismatched_asset_is_refused(self, monkeypatch): + base_dir = workflow_dir_with_asset(monkeypatch, ("a.mp4", 12), ("b.mp4", 12)) + write_mp4( + os.path.join(base_dir, "assets", "b.mp4"), frames=12, width=64, height=32 + ) + definition = join_workflow( + "concat_videos", ["asset:a.mp4", {"location": "asset:b.mp4"}] + ) + + problems = video_size_errors(definition, base_dir=base_dir) + + assert len(problems) == 1 + assert "32x16" in problems[0]["message"] + assert "64x32" in problems[0]["message"] + + def test_a_location_dict_wrapping_an_unresolvable_entry_is_left_to_the_run( + self, monkeypatch + ): + base_dir = workflow_dir_with_asset(monkeypatch, ("a.mp4", 12)) + definition = join_workflow( + "dissolve_videos", + ["asset:a.mp4", {"location": "previous_result:make_b"}], + ) + + assert video_size_errors(definition, base_dir=base_dir) == [] + class TestTheValidationPass: def test_wired_into_validation_errors(self, monkeypatch): From 369132a25715a0507ea78c9e5b41f67880c4ec7a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 17:59:01 -0500 Subject: [PATCH 007/251] feat(templates): #497 - say the limit ceiling holds on the mix, not the AAC-muxed film Co-Authored-By: Claude Opus 5.5 --- tests/test_assemble_and_score_target_lufs.py | 9 +++++++++ workflows/templates/assemble-and-score.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_assemble_and_score_target_lufs.py b/tests/test_assemble_and_score_target_lufs.py index 060a6735..eff23e7e 100644 --- a/tests/test_assemble_and_score_target_lufs.py +++ b/tests/test_assemble_and_score_target_lufs.py @@ -115,3 +115,12 @@ def balanced(variables): true_peak = numpy.abs(scipy.signal.resample_poly(limited, 4, 1, axis=1)).max() assert 20 * numpy.log10(true_peak) <= -3.0 + 0.05 assert limited.shape == wave.shape + + +def test_the_description_places_the_limit_ceiling_on_the_mix_not_the_film(): + # #497 v2 (#474 Q6): the limiter holds -3 dBTP on the mix; the AAC mux + # of the film measured -2.54 dBTP, so the description must say where the + # ceiling holds rather than promise it on the film. + description = load_definition()["description"] + assert "holds on the mix 'balanced' writes, not on the film" in description + assert "about 1 dB above it" in description diff --git a/workflows/templates/assemble-and-score.json b/workflows/templates/assemble-and-score.json index b550a86c..fb232f1a 100644 --- a/workflows/templates/assemble-and-score.json +++ b/workflows/templates/assemble-and-score.json @@ -1,6 +1,6 @@ { "id": "assemble-and-score", - "description": "Cuts existing shots into one scored film. It is the pass you re-run while cutting, when re-generating the footage each time would cost GPU hours to change a fade. The shots go straight into concat_videos as hard cuts - nothing is stabilized or rescaled on the way, so a deliberate camera move survives the edit; they must already share one size and frame rate, and the task refuses a set that does not. 'shots' is a list, so a diptych is two entries and a reel is however many the cut needs. The shots' own recorded sound is carried up to the score's sample rate and mixed underneath it, so the room tone of each world survives the edit instead of being replaced by music. Supply the shots and the score as asset: references - 'shot_1.mp4' and friends, uploaded with upload_asset or promoted from a generated run with keep_output. Shots generated independently also drift in loudness - 10 dB between two shots of one scene is ordinary - and no seam control can hide a level jump, because it is either side of the cut rather than at it; 'match_levels' ('rms' for perceived level, 'peak' for the loudest sample) evens the shots out before they are joined, and left null, as it is by default, a wide spread is warned about in the log rather than passing in silence. 'match_levels_dbfs' sets the level match_levels moves every shot to, below its own default; a shot whose gain to that level would clip is held short of it instead, and the run says so rather than leaving the shots still apart after a match. 'audio_bleed_gain_db' ducks 'audio_bleed_ms'' full-scale bleed, left at 0 for no change. 'total_frames' is the length of the cut in frames and 'score' is a separate asset with its own length: the score is sliced to 'total_frames' from 'score_start_frame', and a score that does not reach that far is padded to it with digital silence, which leaves the rest of the film unscored under the shots' own sound (the run says so as a 'slice_past_end' warning, but the film itself sounds plausible). A score must therefore be at least as long as the cut; to stretch a short bed to reach, make a longer one with the 'loop_audio' task first and pass that as 'score'. The world track runs at a constant level through the score's own ending, so a score that decays into silence can sit 20+ dB under it there and be inaudible even though the mix's peak looks healthy; 'world_fade_out_ms', 0 by default, fades the world track out over its last N ms so the score's written ending is what is heard. The final mix is normalized by 'balanced', which always holds the peak at -3 dBFS; 'target_lufs', null by default, additionally gains the mix toward that integrated loudness (BS.1770) before the -3 dBFS ceiling is enforced - gain down succeeds unless the mix itself already peaks above the ceiling, in which case the ceiling pulls it lower still; gaining up past what the peak allows is capped the same way, and either direction a 'target_lufs_capped' warning names which side of the target the result landed on. To match loudness across several cuts of a series, pass the same 'target_lufs' to each: pick it no louder than the quietest cut's own ceiling allows, since gaining a louder cut down to meet it usually works but can still be capped by that cut's own peak. 'limit', false by default, makes -3 dBFS a true-peak ceiling held by a look-ahead limiter instead of a cap on the gain, so 'target_lufs' is reached past a single loud transient (a laugh, a sting) - pass it only when the cut must sit louder than its own peak allows (e.g. -16 for streaming); a 'limiter_heavy' warning means the target is asking for audible squashing and should come down, and past 12 dB of reduction 'target_lufs_capped' fires with 'limited: true'.", + "description": "Cuts existing shots into one scored film. It is the pass you re-run while cutting, when re-generating the footage each time would cost GPU hours to change a fade. The shots go straight into concat_videos as hard cuts - nothing is stabilized or rescaled on the way, so a deliberate camera move survives the edit; they must already share one size and frame rate, and the task refuses a set that does not. 'shots' is a list, so a diptych is two entries and a reel is however many the cut needs. The shots' own recorded sound is carried up to the score's sample rate and mixed underneath it, so the room tone of each world survives the edit instead of being replaced by music. Supply the shots and the score as asset: references - 'shot_1.mp4' and friends, uploaded with upload_asset or promoted from a generated run with keep_output. Shots generated independently also drift in loudness - 10 dB between two shots of one scene is ordinary - and no seam control can hide a level jump, because it is either side of the cut rather than at it; 'match_levels' ('rms' for perceived level, 'peak' for the loudest sample) evens the shots out before they are joined, and left null, as it is by default, a wide spread is warned about in the log rather than passing in silence. 'match_levels_dbfs' sets the level match_levels moves every shot to, below its own default; a shot whose gain to that level would clip is held short of it instead, and the run says so rather than leaving the shots still apart after a match. 'audio_bleed_gain_db' ducks 'audio_bleed_ms'' full-scale bleed, left at 0 for no change. 'total_frames' is the length of the cut in frames and 'score' is a separate asset with its own length: the score is sliced to 'total_frames' from 'score_start_frame', and a score that does not reach that far is padded to it with digital silence, which leaves the rest of the film unscored under the shots' own sound (the run says so as a 'slice_past_end' warning, but the film itself sounds plausible). A score must therefore be at least as long as the cut; to stretch a short bed to reach, make a longer one with the 'loop_audio' task first and pass that as 'score'. The world track runs at a constant level through the score's own ending, so a score that decays into silence can sit 20+ dB under it there and be inaudible even though the mix's peak looks healthy; 'world_fade_out_ms', 0 by default, fades the world track out over its last N ms so the score's written ending is what is heard. The final mix is normalized by 'balanced', which always holds the peak at -3 dBFS; 'target_lufs', null by default, additionally gains the mix toward that integrated loudness (BS.1770) before the -3 dBFS ceiling is enforced - gain down succeeds unless the mix itself already peaks above the ceiling, in which case the ceiling pulls it lower still; gaining up past what the peak allows is capped the same way, and either direction a 'target_lufs_capped' warning names which side of the target the result landed on. To match loudness across several cuts of a series, pass the same 'target_lufs' to each: pick it no louder than the quietest cut's own ceiling allows, since gaining a louder cut down to meet it usually works but can still be capped by that cut's own peak. 'limit', false by default, makes -3 dBFS a true-peak ceiling held by a look-ahead limiter instead of a cap on the gain, so 'target_lufs' is reached past a single loud transient (a laugh, a sting). That ceiling holds on the mix 'balanced' writes, not on the film: the AAC mux into the mp4 can land up to about 1 dB above it on limited material (a laugh-track episode limited to -3.0 dBTP came out of the mux at -2.54), still under the -1 dBTP ceiling of streaming delivery. Pass it only when the cut must sit louder than its own peak allows (e.g. -16 for streaming); a 'limiter_heavy' warning means the target is asking for audible squashing and should come down, and past 12 dB of reduction 'target_lufs_capped' fires with 'limited: true'.", "cost": [ { "device": "cuda", From 6b8e87be173cebda16e7a6b6de57719f3370c9b1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 18:09:53 -0500 Subject: [PATCH 008/251] fix(docs): #525 - extend-clip's clip constraints are width/height and length, not clip_frames clip_frames only sizes the generated opening; it is unused once 'clip' is supplied. The description and the ltx-2.5 skill's 'Longer' line stated a clip_frames match requirement that nothing on that code path checks. Reworded both to the real constraints: width/height must match the clip, and the clip must be shorter than num_frames. Co-Authored-By: Claude Sonnet 5 --- plugins/dw/skills/ltx-2.5/SKILL.md | 8 ++++---- workflows/templates/ltx2/extend-clip.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/dw/skills/ltx-2.5/SKILL.md b/plugins/dw/skills/ltx-2.5/SKILL.md index 7756cea9..f582e0c1 100644 --- a/plugins/dw/skills/ltx-2.5/SKILL.md +++ b/plugins/dw/skills/ltx-2.5/SKILL.md @@ -60,10 +60,10 @@ caption spec below. removes motion blur or grain - so say which defect you think it is and let the user correct you. - **Longer**: `templates/ltx2/extend-clip` continues an opening conditioned - on all of it, not one frame; `clip` extends an existing clip (matching - `width`/`height`/`clip_frames`) instead of generating one. - `templates/ltx2/chained-segments` re-runs per segment on the previous last - frame and stitches. Neither is a Lightricks recipe; both are dw's, and a + on all of it, not one frame; `clip` extends an existing clip (`width`/ + `height` matched, shorter than `num_frames`; clip_frames unused) instead + of generating. `templates/ltx2/chained-segments` re-runs per segment on + the previous last frame and stitches. Neither is a Lightricks recipe; a single 481-frame pass reaches 20 seconds before either is needed. If none fits, compose from `list_tasks` before authoring a new workflow, and diff --git a/workflows/templates/ltx2/extend-clip.json b/workflows/templates/ltx2/extend-clip.json index c8e89be4..37c2f789 100644 --- a/workflows/templates/ltx2/extend-clip.json +++ b/workflows/templates/ltx2/extend-clip.json @@ -1,6 +1,6 @@ { "id": "LTX2Extend", - "description": "Continuing a clip rather than restarting one. By default the opening is generated, then conditioned on in full so the second pass carries its motion and framing forward instead of inferring them from a single frame. Passing 'clip' (an 'asset:' or 'output:' reference to a clip already on hand - width, height and 'clip_frames' must match it exactly) skips generating the opening and conditions on that clip instead; the 'opening' step is then unreferenced and elided, so it costs nothing. Each step loads its own copy of the model rather than sharing one, because the opening step it would otherwise share from may be elided.", + "description": "Continuing a clip rather than restarting one. By default the opening is generated, then conditioned on in full so the second pass carries its motion and framing forward instead of inferring them from a single frame. Passing 'clip' (an 'asset:' or 'output:' reference to a clip already on hand - 'width' and 'height' must match it exactly, and it must be shorter than 'num_frames') skips generating the opening and conditions on that clip instead; the 'opening' step is then unreferenced and elided, so it costs nothing. 'clip_frames' only sizes the generated opening and has no effect once 'clip' is supplied. Each step loads its own copy of the model rather than sharing one, because the opening step it would otherwise share from may be elided.", "summary": "A longer LTX-2.5 clip: an opening is generated or supplied, then continued conditioned on the whole clip.", "cost_drivers": ["num_frames", "width", "height"], "variables": { From bb762daaf0c701e1958e4b98770595a2b5202a57 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 18:27:46 -0500 Subject: [PATCH 009/251] feat(tasks): #513 - join_into_song places the song from the measured dialogue Co-Authored-By: Claude Opus 5.5 --- dw/shots.py | 18 ++ dw/task_domains.py | 7 + dw/tasks/join_into_song.py | 403 +++++++++++++++++++++++++++++++++++++ dw/tasks/task.py | 11 + dw/workflow.py | 6 +- tests/test_shots.py | 1 + 6 files changed, 443 insertions(+), 3 deletions(-) create mode 100644 dw/tasks/join_into_song.py diff --git a/dw/shots.py b/dw/shots.py index 10ee8df7..ce81642c 100644 --- a/dw/shots.py +++ b/dw/shots.py @@ -267,6 +267,24 @@ def shot_reference_names(references): return names +def shot_references(arguments): + """The list of references naming a step's joined inputs, in join order. + + concat_videos and dissolve_videos take theirs as `videos`; + join_into_song takes two lists, `dialogue` then `song_shots`, and joins + them in that order, so the two together are its inputs' references. + """ + if not isinstance(arguments, dict): + return None + videos = arguments.get("videos") + if videos is not None: + return videos + dialogue, song_shots = arguments.get("dialogue"), arguments.get("song_shots") + if isinstance(dialogue, list) and isinstance(song_shots, list): + return dialogue + song_shots + return None + + def named_shots(shots, names): """The shots with each renamed where the step named its source input. diff --git a/dw/task_domains.py b/dw/task_domains.py index e61bc7bc..a0ddfdf2 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -128,6 +128,13 @@ "fps": POSITIVE, "sample_rate": POSITIVE, }, + "join_into_song": { + "cue_seconds": NON_NEGATIVE, + "duck_delay_ms": NON_NEGATIVE, + "duck_db": NON_POSITIVE, + "duck_ramp_ms": NON_NEGATIVE, + "fps": POSITIVE, + }, "loop_frames": {"num_frames": POSITIVE}, "frame_grid": {"count": POSITIVE, "columns": POSITIVE, "tile_width": POSITIVE}, "dissolve_videos": { diff --git a/dw/tasks/join_into_song.py b/dw/tasks/join_into_song.py new file mode 100644 index 00000000..6b2ccae9 --- /dev/null +++ b/dw/tasks/join_into_song.py @@ -0,0 +1,403 @@ +"""Join a spoken scene into a song: dialogue shots, then shots sung to one track. + +A musical number breaks out of dialogue - the last spoken line plays, the +song enters under it, and the sung shots play over the *unbroken* song +rather than over the separate slices each was generated against. Putting +that together from the audio tasks needs a song offset of `dialogue length - +cue`, and a workflow cannot do arithmetic: a hand-computed literal goes stale +the moment one dialogue shot is regenerated at another length (#486). This +task measures the joined dialogue at run time and places the song from it. + +The timeline, in samples at the song's rate: + +- the dialogue shots are joined end to end, each track fitted to its own + frames, so the joined dialogue ends exactly where the first song shot's + frame 0 is - `D` +- the song is placed at `D - cue_seconds * sr`, so song time `cue_seconds` + lands on that frame, and runs to the end of the picture +- the dialogue ducks by `duck_db` from `duck_delay_ms` after the song enters, + over a linear `duck_ramp_ms` ramp +- the song shots' own audio is discarded + +No final level is chosen here: what a deliverable sits at is the workflow's +to decide, with `normalize_audio` after this step, and the headroom warnings +at save apply as they do to any track. +""" + +import logging + +import numpy + +from ..events import emit_log, emit_warning +from ..loudness import MIN_LUFS_SECONDS, integrated_lufs +from ..result import AudioVideo +from ..shots import measured_num_samples, shot_record +from ..task_domains import check_arguments +from .audio_utils import ( + as_channels_samples, + frames_to_samples, + load_audio, + resample_waveform, + slice_samples, +) +from .concat_videos import video_names +from .video_utils import ( + check_same_frame_size, + frames_as_pil_list, + is_video_location, + load_audio_video, +) + +logger = logging.getLogger("dw") + +COMMAND = "join_into_song" + + +def join_into_song( + dialogue, + song_shots, + song, + cue_seconds=0, + dialogue_target_lufs=None, + duck_delay_ms=0, + duck_db=-12, + duck_ramp_ms=250, + fps=None, +): + """Join dialogue shots and song shots into one video over the unbroken song. + + Args: + dialogue: The spoken shots, in order - each keeps its own audio. The + same entries concat_videos' `videos` takes: previous results, + paths of files an earlier run wrote, or {"location": ...} dicts. + A shot with no track is filled with silence for its length + song_shots: The sung shots, in order. Their audio is discarded - they + play over `song` + song: The unbroken track the song shots were sliced from - an audio + result, a video or audio file's path, or anything carrying + '.audio' and '.sample_rate'. Its rate is the output's + cue_seconds: The song time that lands on the first song shot's + frame 0 - the start of the slice that shot was generated against. + 0 starts the song exactly at the cut; above 0 the song enters + that long before it, under the last spoken line. Longer than the + dialogue is refused: the song would have to start before the film + dialogue_target_lufs: Integrated loudness (BS.1770) each dialogue + shot is gained to, with one static gain per shot. Omitted, the + shots keep their own levels. A shot too short (under 400 ms) or + too quiet to measure is left as it is, with a warning + duck_delay_ms: How long after the song enters the dialogue starts to + duck. With it at or past the dialogue's end, nothing ducks + duck_db: How far the dialogue ducks, in dB - 0 or below + duck_ramp_ms: The length of the linear ramp into the duck + fps: The rate the videos play at, when they do not carry one - a + pipeline's frames carry none, a file brings its own. The joined + file is written at it unless the step's result.fps overrides it + + Returns: + One AudioVideo: the dialogue's frames then the song shots', over the + mix, with one shot record per input + """ + check_arguments( + COMMAND, + cue_seconds=cue_seconds, + duck_delay_ms=duck_delay_ms, + duck_db=duck_db, + duck_ramp_ms=duck_ramp_ms, + fps=fps, + ) + # A number handed in through an untyped variable arrives as a string + cue_seconds = float(cue_seconds) + duck_delay_ms, duck_db = float(duck_delay_ms), float(duck_db) + duck_ramp_ms = float(duck_ramp_ms) + if dialogue_target_lufs is not None: + dialogue_target_lufs = float(dialogue_target_lufs) + if fps is not None: + fps = float(fps) + for name, value in (("dialogue", dialogue), ("song_shots", song_shots)): + if not isinstance(value, list) or not value: + raise ValueError( + f"{COMMAND} needs a non-empty list of videos for '{name}' - a " + "join into a song has spoken shots before it and sung shots " + "after it" + ) + + names = video_names(dialogue + song_shots) + dialogue = [_loaded(video) for video in dialogue] + song_shots = [_loaded(video) for video in song_shots] + videos = dialogue + song_shots + clips = [frames_as_pil_list(video) for video in videos] + check_same_frame_size(clips, COMMAND) + fps = _one_fps(videos, names, fps) + + song_waveform, sample_rate = _song_track(song) + channels = song_waveform.shape[0] + + # The dialogue's track, shot by shot. Each shot's boundary is placed on + # the frame grid from the running frame count, so rounding never + # accumulates and D is the sample the first song frame sits at + frames = [] + shots = [] + pieces = [] + for index, (video, clip) in enumerate(zip(dialogue, clips)): + start_frame = len(frames) + frames.extend(clip) + start = frames_to_samples(start_frame, fps, sample_rate) + end = frames_to_samples(len(frames), fps, sample_rate) + piece = _dialogue_track( + video, names[index], end - start, fps, sample_rate, channels + ) + if dialogue_target_lufs is not None: + piece = _match_loudness( + piece, sample_rate, dialogue_target_lufs, names[index] + ) + pieces.append(piece) + shots.append( + _shot(names[index], index, start_frame, len(frames) - start_frame, start) + ) + dialogue_track = numpy.concatenate(pieces, axis=1) + dialogue_samples = dialogue_track.shape[1] + + for offset, clip in enumerate(clips[len(dialogue) :]): + index = len(dialogue) + offset + start_frame = len(frames) + frames.extend(clip) + shots.append( + _shot( + names[index], + index, + start_frame, + len(frames) - start_frame, + frames_to_samples(start_frame, fps, sample_rate), + ) + ) + total_samples = frames_to_samples(len(frames), fps, sample_rate) + + cue_samples = int(round(float(cue_seconds) * sample_rate)) + if cue_samples > dialogue_samples: + raise ValueError( + f"{COMMAND}: cue_seconds={cue_seconds:g} is longer than the " + f"{dialogue_samples / sample_rate:.2f} s of dialogue, so the song " + "would have to start before the film does. Pass a cue_seconds no " + "longer than the dialogue - it is the song time the first song " + "shot's frame 0 lands on, the start of the slice it was " + "generated against" + ) + song_entry = dialogue_samples - cue_samples + + mix = numpy.zeros((channels, total_samples), dtype=numpy.float32) + mix[:, :dialogue_samples] += _ducked( + dialogue_track, song_entry, sample_rate, duck_delay_ms, duck_db, duck_ramp_ms + ) + mix[:, song_entry:] += _placed_song( + song_waveform, total_samples - song_entry, sample_rate + ) + + measured_num_samples(shots, mix.shape[1]) + emit_log( + f"{COMMAND}: song enters at {song_entry / sample_rate:.3f} s, " + f"{cue_seconds:g} s before the first song shot at " + f"{dialogue_samples / sample_rate:.3f} s", + command=COMMAND, + song_entry_sample=song_entry, + dialogue_samples=dialogue_samples, + sample_rate=sample_rate, + ) + logger.debug( + f"Joined {len(dialogue)} dialogue and {len(song_shots)} song shots " + f"into {len(frames)} frames" + ) + return AudioVideo(frames, mix, sample_rate, fps=fps, shots=shots) + + +def _loaded(video): + """A video entry as something with frames, loading a path with its audio.""" + return load_audio_video(video) if is_video_location(video) else video + + +def _one_fps(videos, names, fps): + """The rate the join plays at: the one given, else the one the inputs agree on. + + Frames are joined one for one, so a song shot at another rate would play + at the wrong speed against the song - refused, not resampled. + """ + carried = [ + (name, getattr(video, "fps", None)) for name, video in zip(names, videos) + ] + rates = {float(rate) for _name, rate in carried if rate} + if len(rates) > 1: + raise ValueError( + f"{COMMAND} needs every video at one frame rate, got " + + ", ".join( + f"{name}: {float(rate):g} fps" for name, rate in carried if rate + ) + + ". Frames are joined one for one, so a shot at another rate would " + "play at the wrong speed against the song" + ) + if fps is None: + fps = next((rate for _name, rate in carried if rate), None) + if fps is None: + raise ValueError( + f"{COMMAND} needs 'fps' - none of the videos carries a frame rate " + "of its own, and the song is placed from the dialogue's length in " + "time" + ) + return fps + + +def _song_track(song): + """The song as a (channels, samples) waveform and its rate.""" + if is_video_location(song): + location = song["location"] if isinstance(song, dict) else song + waveform, rate = load_audio(location) + else: + waveform = getattr(song, "audio", song) + rate = getattr(song, "sample_rate", None) + if waveform is None: + raise ValueError( + f"{COMMAND} needs a song track - the one given carries no audio" + ) + waveform = as_channels_samples(waveform) + if not rate: + raise ValueError( + f"{COMMAND} needs the song's sample rate - the track it was given " + "does not carry one" + ) + return waveform, int(rate) + + +def _dialogue_track(video, name, length, fps, sample_rate, channels): + """One dialogue shot's track at the song's rate, fitted to its own frames. + + Fitted rather than measured as-is: the song is placed from where the + dialogue ends, so a track running past its frames would push the song + off the first song shot's picture by the overrun. A shot with no track + is silence for its length - skipping it would land every later shot's + audio early. + """ + audio = getattr(video, "audio", None) + if audio is None: + emit_log( + f"{COMMAND}: {name} carries no audio - filled with silence", + command=COMMAND, + shot=name, + ) + return numpy.zeros((channels, length), dtype=numpy.float32) + waveform = as_channels_samples(audio) + rate = getattr(video, "sample_rate", None) or sample_rate + if rate != sample_rate: + emit_log( + f"{COMMAND}: resampling {name} from {rate} Hz to the song's " + f"{sample_rate} Hz", + command=COMMAND, + shot=name, + sample_rate=sample_rate, + ) + waveform = resample_waveform(waveform, rate, sample_rate) + waveform = _with_channels(waveform, channels) + difference = waveform.shape[1] - length + if abs(difference) >= sample_rate / float(fps): + emit_warning( + f"{COMMAND}: {name}'s track is {waveform.shape[1] / sample_rate:.3f} s " + f"and its frames are {length / sample_rate:.3f} s - " + + ("trimmed" if difference > 0 else "padded with silence") + + " to its frames, so the song lands on the first song shot's " + "frame 0.", + kind="dialogue_fitted_to_frames", + command=COMMAND, + shot=name, + difference_samples=int(difference), + ) + return slice_samples(waveform, 0, length) + + +def _with_channels(waveform, channels): + """The waveform with the song's channel count - mono spread, extra dropped.""" + have = waveform.shape[0] + if have == channels: + return waveform + if have == 1: + return numpy.repeat(waveform, channels, axis=0) + if channels == 1: + return waveform.mean(axis=0, keepdims=True) + return waveform[:channels] + + +def _match_loudness(piece, sample_rate, target_lufs, name): + """One dialogue shot gained by one static amount to target_lufs.""" + measured = integrated_lufs(piece.T, sample_rate) + if measured is None: + seconds = piece.shape[1] / float(sample_rate) + reason = ( + f"it is {seconds:.2f} s, under the {MIN_LUFS_SECONDS * 1000:.0f} ms " + "loudness measurement needs" + if seconds < MIN_LUFS_SECONDS + else "its loudness could not be measured (silent or near it)" + ) + emit_warning( + f"{COMMAND}: {name} was left at its own level rather than matched " + f"to {target_lufs:g} LUFS - {reason}.", + kind="dialogue_unmatched", + command=COMMAND, + shot=name, + target_lufs=target_lufs, + ) + return piece + gain_db = float(target_lufs) - measured + emit_log( + f"{COMMAND}: {name} {measured:.1f} LUFS, gained {gain_db:+.1f} dB", + command=COMMAND, + shot=name, + measured_lufs=round(measured, 1), + gain_db=round(gain_db, 2), + ) + return (piece * (10.0 ** (gain_db / 20.0))).astype(numpy.float32) + + +def _ducked(track, song_entry, sample_rate, delay_ms, duck_db, ramp_ms): + """The dialogue with its duck applied: unity, a linear ramp, then held.""" + length = track.shape[1] + start = song_entry + int(round(float(delay_ms) * sample_rate / 1000.0)) + if start >= length or duck_db == 0: + return track + ramp = int(round(float(ramp_ms) * sample_rate / 1000.0)) + floor = 10.0 ** (float(duck_db) / 20.0) + envelope = numpy.ones(length, dtype=numpy.float32) + if ramp: + # Linear in amplitude from unity at `start` to the floor at + # `start + ramp`, the ramp the plan names rather than a step + steps = numpy.arange(ramp, dtype=numpy.float32) / ramp + section = 1.0 + (floor - 1.0) * steps + end = min(length, start + ramp) + envelope[start:end] = section[: end - start] + envelope[end:] = floor + else: + envelope[start:] = floor + return track * envelope + + +def _placed_song(song, length, sample_rate): + """The song from its start, `length` samples of it - padded when short.""" + have = song.shape[1] + if have < length: + short = (length - have) / float(sample_rate) + emit_warning( + f"{COMMAND}: the song is {have / sample_rate:.2f} s and the picture " + f"needs {length / sample_rate:.2f} s of it from where it enters - " + f"padded with {short:.2f} s of silence, so the end of the cut has " + "no song under it. A longer song, or fewer song shots, covers it.", + kind="song_short", + command=COMMAND, + song_seconds=have / float(sample_rate), + needed_seconds=length / float(sample_rate), + ) + return slice_samples(song, 0, length) + + +def _shot(name, index, start_frame, num_frames, start_sample): + """A shot record tagged with the input it came from, for named_shots.""" + shot = shot_record(name, start_frame, num_frames, start_sample) + if index: + # Every seam here is a cut this step draws, as concat_videos marks it + shot["hard_cut"] = True + shot["source_index"] = index + return shot diff --git a/dw/tasks/task.py b/dw/tasks/task.py index 4ab15892..0f6a075e 100644 --- a/dw/tasks/task.py +++ b/dw/tasks/task.py @@ -194,6 +194,17 @@ def _handle_dissolve_videos(task, arguments, previous_pipelines): return dissolve_videos(**arguments) +@register_command( + "join_into_song", implementation="dw.tasks.join_into_song.join_into_song" +) +def _handle_join_into_song(task, arguments, previous_pipelines): + """Join dialogue shots and song shots into one video over the unbroken song""" + logger.debug("Joining dialogue into a song") + from .join_into_song import join_into_song + + return join_into_song(**arguments) + + @register_command("fade_audio", implementation="dw.tasks.audio_utils.fade_audio") def _handle_fade_audio(task, arguments, previous_pipelines): """Fade an audio track in from silence and out to it""" diff --git a/dw/workflow.py b/dw/workflow.py index eda455d6..bb5e9e51 100644 --- a/dw/workflow.py +++ b/dw/workflow.py @@ -50,7 +50,7 @@ resolve_constraint_references, ) from .result_fps import fps_errors -from .shots import duplicate_shot_names, step_shots +from .shots import duplicate_shot_names, shot_references, step_shots from .subfolders import step_subfolder, subfolder_errors from .reference_names import reference_name_errors from .video_extensions import video_extension_errors @@ -1640,11 +1640,11 @@ def run( if selected is not None: manifest_entry["selected"] = selected # Where each joined shot sits in the file, named by the - # step's own `videos` references (dw/shots.py) + # step's own input references (dw/shots.py) shots = step_shots( getattr(result, "saved_shots", None), saved_files, - step_data.get("task", {}).get("arguments", {}).get("videos"), + shot_references(step_data.get("task", {}).get("arguments", {})), ) if shots: manifest_entry["shots"] = shots diff --git a/tests/test_shots.py b/tests/test_shots.py index 07557e61..aa3a2b61 100644 --- a/tests/test_shots.py +++ b/tests/test_shots.py @@ -71,6 +71,7 @@ def audio_video(num_frames, level, fps=4, sample_rate=100): EXPECTED_SITES = { ("dw/tasks/concat_videos.py", "concat_videos"): ("populates", 1), ("dw/tasks/dissolve_videos.py", "dissolve_videos"): ("populates", 1), + ("dw/tasks/join_into_song.py", "join_into_song"): ("populates", 1), ("dw/pipeline_processors/chain.py", "run_chain"): ("populates", 2), ("dw/tasks/task.py", "_per_frame"): ("carries", 1), ("dw/tasks/stabilize.py", "stabilize_video"): ("carries", 1), From a90e38789e1b029a90993b1bca2ae12cbd96dae6 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 18:40:57 -0500 Subject: [PATCH 010/251] fix(mcp,server,ui): #527 - drop absolute server paths from save_workflow, save_prompt, upload_asset, keep_output save_workflow and save_prompt echoed the server's absolute filesystem path in their response (following #521's pattern for upload_asset/keep_output, which had the same bug). All four now report name/workspace/origin/reference only, matching #521's model: a caller can address the thing it just wrote, but never sees where it lives on disk. Also fixes a stale docstring in dw_mcp/assets.py's upload hint text ("201 with 'path'" -> "201 with 'reference'") and sweeps test mocks across test_mcp_assets.py, test_mcp_authoring.py and test_mcp_prompts.py that still simulated the old path-bearing response shape. Comment 3's validate_workflow bad-model_name leak (repro: upscale_h3_latents, model_name="/etc/passwd") was already fixed by a6f816e (#499), deliberately preserved through 777f2eb (#526); no further change needed for it here. Co-Authored-By: Claude Sonnet 5 --- dw/server/app.py | 11 ++++---- dw_mcp/assets.py | 14 +++++------ tests/test_library_sources.py | 5 +++- tests/test_mcp_assets.py | 28 ++++++++++----------- tests/test_mcp_authoring.py | 4 +-- tests/test_mcp_prompts.py | 2 +- tests/test_server.py | 15 +++++------ tests/test_server_workspaces.py | 13 ++++++---- ui/src/lib/api.ts | 6 ++--- ui/src/lib/editor/MediaArgumentInput.svelte | 2 +- ui/src/lib/pages/AssetsPage.svelte | 2 +- ui/src/lib/pages/EditorPage.svelte | 4 +-- ui/src/lib/pages/PromptEditorPage.svelte | 4 +-- 13 files changed, 56 insertions(+), 54 deletions(-) diff --git a/dw/server/app.py b/dw/server/app.py index 71cd4454..839561e9 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -2251,7 +2251,7 @@ def save_workflow( status_code=400, detail='Provide the definition as {"workflow": {...}}', ) - path, _source = resolve_writable_workflow(_sources_for(ws), name) + path, source = resolve_writable_workflow(_sources_for(ws), name) candidate = Workflow( copy.deepcopy(request.workflow), ws.outputs, @@ -2283,7 +2283,7 @@ def save_workflow( return { "name": name, "workspace": ws.name, - "path": path, + "origin": source.origin, "warnings": warnings, "shape": metadata["shape"], "traits": metadata["traits"], @@ -2560,7 +2560,7 @@ def save_prompt(name: str, request: PromptRequest): json.dump(request.prompt, file, indent=2) file.write("\n") logger.info(f"Saved prompt {name} to {path}") - return {"name": name, "path": path} + return {"name": name} @app.delete("/api/prompts/{name:path}") def delete_prompt(name: str): @@ -3919,7 +3919,7 @@ async def upload_media( if shared or ws.assets: path = f"/inputs/{UPLOADS_SUBDIR}/{quote(name)}" result = { - "path": f"asset:{UPLOADS_SUBDIR}/{name}", + "reference": f"asset:{UPLOADS_SUBDIR}/{name}", "workspace": ws.name, "url": _served_url(path, ws), "shared": shared, @@ -3930,7 +3930,6 @@ async def upload_media( return result path = f"/outputs/{UPLOADS_SUBDIR}/{quote(name)}" result = { - "path": dest, "workspace": ws.name, "url": _served_url(path, ws), } @@ -4155,7 +4154,7 @@ def keep_output_as_asset( return { "reference": f"asset:{asset_name}", "name": asset_name, - "path": destination, + "workspace": ws.name, "linked": linked, "shared": bool(request.shared), } diff --git a/dw_mcp/assets.py b/dw_mcp/assets.py index 73d8dfe4..f34cfa0f 100644 --- a/dw_mcp/assets.py +++ b/dw_mcp/assets.py @@ -154,8 +154,8 @@ def _upload_route_hint(client, workspace=None): "127.0.0.1, which on your machine is your machine. " ) return text + ( - "It answers 201 with 'path', the 'asset:' reference to use in a " - "workflow argument; the guide's References section " + "It answers 201 with 'reference', the 'asset:' reference to use in " + "a workflow argument; the guide's References section " '(get_guide("workflows", section="Authoring a workflow from an ' 'agent")) has the same command.' ) @@ -382,11 +382,11 @@ def upload_asset( if shared: params["shared"] = "true" result = client.post_bytes("/api/uploads", body, params=params, workspace=workspace) - # 'path' from a server with no asset library is an absolute path on that - # machine; from one with a library it is already the reference. Report - # whichever it gave, named for what it is + # A server with no asset library gives no reference at all - there is + # nothing an `asset:` argument could name - only a URL to fetch the file + # back from. One with a library gives the reference straight. return { - "reference": result.get("path"), + "reference": result.get("reference"), "workspace": result.get("workspace"), "url": result.get("url"), "uploaded": os.path.basename(path), @@ -430,7 +430,7 @@ def _upload_inline(client, content, asset_name=None, shared=False, workspace=Non params["shared"] = "true" result = client.post_bytes("/api/uploads", body, params=params, workspace=workspace) return { - "reference": result.get("path"), + "reference": result.get("reference"), "workspace": result.get("workspace"), "url": result.get("url"), "uploaded": asset_name, diff --git a/tests/test_library_sources.py b/tests/test_library_sources.py index b73f969b..12bbf59e 100644 --- a/tests/test_library_sources.py +++ b/tests/test_library_sources.py @@ -197,7 +197,10 @@ def test_saving_an_example_prompt_writes_a_copy(self, client): json={"prompt": {"text": "changed"}}, ) assert saved.status_code == 200 - assert saved.json()["path"].startswith(os.path.abspath(workspace.prompts)) + assert saved.json()["name"] == "flux/daffodil" + assert os.path.isfile( + os.path.join(workspace.prompts, "flux", "daffodil.json") + ) with open(checkout / "prompts" / "flux" / "daffodil.json") as file: assert json.load(file)["text"] == "a biomechanical daffodil" assert api.get("/api/prompts/flux/daffodil").json()["text"] == "changed" diff --git a/tests/test_mcp_assets.py b/tests/test_mcp_assets.py index 8b8483c2..97161ec6 100644 --- a/tests/test_mcp_assets.py +++ b/tests/test_mcp_assets.py @@ -36,7 +36,7 @@ def handler(request): return httpx.Response( 201, json={ - "path": "asset:uploads/deadbeef.png", + "reference": "asset:uploads/deadbeef.png", "url": "/inputs/uploads/deadbeef.png", }, ) @@ -156,20 +156,20 @@ def test_an_oversized_file_fails_before_it_is_read(self, tmp_path, monkeypatch): upload_asset(client, str(source)) assert "body" not in seen - def test_a_server_with_no_library_still_reports_what_it_got(self, tmp_path): - """Older servers answer with an absolute path rather than a - reference - report whatever came back rather than inventing one.""" + def test_a_server_with_no_library_still_reports_the_url(self, tmp_path): + """A server with no asset library configured gives no reference at + all - there is nothing an `asset:` argument could name - only a URL + to fetch the file back from. Report that rather than inventing a + reference out of it (#527).""" source = tmp_path / "iris.png" source.write_bytes(b"png") def handler(request): - return httpx.Response( - 201, - json={"path": "/srv/outputs/uploads/x.png", "url": "/outputs/x.png"}, - ) + return httpx.Response(201, json={"url": "/outputs/x.png"}) result = upload_asset(client_over(handler), str(source)) - assert result["reference"] == "/srv/outputs/uploads/x.png" + assert result["reference"] is None + assert result["url"] == "/outputs/x.png" def test_a_workspace_can_be_named_for_one_request(self, tmp_path): """#463: an upload pinned to another workspace must not depend on @@ -183,7 +183,7 @@ def handler(request): return httpx.Response( 201, json={ - "path": "asset:uploads/deadbeef.png", + "reference": "asset:uploads/deadbeef.png", "workspace": "A", "url": "/inputs/uploads/deadbeef.png", }, @@ -389,7 +389,7 @@ def handler(request): return httpx.Response( 201, json={ - "path": "asset:uploads/deadbeef.png", + "reference": "asset:uploads/deadbeef.png", "url": "/inputs/uploads/deadbeef.png", }, ) @@ -500,7 +500,7 @@ def handler(request): return httpx.Response( 201, json={ - "path": "asset:uploads/deadbeef.png", + "reference": "asset:uploads/deadbeef.png", "url": "/inputs/uploads/deadbeef.png", }, ) @@ -620,7 +620,7 @@ def test_the_command_carries_token_workspace_and_return(self, tmp_path): assert "Authorization: Bearer $DW_API_TOKEN" in text assert "--data-binary @" in text assert '&asset_name=&workspace=qa-series"' in text - assert "201 with 'path', the 'asset:' reference" in text + assert "201 with 'reference', the 'asset:' reference" in text assert f"up to {MAX_UPLOAD_BYTES // (1024 * 1024)}MB" in text assert "no size cap" not in text @@ -645,5 +645,5 @@ def test_the_inline_limit_refusal_carries_the_same_command(self): text = str(refused.value) assert '"http://:8765/api/uploads?filename=' in text assert '&workspace=qa-series"' in text - assert "201 with 'path', the 'asset:' reference" in text + assert "201 with 'reference', the 'asset:' reference" in text assert "no size cap" not in text diff --git a/tests/test_mcp_authoring.py b/tests/test_mcp_authoring.py index 207540af..0799e0b7 100644 --- a/tests/test_mcp_authoring.py +++ b/tests/test_mcp_authoring.py @@ -130,7 +130,7 @@ def handler(request): body_seen["path"] = request.url.path body_seen["body"] = request.read() return httpx.Response( - 200, json={"name": "mine", "path": "/w/mine.json", "warnings": []} + 200, json={"name": "mine", "workspace": "default", "warnings": []} ) client = DwClient(transport=httpx.MockTransport(handler)) @@ -227,7 +227,7 @@ def test_save_can_name_a_workspace_for_one_request(): { ("PUT", "/api/workflows/mine"): ( 200, - {"name": "mine", "workspace": "A", "path": "/w/mine.json"}, + {"name": "mine", "workspace": "A"}, ) } ) diff --git a/tests/test_mcp_prompts.py b/tests/test_mcp_prompts.py index 88e64ea1..9425d5fe 100644 --- a/tests/test_mcp_prompts.py +++ b/tests/test_mcp_prompts.py @@ -98,7 +98,7 @@ def handler(request): def test_save_prompt_puts_the_definition_under_its_name(): client, seen = recording( - httpx.Response(200, json={"name": "duke", "path": "/p/duke.json"}) + httpx.Response(200, json={"name": "duke", "workspace": "default"}) ) result = prompts.save_prompt(client, "duke", PROMPT) diff --git a/tests/test_server.py b/tests/test_server.py index 48f4e0ca..a5d80e34 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -2586,7 +2586,7 @@ def test_gallery_delete_accepts_an_output_reference(server, tmp_path): assert not (outputs / "victim.png").exists() -def test_upload_media_saves_file_and_returns_absolute_path(server, tmp_path): +def test_upload_media_saves_file_and_returns_no_server_path(server, tmp_path): with server(success_script) as client: response = client.post( "/api/uploads", @@ -2601,8 +2601,7 @@ def test_upload_media_saves_file_and_returns_absolute_path(server, tmp_path): files = list(saved.iterdir()) assert len(files) == 1 assert files[0].read_bytes() == b"not-really-png-bytes" - assert body["path"] == str(files[0]) - assert os.path.isabs(body["path"]) + assert "path" not in body assert body["url"] == f"/outputs/uploads/{files[0].name}" # served back through the same static mount the gallery uses @@ -2670,7 +2669,7 @@ def test_upload_media_lands_in_the_asset_library(asset_server, tmp_path): files = list(saved.iterdir()) assert len(files) == 1 assert files[0].read_bytes() == b"not-really-png-bytes" - assert body["path"] == f"asset:uploads/{files[0].name}" + assert body["reference"] == f"asset:uploads/{files[0].name}" assert body["url"] == f"/inputs/uploads/{files[0].name}" assert not (tmp_path / "outputs" / "uploads").exists() @@ -2691,7 +2690,7 @@ def test_an_upload_can_be_given_a_readable_name(asset_server, tmp_path): content=b"not-really-wav-bytes", ) assert response.status_code == 201 - assert response.json()["path"] == "asset:uploads/cast/priya-voice.wav" + assert response.json()["reference"] == "asset:uploads/cast/priya-voice.wav" stored = tmp_path / "assets" / "uploads" / "cast" / "priya-voice.wav" assert stored.read_bytes() == b"not-really-wav-bytes" @@ -2701,7 +2700,7 @@ def test_an_upload_can_be_given_a_readable_name(asset_server, tmp_path): params={"filename": "clip-02.wav", "asset_name": "cast/hal-voice.wav"}, content=b"more-bytes", ) - assert named.json()["path"] == "asset:uploads/cast/hal-voice.wav" + assert named.json()["reference"] == "asset:uploads/cast/hal-voice.wav" mismatched = client.post( "/api/uploads", @@ -3068,9 +3067,7 @@ def test_saving_an_example_copies_it_into_the_writable_library( response = client.put("/api/workflows/ltx2/Gyre", json={"workflow": edited}) assert response.status_code == 200 - assert response.json()["path"] == str( - tmp_path / "workflows" / "ltx2" / "Gyre.json" - ) + assert response.json()["origin"] == "workspace" # the example on disk did not move or change assert ( json.loads((tmp_path / "examples" / "ltx2" / "Gyre.json").read_text()) diff --git a/tests/test_server_workspaces.py b/tests/test_server_workspaces.py index 6c7076c3..7ee80763 100644 --- a/tests/test_server_workspaces.py +++ b/tests/test_server_workspaces.py @@ -174,7 +174,7 @@ def test_assets_and_uploads_are_per_workspace(self, server, workspace_root): content=b"png", ) assert uploaded.status_code == 201 - assert uploaded.json()["path"].startswith("asset:uploads/") + assert uploaded.json()["reference"].startswith("asset:uploads/") assert client.get("/api/assets").json()["assets"] == [] scoped = client.get("/api/assets?workspace=shots").json() @@ -450,7 +450,8 @@ def test_it_links_rather_than_copying_when_it_can(self, server, workspace_root): # outputs and assets share one temporary filesystem, so a link is # always possible here assert body["linked"] is True - assert os.stat(source).st_ino == os.stat(body["path"]).st_ino + kept_path = os.path.join(workspace_root.assets, "clip.mp4") + assert os.stat(source).st_ino == os.stat(kept_path).st_ino def test_it_copies_when_it_cannot_link(self, server, workspace_root, monkeypatch): """A different filesystem, or one with no links, still keeps the @@ -466,8 +467,10 @@ def no_links(*args, **kwargs): "/api/assets/keep", json={"name": "Gyre/run/clip.mp4"} ).json() assert body["linked"] is False - assert os.stat(source).st_ino != os.stat(body["path"]).st_ino - with open(body["path"], "rb") as kept: + assert body["reference"] == "asset:clip.mp4" + kept_path = os.path.join(workspace_root.assets, "clip.mp4") + assert os.stat(source).st_ino != os.stat(kept_path).st_ino + with open(kept_path, "rb") as kept: assert kept.read() == b"clip" def test_the_name_defaults_to_the_files_own(self, server, workspace_root): @@ -991,7 +994,7 @@ def test_a_shared_upload_is_visible_from_every_workspace( ) assert uploaded.status_code == 201 body = uploaded.json() - assert body["path"] == "asset:uploads/cast/priya.png" + assert body["reference"] == "asset:uploads/cast/priya.png" assert body["shared"] is True for workspace in ("", "?workspace=episode-four"): diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index 7a567808..34056720 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -469,7 +469,7 @@ export const api = { * than the random one a browser upload gets, and `shared` puts it in the * library every workspace under this root shares. */ uploadMedia: (file: File, assetName?: string, shared = false) => - request<{ path: string; url: string; reference?: string }>( + request<{ url: string; reference?: string }>( `/api/uploads?filename=${encodeURIComponent(file.name)}` + (assetName ? `&asset_name=${encodeURIComponent(assetName)}` : '') + (shared ? '&shared=true' : ''), @@ -565,7 +565,7 @@ export const api = { workflowDownloadUrl: (name: string) => withToken(`/api/workflows/${encodePath(name)}/download`), saveWorkflow: (name: string, workflow: WorkflowDefinition) => - request<{ name: string; path: string; warnings: string[] }>( + request<{ name: string; workspace: string; origin: string; warnings: string[] }>( `/api/workflows/${encodePath(name)}`, { method: 'PUT', @@ -596,7 +596,7 @@ export const api = { }), ), savePrompt: (name: string, prompt: PromptDefinition) => - request<{ name: string; path: string }>( + request<{ name: string }>( `/api/prompts/${encodePath(name)}`, { method: 'PUT', diff --git a/ui/src/lib/editor/MediaArgumentInput.svelte b/ui/src/lib/editor/MediaArgumentInput.svelte index b5e95ab3..ffeabf14 100644 --- a/ui/src/lib/editor/MediaArgumentInput.svelte +++ b/ui/src/lib/editor/MediaArgumentInput.svelte @@ -56,7 +56,7 @@ uploading = true try { const result = await api.uploadMedia(file) - onchange(result.path) + onchange(result.reference ?? result.url) setLocalPreview(result.url) } catch (err) { error = err instanceof Error ? err.message : 'upload failed' diff --git a/ui/src/lib/pages/AssetsPage.svelte b/ui/src/lib/pages/AssetsPage.svelte index ebd236a6..829c99b5 100644 --- a/ui/src/lib/pages/AssetsPage.svelte +++ b/ui/src/lib/pages/AssetsPage.svelte @@ -221,7 +221,7 @@ assetName || undefined, uploadTarget === 'shared', ) - notify.success(`Uploaded as ${result.reference ?? result.path}`) + notify.success(`Uploaded as ${result.reference ?? result.url}`) await load() } catch (e) { notify.error(e instanceof Error ? e.message : String(e)) diff --git a/ui/src/lib/pages/EditorPage.svelte b/ui/src/lib/pages/EditorPage.svelte index 6cb8bfbd..203365f5 100644 --- a/ui/src/lib/pages/EditorPage.svelte +++ b/ui/src/lib/pages/EditorPage.svelte @@ -341,7 +341,7 @@ if (!(await validate())) return busy = true try { - const result = await api.saveWorkflow( + await api.saveWorkflow( path, $state.snapshot(workflow) as WorkflowDefinition, ) @@ -356,7 +356,7 @@ workflowFiles = [...workflowFiles, `${path}.json`] } baseline = JSON.stringify($state.snapshot(workflow)) - notify.success(`Saved to ${result.path}`) + notify.success(`Saved to ${path}`) } catch (e) { notify.error(e instanceof Error ? e.message : String(e)) } finally { diff --git a/ui/src/lib/pages/PromptEditorPage.svelte b/ui/src/lib/pages/PromptEditorPage.svelte index 8c98a5bc..46673f67 100644 --- a/ui/src/lib/pages/PromptEditorPage.svelte +++ b/ui/src/lib/pages/PromptEditorPage.svelte @@ -390,7 +390,7 @@ const path = savePath()! busy = true try { - const result = await api.savePrompt( + await api.savePrompt( path, $state.snapshot(doc) as PromptDefinition, ) @@ -402,7 +402,7 @@ // listing, so a newly created one must be added or the select resets if (!promptFiles.includes(path)) promptFiles = [...promptFiles, path] baseline = JSON.stringify($state.snapshot(doc)) - notify.success(`Saved to ${result.path}`) + notify.success(`Saved to ${path}`) loadPromptLibrary() } catch (e) { notify.error(e instanceof Error ? e.message : String(e)) From 07122af266cb4a492b06e0c19539641f5757ba09 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 18:43:16 -0500 Subject: [PATCH 011/251] feat(tasks): #513 - join_into_song unit tests and TASKS.md section Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 65 +++++ tests/test_join_into_song.py | 461 +++++++++++++++++++++++++++++++++++ 2 files changed, 526 insertions(+) create mode 100644 tests/test_join_into_song.py diff --git a/docs/TASKS.md b/docs/TASKS.md index 47cbe467..9d51c494 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -375,6 +375,71 @@ when any input is silent the result is, and `pair_audio` puts a score under it. **Example:** [dissolve-between-shots.json](../workflows/templates/dissolve-between-shots.json) +### join_into_song + +Join a spoken scene into a musical number: dialogue shots keep their own +audio, and the shots sung after them play over the *unbroken* song rather +than the separate slices each was generated against. The song's entry point +is `dialogue length - cue_seconds`, and a workflow cannot do that arithmetic +itself - a hand-computed literal goes stale the moment one dialogue shot is +regenerated at another length. This task measures the joined dialogue at run +time and places the song from it, so the offset never goes stale: + +```json +{ + "task": { + "command": "join_into_song", + "arguments": { + "dialogue": ["previous_result:line_a", "previous_result:line_b"], + "song_shots": "gather:sung", + "song": "asset:song.mp3", + "cue_seconds": 1.5 + } + }, + "result": { "content_type": "video/mp4" } +} +``` + +| Argument | Required | Description | +| -------- | -------- | ----------- | +| `dialogue` | Yes | The spoken shots, in order, each keeping its own audio - a non-empty list, the same entries `concat_videos`' `videos` takes: `previous_result` references, or the path or URL of a video file an earlier run wrote, each optionally wrapped in a `{"location": ...}` dict. A shot with no track is filled with silence for its length | +| `song_shots` | Yes | The sung shots, in order - a non-empty list in the same shapes as `dialogue`. Their own audio is discarded; they play over `song` | +| `song` | Yes | The unbroken track the song shots were sliced from - an audio result, a video or audio file's path, or anything carrying `.audio` and `.sample_rate`. Its rate is the output's | +| `cue_seconds` | No | The song time that lands on the first song shot's frame 0 - the start of the slice that shot was generated against. `0` (the default) starts the song exactly at the cut; above `0` the song enters that long before it, under the last spoken line. Longer than the dialogue is refused - the song would have to start before the film. Must be ≥ 0 | +| `dialogue_target_lufs` | No | Integrated loudness (BS.1770) each dialogue shot is gained to, with one static gain per shot. Omitted (the default), the shots keep their own levels. A shot too short (under 400 ms) or too quiet to measure is left at its own level, with a `dialogue_unmatched` warning | +| `duck_delay_ms` | No | How long after the song enters the dialogue starts to duck (default: `0`). At or past the dialogue's end, nothing ducks. Must be ≥ 0 | +| `duck_db` | No | How far the dialogue ducks, in dB (default: `-12`). Must be ≤ 0 | +| `duck_ramp_ms` | No | The length of the linear ramp into the duck (default: `250`). Must be ≥ 0 | +| `fps` | No | The rate the videos play at, needed only when none of them carries one of its own - a pipeline's frames carry none, a file brings its own. Must be > 0 | + +The timeline, in samples at the song's rate: each dialogue shot's track is +fitted to its own frames (trimmed or padded with silence, warning +`dialogue_fitted_to_frames` past a frame's worth of difference), so the +joined dialogue ends exactly where the first song shot's frame 0 is - call +that sample `D`. The song is placed at `D - cue_seconds * sample_rate`, so +song time `cue_seconds` lands on that frame, and it runs to the end of the +picture; a song shorter than that warns `song_short` and is padded with +silence rather than looped. The dialogue ducks by `duck_db` from +`duck_delay_ms` after the song enters, over a linear `duck_ramp_ms` ramp. The +song shots' own audio is discarded, and a dialogue shot with no track of its +own is filled with silence for its length rather than skipped, so later +shots do not land early. + +Frames are joined one for one, so every video must share one frame size and +one frame rate: a mismatch is refused rather than resampled, as is a join +where no video carries a rate and `fps` is not given. + +There is no final normalization here - what level a deliverable sits at is +the workflow's to decide, with [`normalize_audio`](#normalize_audio) after +this step; the templates that mux to video normalize to -3 dBFS peak, as +with any `pair_audio` mux. + +The result is one `video/mp4`: the dialogue's frames then the song shots', +over the mix, with a shot record per input (`shot@`, named from the +step's `dialogue` then `song_shots` references, as `for_each` names a +member), each shot's samples measured off the built waveform rather than +derived from its frames. + ### stabilize_video Remove a generated clip's accumulated framing drift - the slow wander a video diff --git a/tests/test_join_into_song.py b/tests/test_join_into_song.py new file mode 100644 index 00000000..0ea9a22d --- /dev/null +++ b/tests/test_join_into_song.py @@ -0,0 +1,461 @@ +"""Unit tests for join_into_song - dialogue shots, then shots sung to one track.""" + +import os +import sys + +import numpy +import pytest +from PIL import Image + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from dw.loudness import integrated_lufs +from dw.result import AudioTrack, AudioVideo +from dw.shots import shot_references +from dw.tasks.audio_utils import frames_to_samples +from dw.tasks.join_into_song import join_into_song +from dw.tasks.task import _COMMAND_REGISTRY + +FPS = 8 +SR = 8000 + + +def frames(count, size=(16, 16), color=(0, 0, 0)): + return [Image.new("RGB", size, color) for _ in range(count)] + + +def clip(num_frames, fps, sample_rate, level=None, channels=2, size=(16, 16)): + """An AudioVideo with constant-level audio exactly matching its frames, or + silent when level is None. Carries no fps of its own - the caller passes + fps to join_into_song directly, matching the plain-list idiom.""" + if level is None: + audio = None + else: + samples = frames_to_samples(num_frames, fps, sample_rate) + audio = numpy.full((channels, samples), float(level), dtype=numpy.float32) + return AudioVideo(frames(num_frames, size=size), audio, sample_rate) + + +def song_track(num_samples, sample_rate, channels=2, ramp=False, value=0.0): + if ramp: + base = numpy.arange(1, num_samples + 1, dtype=numpy.float32) + audio = numpy.tile(base, (channels, 1)) + else: + audio = numpy.full((channels, num_samples), float(value), dtype=numpy.float32) + return AudioTrack(audio, sample_rate) + + +def sine(num_samples, sample_rate, frequency, amplitude, channels=2): + t = numpy.arange(num_samples) / sample_rate + wave = amplitude * numpy.sin(2 * numpy.pi * frequency * t) + return numpy.tile(wave, (channels, 1)).astype(numpy.float32) + + +# --- 1. Song landing sample ------------------------------------------------- + + +def test_song_lands_exactly_at_song_entry_with_zero_cue(): + dialogue = [clip(8, FPS, SR, level=None)] + song_shots = [clip(8, FPS, SR, level=None)] + song = song_track(8000, SR, ramp=True) + + result = join_into_song(dialogue, song_shots, song, fps=FPS) + + d = frames_to_samples(8, FPS, SR) + assert result.shots[1]["start_sample"] == d + assert numpy.allclose(result.audio[:, d], song.audio[:, 0]) + + +def test_song_lands_cue_seconds_before_dialogue_ends(): + dialogue = [clip(8, FPS, SR, level=None)] + song_shots = [clip(8, FPS, SR, level=None)] + song = song_track(8000, SR, ramp=True) + + result = join_into_song(dialogue, song_shots, song, fps=FPS, cue_seconds=0.5) + + d = frames_to_samples(8, FPS, SR) + entry = d - round(0.5 * SR) + assert numpy.allclose(result.audio[:, entry], song.audio[:, 0]) + + +# --- 2. Ducking -------------------------------------------------------------- + + +def test_duck_ramps_dialogue_under_the_song_entry(): + level = 1.0 + dialogue = [clip(16, FPS, SR, level=level)] + song_shots = [clip(8, FPS, SR, level=None)] + song = song_track(20000, SR, value=0.0) + + result = join_into_song( + dialogue, + song_shots, + song, + fps=FPS, + cue_seconds=1.0, + duck_delay_ms=100, + duck_db=-6, + duck_ramp_ms=250, + ) + + d = frames_to_samples(16, FPS, SR) + entry = d - round(1.0 * SR) + start = entry + round(100 * SR / 1000.0) + ramp = round(250 * SR / 1000.0) + floor = 10.0 ** (-6 / 20.0) + mix = result.audio + + assert numpy.allclose(mix[:, start - 1], level) + for k in (0, 500, ramp - 1): + expected = level * (1.0 + (floor - 1.0) * k / ramp) + assert numpy.allclose(mix[:, start + k], expected, atol=1e-4) + assert numpy.allclose(mix[:, start + ramp + 50], level * floor, atol=1e-4) + assert numpy.allclose(mix[:, d - 1], level * floor, atol=1e-4) + + +def test_duck_delay_past_dialogue_end_leaves_dialogue_unchanged(): + level = 1.0 + dialogue = [clip(16, FPS, SR, level=level)] + song_shots = [clip(8, FPS, SR, level=None)] + song = song_track(20000, SR, value=0.0) + + result = join_into_song( + dialogue, + song_shots, + song, + fps=FPS, + cue_seconds=1.0, + duck_delay_ms=20000, + duck_db=-6, + ) + + d = frames_to_samples(16, FPS, SR) + assert numpy.allclose(result.audio[:, :d], level) + + +def test_duck_db_zero_leaves_dialogue_unchanged(): + level = 1.0 + dialogue = [clip(16, FPS, SR, level=level)] + song_shots = [clip(8, FPS, SR, level=None)] + song = song_track(20000, SR, value=0.0) + + result = join_into_song( + dialogue, + song_shots, + song, + fps=FPS, + cue_seconds=1.0, + duck_delay_ms=100, + duck_db=0, + ) + + d = frames_to_samples(16, FPS, SR) + assert numpy.allclose(result.audio[:, :d], level) + + +# --- 3. Song shots' own audio is discarded ---------------------------------- + + +def test_song_shot_audio_is_discarded(): + dialogue = [clip(8, FPS, SR, level=None)] + song_shots = [clip(8, FPS, SR, level=5.0)] + song = song_track(8000, SR, value=0.0) + + result = join_into_song(dialogue, song_shots, song, fps=FPS) + + d = frames_to_samples(8, FPS, SR) + assert numpy.allclose(result.audio[:, d:], 0.0) + + +# --- 4. dialogue_target_lufs ------------------------------------------------- + + +def test_dialogue_target_lufs_matches_each_shot_independently(): + quiet = AudioVideo(frames(8), sine(8000, SR, 440, 0.05), SR, fps=FPS) + loud = AudioVideo(frames(8), sine(8000, SR, 440, 0.5), SR, fps=FPS) + song_shots = [clip(8, FPS, SR, level=None)] + song = song_track(24000, SR, value=0.0) + target = -20.0 + + result = join_into_song( + [quiet, loud], song_shots, song, fps=FPS, dialogue_target_lufs=target + ) + + for shot in result.shots[:2]: + start, num = shot["start_sample"], shot["num_samples"] + segment = result.audio[:, start : start + num] + measured = integrated_lufs(segment.T, SR) + assert measured is not None + assert abs(measured - target) < 0.5 + + +def test_dialogue_target_lufs_leaves_a_short_shot_unmatched_with_a_warning(caplog): + loud = AudioVideo(frames(8), sine(8000, SR, 440, 0.5), SR, fps=FPS) + short = AudioVideo(frames(1), sine(1000, SR, 440, 0.5), SR, fps=FPS) + song_shots = [clip(8, FPS, SR, level=None)] + song = song_track(24000, SR, value=0.0) + + with caplog.at_level("WARNING"): + result = join_into_song( + [loud, short], song_shots, song, fps=FPS, dialogue_target_lufs=-20.0 + ) + + assert "left at its own level" in caplog.text + short_shot = result.shots[1] + start, num = short_shot["start_sample"], short_shot["num_samples"] + # Unmatched: still exactly the un-gained amplitude 0.5 sine. + assert numpy.allclose( + result.audio[:, start : start + num], + sine(num, SR, 440, 0.5), + atol=1e-4, + ) + + +# --- 5. Shot map -------------------------------------------------------------- + + +def test_shot_map_partitions_frames_and_samples_with_names_and_hard_cuts(): + fps, sr = 4, 100 + dialogue = [frames(4), frames(6)] + song_shots = [frames(5), frames(3)] + song = song_track(1000, sr, value=0.0) + + result = join_into_song(dialogue, song_shots, song, fps=fps) + + shots = result.shots + assert len(shots) == 4 + assert [shot["name"] for shot in shots] == [ + "video 1", + "video 2", + "video 3", + "video 4", + ] + + total_frames = 4 + 6 + 5 + 3 + frame_cursor = 0 + for shot in shots: + assert shot["start_frame"] == frame_cursor + frame_cursor += shot["num_frames"] + assert frame_cursor == total_frames + + assert sum(shot["num_samples"] for shot in shots) == result.audio.shape[1] + sample_cursor = 0 + for shot in shots: + assert shot["start_sample"] == sample_cursor + sample_cursor += shot["num_samples"] + assert sample_cursor == result.audio.shape[1] + + assert "hard_cut" not in shots[0] + for shot in shots[1:]: + assert shot["hard_cut"] is True + + assert [shot["source_index"] for shot in shots] == [0, 1, 2, 3] + + assert result.fps == fps + assert result.audio.shape[1] == frames_to_samples(total_frames, fps, sr) + + +# --- 6. Silent dialogue shot --------------------------------------------------- + + +def test_silent_dialogue_shot_leaves_silence_and_next_shot_starts_on_time(): + fps, sr = 4, 100 + silent = frames(4) + loud = clip(4, fps, sr, level=2.0) + song_shots = [frames(2)] + song = song_track(1000, sr, value=0.0) + + result = join_into_song([silent, loud], song_shots, song, fps=fps) + + per_shot = frames_to_samples(4, fps, sr) + mix = result.audio + assert numpy.allclose(mix[:, :per_shot], 0.0) + assert numpy.allclose(mix[:, per_shot : 2 * per_shot], 2.0) + + +# --- 7. Dialogue fitted to its frames ----------------------------------------- + + +def test_dialogue_audio_longer_than_its_frames_is_trimmed_with_a_warning(caplog): + frame_count = 8 + length = frames_to_samples(frame_count, FPS, SR) + over = numpy.full((2, length + 2000), 3.0, dtype=numpy.float32) + dialogue = [AudioVideo(frames(frame_count), over, SR, fps=FPS)] + song_shots = [frames(8)] + song = song_track(20000, SR, ramp=True) + + with caplog.at_level("WARNING"): + result = join_into_song(dialogue, song_shots, song, fps=FPS) + + assert "trimmed" in caplog.text + d = frames_to_samples(frame_count, FPS, SR) + assert numpy.allclose(result.audio[:, d], song.audio[:, 0]) + assert numpy.allclose(result.audio[:, :d], 3.0) + + +def test_dialogue_audio_shorter_than_its_frames_is_padded_with_a_warning(caplog): + frame_count = 8 + length = frames_to_samples(frame_count, FPS, SR) + under = numpy.full((2, length - 2000), 3.0, dtype=numpy.float32) + dialogue = [AudioVideo(frames(frame_count), under, SR, fps=FPS)] + song_shots = [frames(8)] + song = song_track(20000, SR, ramp=True) + + with caplog.at_level("WARNING"): + result = join_into_song(dialogue, song_shots, song, fps=FPS) + + assert "padded with silence" in caplog.text + d = frames_to_samples(frame_count, FPS, SR) + assert numpy.allclose(result.audio[:, d], song.audio[:, 0]) + assert numpy.allclose(result.audio[:, : length - 2000], 3.0) + assert numpy.allclose(result.audio[:, length - 2000 : d], 0.0) + + +# --- 8. Dialogue at a different sample rate ----------------------------------- + + +def test_dialogue_at_a_different_sample_rate_is_resampled_to_the_songs_rate(): + frame_count = 8 + dialogue_sr = 4000 + dialogue_length = frames_to_samples(frame_count, FPS, dialogue_sr) + audio = numpy.full((2, dialogue_length), 1.0, dtype=numpy.float32) + dialogue = [AudioVideo(frames(frame_count), audio, dialogue_sr, fps=FPS)] + song_shots = [frames(8)] + song = song_track(20000, SR, value=0.0) + + result = join_into_song(dialogue, song_shots, song, fps=FPS) + + assert result.sample_rate == SR + + +# --- 9. Song too short --------------------------------------------------------- + + +def test_song_too_short_is_padded_with_silence_and_warns(caplog): + dialogue = [clip(8, FPS, SR, level=None)] + song_shots = [frames(8)] + song = song_track(1000, SR, value=1.0) + + with caplog.at_level("WARNING"): + result = join_into_song(dialogue, song_shots, song, fps=FPS) + + assert "padded with" in caplog.text + d = frames_to_samples(8, FPS, SR) + assert numpy.allclose(result.audio[:, d : d + 1000], 1.0) + assert numpy.allclose(result.audio[:, d + 1000 :], 0.0) + + +# --- 10. Refusals --------------------------------------------------------------- + + +def test_refuses_empty_dialogue(): + with pytest.raises(ValueError, match="non-empty"): + join_into_song([], [frames(4)], song_track(1000, SR), fps=4) + + +def test_refuses_empty_song_shots(): + with pytest.raises(ValueError, match="non-empty"): + join_into_song([frames(4)], [], song_track(1000, SR), fps=4) + + +def test_refuses_non_list_dialogue(): + with pytest.raises(ValueError, match="non-empty"): + join_into_song("not a list", [frames(4)], song_track(1000, SR), fps=4) + + +def test_refuses_cue_seconds_longer_than_dialogue(): + dialogue = [clip(8, FPS, SR, level=None)] + song_shots = [frames(8)] + song = song_track(20000, SR) + + with pytest.raises(ValueError, match="before the film"): + join_into_song(dialogue, song_shots, song, fps=FPS, cue_seconds=2.0) + + +def test_refuses_mismatched_fps_among_the_inputs(): + dialogue = [AudioVideo(frames(4), None, SR, fps=8)] + song_shots = [AudioVideo(frames(4), None, SR, fps=4)] + + with pytest.raises(ValueError, match="one frame rate"): + join_into_song(dialogue, song_shots, song_track(1000, SR)) + + +def test_refuses_when_no_fps_is_available_anywhere(): + with pytest.raises(ValueError, match="needs 'fps'"): + join_into_song([frames(4)], [frames(4)], song_track(1000, SR)) + + +def test_refuses_mismatched_frame_sizes(): + dialogue = [frames(4, size=(16, 16))] + song_shots = [frames(4, size=(8, 8))] + + with pytest.raises(ValueError, match=r"\d+x\d+"): + join_into_song(dialogue, song_shots, song_track(1000, SR), fps=4) + + +def test_refuses_negative_cue_seconds(): + with pytest.raises(ValueError, match="cue_seconds"): + join_into_song([], [], None, cue_seconds=-1) + + +def test_refuses_negative_duck_delay_ms(): + with pytest.raises(ValueError, match="duck_delay_ms"): + join_into_song([], [], None, duck_delay_ms=-1) + + +def test_refuses_negative_duck_ramp_ms(): + with pytest.raises(ValueError, match="duck_ramp_ms"): + join_into_song([], [], None, duck_ramp_ms=-1) + + +def test_refuses_positive_duck_db(): + with pytest.raises(ValueError, match="duck_db"): + join_into_song([], [], None, duck_db=1) + + +def test_refuses_fps_zero(): + with pytest.raises(ValueError, match="fps"): + join_into_song([], [], None, fps=0) + + +# --- 11. Command registration and shot_references ------------------------------- + + +def test_is_registered_as_a_task_command(): + assert "join_into_song" in _COMMAND_REGISTRY + + +def test_shot_references_reports_dialogue_then_song_shots(): + dialogue, song_shots = ["a", "b"], ["c"] + assert shot_references({"dialogue": dialogue, "song_shots": song_shots}) == [ + "a", + "b", + "c", + ] + + +def test_shot_references_reports_videos_for_concat_style_arguments(): + assert shot_references({"videos": ["x", "y"]}) == ["x", "y"] + + +def test_shot_references_reports_none_for_neither(): + assert shot_references({"other": 1}) is None + + +# --- 12. Song passed as a file path --------------------------------------------- + + +def test_song_as_a_file_path(tmp_path): + import soundfile + + song_samples = 20000 + path = tmp_path / "song.wav" + soundfile.write(path, numpy.zeros((song_samples, 2), dtype=numpy.float32), SR) + dialogue = [clip(8, FPS, SR, level=None)] + song_shots = [frames(8)] + + result = join_into_song(dialogue, song_shots, str(path), fps=FPS) + + assert result.sample_rate == SR + d = frames_to_samples(8, FPS, SR) + assert result.audio.shape[1] >= d From 0b19ed11234c71f2648be122e0fb9ef38f0f2bad Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 18:44:22 -0500 Subject: [PATCH 012/251] fix(ui): #527 - update stale upload test to the reference/url response shape ArgumentsEditor.test.ts still mocked uploadMedia's old {path, url} response and asserted the argument became the absolute path - the same shape #527 removed from the server in the prior commit. api.uploadMedia's type never had a path field to begin with; MediaArgumentInput.svelte already writes result.reference ?? result.url. Bring the test's mock and assertion into line with that, which is what npm run check (svelte-check + tsc) caught as a compile error on the removed field. Co-Authored-By: Claude Sonnet 5 --- ui/src/lib/editor/ArgumentsEditor.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ui/src/lib/editor/ArgumentsEditor.test.ts b/ui/src/lib/editor/ArgumentsEditor.test.ts index 45442432..cbd9598f 100644 --- a/ui/src/lib/editor/ArgumentsEditor.test.ts +++ b/ui/src/lib/editor/ArgumentsEditor.test.ts @@ -44,9 +44,9 @@ describe('ArgumentsEditor media arguments', () => { expect(input).not.toHaveAttribute('placeholder', 'image path or URL') }) - it('picking a file uploads it and writes the server path into the argument', async () => { + it('picking a file uploads it and writes the asset reference into the argument', async () => { vi.spyOn(api, 'uploadMedia').mockResolvedValue({ - path: '/abs/outputs/uploads/abc123.png', + reference: 'asset:abc123.png', url: '/outputs/uploads/abc123.png', }) @@ -68,8 +68,6 @@ describe('ArgumentsEditor media arguments', () => { // resolves asynchronously - wait for the mocked upload to settle await vi.waitFor(() => expect(api.uploadMedia).toHaveBeenCalledWith(file)) - await vi.waitFor(() => - expect(args.image).toBe('/abs/outputs/uploads/abc123.png'), - ) + await vi.waitFor(() => expect(args.image).toBe('asset:abc123.png')) }) }) From 2628c0b5adfae10092dc2c51ad44c425d1bfe9b8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 18:45:28 -0500 Subject: [PATCH 013/251] style: ruff format tests/test_library_sources.py Caught by the pre-hand-off ruff format gate for #527 - a line from develop's own recent history exceeded the formatter's line length. Co-Authored-By: Claude Sonnet 5 --- tests/test_library_sources.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_library_sources.py b/tests/test_library_sources.py index 12bbf59e..9b096628 100644 --- a/tests/test_library_sources.py +++ b/tests/test_library_sources.py @@ -198,9 +198,7 @@ def test_saving_an_example_prompt_writes_a_copy(self, client): ) assert saved.status_code == 200 assert saved.json()["name"] == "flux/daffodil" - assert os.path.isfile( - os.path.join(workspace.prompts, "flux", "daffodil.json") - ) + assert os.path.isfile(os.path.join(workspace.prompts, "flux", "daffodil.json")) with open(checkout / "prompts" / "flux" / "daffodil.json") as file: assert json.load(file)["text"] == "a biomechanical daffodil" assert api.get("/api/prompts/flux/daffodil").json()["text"] == "changed" From 3b84b3788816730554ceb41ed3275b702beb4860 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 18:50:49 -0500 Subject: [PATCH 014/251] docs(proposals): #474 - design record for normalize_audio(limit=true) Co-Authored-By: Claude Opus 5.5 --- .../normalize-audio-limiter-complete.md | 137 ++++++++++++++++++ docs/proposals/todo.md | 5 + 2 files changed, 142 insertions(+) create mode 100644 docs/proposals/complete/normalize-audio-limiter-complete.md diff --git a/docs/proposals/complete/normalize-audio-limiter-complete.md b/docs/proposals/complete/normalize-audio-limiter-complete.md new file mode 100644 index 00000000..132a022f --- /dev/null +++ b/docs/proposals/complete/normalize-audio-limiter-complete.md @@ -0,0 +1,137 @@ +# A true-peak limit mode for `normalize_audio` (#474) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out session, +2026-09-27). Designed on #474 (split from #467). Plan v1 was approved with its +defaults to Q1-Q5. Plan v2 re-planned stage 2 after a bounce and was approved +with Q6's default (a). Built as stages #496 (1) and #497 (2). + +## The problem + +`normalize_audio(target_lufs=T, peak_dbfs=C)` applies one static gain, +`min(T - measured, C - sample_peak)`. When a single transient sets the peak, +the track can't reach T and the step warns `target_lufs_capped`. An example is +the studio-audience laugh in #467: a 22.7 dB crest held Rollback at -20.7 LUFS +with a -3 dBFS peak. #467 fixed loudness *matching* by matching downward. +This feature handles the other case: a series or a song that should sit +louder than its most dynamic material allows, such as the -14 to -16 LUFS +that streaming targets. The #486 field session got there only by leaving dw +for ffmpeg `alimiter`, and lost its provenance on the way. + +## Verdict and what was decided + +The verdict was **build smaller**: one `limit` boolean on `normalize_audio`, +with fixed timings and no knobs. +- Q1: a mode on the existing task, not a `limit_audio` task. +- Q2: `compress_audio(mode="limit")` is left alone and documented. +- Q3: timings and thresholds are fixed. +- Q4: with `limit`, `peak_dbfs` is a true-peak ceiling. +- Q5: only `assemble-and-score` exposes it. +- Q6: the ceiling is held on the mix, not the muxed film. + +## What was built + +### Stage 1 (#496): `normalize_audio(limit=true)` + +The signature is now `normalize_audio(audio, peak_dbfs=-1.0, target_lufs=None, +limit=False, sample_rate=None)`, in `dw/tasks/audio_utils.py`. `limit=False` +is today's path unchanged. A non-bool `limit` is refused. + +With `limit=True`: +- **Ceiling.** `peak_dbfs` is a true-peak ceiling: 4x `resample_poly`, + linked across channels, and oversampled in blocks so that a long track + never holds 4x of itself. +- **Gain.** It starts at `target_lufs - measured`, or at the peak-normalising + gain when there is no target, with no ceiling cap. The makeup gain comes + only from the target, so the limiter can't auto-level the way `alimiter` + does by default. +- **Limiter** (`_limiter_curve`, `_limit_at`). Vectorised, with no + per-sample Python loop. Look-ahead 5 ms, hold 20 ms, release 150 ms per + 6 dB (`LIMITER_*`). The look-ahead is applied to the gain curve and never + to the signal, so there is no delay and sync with picture is untouched. +- **Gain search** (`_search_gain`). Limiting lowers loudness by an amount + that depends on how dense the material is. The plan said to make one + correction pass; stage 1 bounced on that (below), so the gain is + **searched for** instead: + - it starts at the target's own gain; + - it tries the 12 dB cap next; + - it closes by regula falsi to within 0.1 LU (`LIMITER_TOLERANCE_LU`), in + at most 8 limiting passes. +- **Caps and warnings.** + - Reduction stops at 12 dB (`LIMITER_MAX_REDUCTION_DB`). + - A final result further short of the target than the tolerance, whether + at the cap or not, warns `target_lufs_capped` with `limited: true` and + `shortfall_lu`, both on the warning event. + - `limiter_heavy` fires past 6 dB of reduction. +- **Reporting.** It extends the #392 log event: `constraint: "limiter"`, + `gain_db`, `max_gain_reduction_db`, `limited_fraction`, + `output_true_peak_dbfs`, `output_lufs`. + +Also in stage 1: +- `scipy` is declared in `pyproject.toml`. It was already imported by + `dw/loudness.py` but came in only transitively. +- docs/TASKS.md has the `target_lufs` and `limit` rows, and a note that + `compress_audio`'s `limit` mode is sample-peak with no look-ahead. +- The CLAUDE.md `normalize_audio` bullet was updated. + +### Stage 2 (#497): `assemble-and-score` and the skills + +- **The template.** `workflows/templates/assemble-and-score.json` has a + `limit` variable, default `false`, passed to `balanced` beside #467's + `target_lufs` (`peak_dbfs: -3.0` unchanged). +- **The description (v2).** It says the ceiling holds on the mix `balanced` + writes, not on the film: the AAC mux can land up to about 1 dB above it on + limited material. The measured case was -2.54 dBTP, which is still under + the -1 dBTP ceiling of streaming delivery. +- **The skills.** + - `series-episodes`: match downward by default. Pass `limit: true` beside + the same `target_lufs` only when the series should sit louder than its + most dynamic episode allows. `limiter_heavy` means lower the target. + - `minimax-music3`: one line saying the same for a song master. + +## Measured on the fixture + +The fixture is `asset:qa-cast/ep11-coldopen.mp4`, at -17.43 LUFS and +-0.96 dBTP. Runs at `target_lufs: -16`, `peak_dbfs: -3`: +- **Without `limit`:** capped at about -19.4/-19.5 LUFS (shortfall about + 3.4 LU), `constraint: "peak_ceiling"`. +- **`limit: true`, the task alone:** -15.96 LUFS and -2.999 dBTP. + `max_gain_reduction_db` 4.46, `limited_fraction` 0.162. +- **`limit: true`, the `assemble-and-score` film:** + - the `balanced` mix is at -3.0 dBTP / -16.0 LUFS; + - the AAC-muxed film is at -16.01 LUFS / **-2.537 dBTP**, an encode + overshoot of 0.46 dB; + - an unlimited mix at a -3.0 sample peak came out of the same mux at + -3.10 dBTP. + +## Deferred, and why + +- **Holding the ceiling on the encoded film** (Q6 (b)/(c)). No fixed margin + can be known to be enough, because the overshoot depends on the material. + A trim after the encode would lift the non-goal "changing how the encoded + file is checked" and would be a stage of its own. Don chose (a): the film + gate is ≤ -2.0 dBTP with no `audio_clipped`. +- **`warn_if_written_above_full_scale` reads sample peak** (`dw/result.py`), + although `probe_media` measures `true_peak_dbfs`. This is a proposed + follow-up that was not filed. +- **`compress_audio(mode="limit")`** is unchanged (Q2). It is sample-peak, + has no look-ahead, and nothing uses it. Routing it through the new limiter, + or removing it, would each be an issue of its own. +- **`limit` on other templates** (Q5): `dissolve-between-shots` and + `minimax/music-video` need `target_lufs` exposed first. +- **Tunable timings** (Q3). A field report of pumping would be what brings + them back. + +## Cost and bounces + +The stage comments record no usage figures, so no cost is given here. The +plan estimated $4-6 for stage 1, $2-3 for stage 2, and about $0.5 for the v2 +change. + +| stage | bounces | why | +|---|---|---| +| #496 | 1 | C-F133 `heavy` (-12 LUFS target) landed 2.06 LU short with no `target_lufs_capped`. The plan's single correction pass didn't converge on dense material. Fixed by the gain search. | +| #497 | 1 | C-F135 asserted ≤ -2.9 dBTP on the film, and the AAC mux gave -2.54. This was the plan's gap, not a build miss. Re-planned as v2 (Q6), and verified against C-F150 and C-F136. | + +Cases: C-F132, C-F133, C-F134 (stage 1), C-F150, C-F136 (stage 2). C-F135 +asserts the v1 film gate; dkackman/harnest#25 asks to retire it in favour of +C-F150. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 7984c436..8fad1581 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -55,6 +55,11 @@ remaining deferred fix is recorded in workflows** (#479, stages #501-#502), shipped 2026-09-27. Record, including what was deferred (the host-memory half, measured calibration): `complete/h3-vram-ceiling-references-complete.md`. +- **A true-peak limit mode for `normalize_audio`** (#474, stages + #496-#497), shipped 2026-09-27. Record, including what was deferred + (holding the ceiling on the encoded film, true-peak in the post-encode + check, `compress_audio`'s limit mode): + `complete/normalize-audio-limiter-complete.md`. ## Declined From b23d21976ddb43c0d43647123c40e8e2a2b51bfe Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 18:53:13 -0500 Subject: [PATCH 015/251] fix(security): #529 - nest the Hub repo-id message in model_name's path/traversal refusals validate_model_name's URL-shaped refusal already nested HFValidationError's "Repo id must be in the form..." message from validate_repo_id inside its own sentence, matching download_model's wording. The path-traversal and out-of-root refusals (raised by validate_media_path/validate_path) did not, so a caller comparing the two entry points saw different wording for the same three shapes even though both refuse. Nest the same HF message onto those refusals too, keeping each one's specific reason as the outer sentence. Co-Authored-By: Claude Sonnet 5 --- dw/locations.py | 17 ++++++++++++++--- tests/test_locations.py | 4 ++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/dw/locations.py b/dw/locations.py index deac8dec..eeb6a7d2 100644 --- a/dw/locations.py +++ b/dw/locations.py @@ -445,9 +445,20 @@ def validate_model_name(name, base_dir=None): f"Refusing a model_name of '{name}': it is a URL, not a Hub " f"repo id or a local model directory ({e})." ) - return validate_media_path( - str(name), base_dir, "a model_name", require_exists=False - ) + # Not a URL either, so it is checked as a path next. Every refusal + # below is nested onto this same HF message (#529) - a caller who + # only reads the leaf ("Repo id must be in the form...") sees the one + # rule `download_model` states for a repo id, whichever shape of it + # tripped; the outer sentence still says *why this particular value* + # was refused (a '..' segment, or a directory outside every root). + try: + return validate_media_path( + str(name), base_dir, "a model_name", require_exists=False + ) + except PathTraversalError as path_error: + raise PathTraversalError(f"{path_error} ({e}).") + except InvalidInputError as path_error: + raise InvalidInputError(f"{path_error} ({e}).") # The key a Hub file inside a model repo is named by - a lora's, an IP diff --git a/tests/test_locations.py b/tests/test_locations.py index 3d9d8f5a..dedc549c 100644 --- a/tests/test_locations.py +++ b/tests/test_locations.py @@ -292,11 +292,11 @@ def test_a_repo_id_is_allowed(self, untrusted): def test_an_absolute_path_outside_every_root_is_refused( self, untrusted, workflow_dir ): - with pytest.raises(PathTraversalError): + with pytest.raises(PathTraversalError, match="Repo id must be in the form"): validate_model_name("/etc/passwd", workflow_dir) def test_a_traversal_shaped_name_is_refused(self, untrusted, workflow_dir): - with pytest.raises(PathTraversalError): + with pytest.raises(PathTraversalError, match="Repo id must be in the form"): validate_model_name("org/name/../../x", workflow_dir) def test_a_local_model_directory_inside_a_root_is_allowed( From 80a4ebd7c9434822f3fc0358362fecd267de04c5 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 19:21:07 -0500 Subject: [PATCH 016/251] feat(tasks): #513 - hand a declared result.fps on, and refuse a contradicting fps A step that wrote its video with result.fps now writes that rate back onto the AudioVideo a later previous_result: consumer reads, as the fitted audio already is (#197). A 24 fps shot written at 12 was otherwise still 24 to join_into_song and was silently re-timed. join_into_song also refuses an explicit fps that contradicts the rate the videos carry. Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 5 ++++- dw/result.py | 8 ++++++++ dw/tasks/join_into_song.py | 9 +++++++++ tests/test_join_into_song.py | 39 ++++++++++++++++++++++++++++++++++++ tests/test_result.py | 16 +++++++++++++++ 5 files changed, 76 insertions(+), 1 deletion(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 9d51c494..2b3be59d 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -427,7 +427,10 @@ shots do not land early. Frames are joined one for one, so every video must share one frame size and one frame rate: a mismatch is refused rather than resampled, as is a join -where no video carries a rate and `fps` is not given. +where no video carries a rate and `fps` is not given, or an `fps` that +contradicts the rate the videos carry. A step that wrote its video with +`result.fps` hands that rate on, so a shot written at 12 fps from 24 fps frames +is a 12 fps shot to the join, as it is when read back with `output:`. There is no final normalization here - what level a deliverable sits at is the workflow's to decide, with [`normalize_audio`](#normalize_audio) after diff --git a/dw/result.py b/dw/result.py index 3cc06e92..cb7eaf45 100644 --- a/dw/result.py +++ b/dw/result.py @@ -1202,6 +1202,14 @@ def save_audio_video(self, artifact, output_path, content_type): content_type: MIME type of the video being written """ fps = self.video_fps(artifact) + # A declared result.fps is the rate this video now plays at, so it is + # written back onto the artifact the way the fitted audio below is: a + # later previous_result: consumer reads this same instance, and a 24 + # fps shot written at 12 still told join_into_song it was 24 - its + # frames silently re-timed rather than refused (#513). A cache hit or + # an output: reload already reads the rate off the written file + if self.result_definition.get("fps") is not None: + artifact.fps = fps # The pipeline reports the sample rate of what it generated - the result # definition can still override it sample_rate = self.result_definition.get( diff --git a/dw/tasks/join_into_song.py b/dw/tasks/join_into_song.py index 6b2ccae9..f668783e 100644 --- a/dw/tasks/join_into_song.py +++ b/dw/tasks/join_into_song.py @@ -233,6 +233,15 @@ def _one_fps(videos, names, fps): + ". Frames are joined one for one, so a shot at another rate would " "play at the wrong speed against the song" ) + if fps is not None and rates and abs(fps - next(iter(rates))) > 0.01: + raise ValueError( + f"{COMMAND} was given fps {fps:g}, but the videos carry " + + ", ".join( + f"{name}: {float(rate):g} fps" for name, rate in carried if rate + ) + + ". 'fps' is only for videos that carry no rate - drop it, or " + "write the shots at the rate you want first" + ) if fps is None: fps = next((rate for _name, rate in carried if rate), None) if fps is None: diff --git a/tests/test_join_into_song.py b/tests/test_join_into_song.py index 0ea9a22d..1b0f75f1 100644 --- a/tests/test_join_into_song.py +++ b/tests/test_join_into_song.py @@ -380,6 +380,45 @@ def test_refuses_mismatched_fps_among_the_inputs(): join_into_song(dialogue, song_shots, song_track(1000, SR)) +def test_refuses_a_shot_a_previous_step_wrote_at_another_rate(tmp_path): + # The #513 bounce: pair_audio kept a 24 fps asset's rate, the step wrote + # it with result.fps 12, and the join read the in-memory 24 - so the shot + # was re-timed in silence. The saved step's rate is what the join sees + from unittest.mock import patch + + from dw.result import Result + + shot = AudioVideo(frames(4), None, SR, fps=24) + written = Result({"content_type": "video/mp4", "fps": 12}) + written.add_result(shot) + with patch("dw.result.export_to_video"): + written.save(str(tmp_path), "co12") + [twelve] = written.get_artifacts() + + dialogue = [AudioVideo(frames(4), None, SR, fps=24)] + song_shots = [twelve, AudioVideo(frames(4), None, SR, fps=24)] + + with pytest.raises(ValueError, match=r"24 fps.*12 fps|12 fps.*24 fps"): + join_into_song(dialogue, song_shots, song_track(1000, SR)) + + +def test_refuses_an_fps_the_videos_contradict(): + dialogue = [AudioVideo(frames(4), None, SR, fps=24)] + song_shots = [AudioVideo(frames(4), None, SR, fps=24)] + + with pytest.raises(ValueError, match="given fps 12"): + join_into_song(dialogue, song_shots, song_track(1000, SR), fps=12) + + +def test_an_fps_matching_the_carried_rate_is_accepted(): + dialogue = [AudioVideo(frames(4), None, SR, fps=8)] + song_shots = [AudioVideo(frames(4), None, SR, fps=8)] + + result = join_into_song(dialogue, song_shots, song_track(8000, SR), fps=8) + + assert result.fps == 8 + + def test_refuses_when_no_fps_is_available_anywhere(): with pytest.raises(ValueError, match="needs 'fps'"): join_into_song([frames(4)], [frames(4)], song_track(1000, SR)) diff --git a/tests/test_result.py b/tests/test_result.py index 662a3020..dd960f6c 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -1504,6 +1504,22 @@ def test_a_declared_rate_still_wins(self): assert encode.call_args.kwargs["fps"] == 12 + def test_a_declared_rate_is_handed_on_to_a_later_step(self): + # A previous_result: consumer reads this same instance - a 24 fps + # shot written at 12 must read as 12 there, as its file does (#513) + artifact = AudioVideo("frames", torch.zeros((2, 100)), 48000, fps=24) + + self.save({"content_type": "video/mp4", "fps": 12}, artifact) + + assert artifact.fps == 12 + + def test_an_undeclared_rate_is_not_written_back(self): + artifact = AudioVideo("frames", torch.zeros((2, 100)), 48000) + + self.save({"content_type": "video/mp4"}, artifact) + + assert artifact.fps is None + def test_the_old_default_holds_when_nothing_knows_the_rate(self): artifact = AudioVideo("frames", torch.zeros((2, 100)), 48000) From 7c56de394deb6f89cb22676a6dd052033f505d86 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 19:36:34 -0500 Subject: [PATCH 017/251] feat(plugin): #514 - teach the dialogue-into-a-song recipe (join_into_song) The minimax-h3 skill states the recipe in brief - slice_audio slices from cue_seconds, join_into_song, normalize_audio to -3 dBFS, pair_audio - and points at the new `workflows` guide section "A spoken scene breaking into a song", which carries the worked steps and a validated JSON tail. Tests pin the parameter names to join_into_song's signature, the -3 dBFS to music-video's `balanced` step, and validate the guide's tail. Co-Authored-By: Claude Opus 5.5 --- docs/WORKFLOW_GUIDE.md | 72 +++++++++++++++++++++++++++ plugins/dw/skills/minimax-h3/SKILL.md | 28 +++++------ tests/test_plugin_skills.py | 53 +++++++++++++++++++- 3 files changed, 138 insertions(+), 15 deletions(-) diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 07168773..c7c2642f 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -756,6 +756,78 @@ workflow. `reference-to-video` and `dialogue-short` templates). The `identity-referenced` trait in the listing marks the workflows that take one. +### A spoken scene breaking into a song + +A musical number: spoken shots, each carrying its own dialogue audio, then +shots lip-synced to one song, with the song entering under the last spoken +line. A `concat_videos` join is wrong here - it keeps each sung shot's own +audio, the separate slices it was generated against, where the number wants +the song unbroken. The task for it is +[`join_into_song`](TASKS.md#join_into_song) (`get_guide("tasks", +section="join_into_song")`), and the recipe is four parts: + +1. **Pick `cue_seconds`**: the song time that lands on the first sung shot's + frame 0. The song enters that long before the cut, under the last line; + `0` starts it exactly at the cut. It may not be longer than the dialogue. +2. **Generate the sung shots against `slice_audio` slices of the song**, as + `templates/minimax/music-video` does (its `slice` and `shot` steps). The + first slice starts at `cue_seconds`, and each next one starts where the one + before it ended, so the slices tile the song with no gap: `start_seconds` + is `cue_seconds` plus the length of every sung shot before it + (`num_frames / fps` - 24 fps for H3), `duration_seconds` its own length. + The shots, not the slices, are what the join reads. +3. **`join_into_song`** with the dialogue shots, the sung shots, the + *unbroken* song and the same `cue_seconds`. It measures the joined + dialogue at run time and places the song so `cue_seconds` lands on the + first sung frame, ducks the dialogue under it, and discards the sung + shots' own audio. It does not set the output level. +4. **`normalize_audio` to -3 dBFS, then `pair_audio`**, as every template + that muxes to video does - the encoder can add up to ~2 dB, and the + written-peak warning reads the file. + +The tail, with the shots from earlier runs (`output:` or `asset:`) or from +earlier steps of the same workflow (`previous_result:`, `gather:`): + +```json +{ + "name": "joined", + "task": { + "command": "join_into_song", + "arguments": { + "dialogue": ["asset:scene/line-1.mp4", "asset:scene/line-2.mp4"], + "song_shots": ["asset:scene/sung-1.mp4", "asset:scene/sung-2.mp4"], + "song": "asset:scene/song.mp3", + "cue_seconds": 1.5 + } + } +}, +{ + "name": "balanced", + "task": { + "command": "normalize_audio", + "arguments": { "audio": "previous_result:joined", "peak_dbfs": -3.0 } + } +}, +{ + "name": "number", + "task": { + "command": "pair_audio", + "arguments": { + "video": "previous_result:joined", + "audio": "previous_result:balanced", + "fit": "video" + } + }, + "result": { "content_type": "video/mp4", "subfolder": "final" } +} +``` + +`joined` and `balanced` save nothing and are read by later steps, so they run +and write no file of their own; `number` is the deliverable, and its shot map +(`get_gallery_metadata`'s `media.shots`) names one shot per input. The frame +rate comes from the shots themselves - pass `fps` to the join only when none +of its inputs carries one. + ### Saying which output is the deliverable A run writes everything into one directory, so a finished episode sits diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index 3c32e5fa..a72bdc93 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -66,10 +66,17 @@ arguments; the prompt format is MiniMax's, from their text not here. audio reference, not a repeated description. Duck a score burying voice-over rather than raising the world track - a `gain_audio` step per shot, negative `gain_db`. +- **Dialogue into a song**, not a concat: sing each shot to a `slice_audio` + slice, the first at `cue_seconds` (song time on the first sung frame; it + enters that long before the cut), each next where the last ended; then + `join_into_song` (dialogue, `song_shots`, unbroken `song`, same + `cue_seconds`), `normalize_audio` to -3 dBFS, `pair_audio`. Recipe: + `workflows` guide, "A spoken scene breaking into a song". - **Unrelated shots, no cut**: `templates/minimax/shots-batch` - one H3 step per `shots` entry, no shared cast, no concat. `keep_output` each clip, then `templates/assemble-and-score` cuts and scores. -- **Music alone**: `templates/minimax/music` (Music3); the `minimax-music3` skill. +- **Music alone**: `templates/minimax/music` (Music3); the `minimax-music3` + skill - its `audio_duration` is a ceiling, not a length. If none fits, compose from `list_tasks` before authoring a new workflow, and read the `workflows` guide's authoring section first. @@ -126,10 +133,7 @@ read the `workflows` guide's authoring section first. SIGKILL near the end where a fresh one would not. - Ref2VA limits: at most 9 images, 3 videos, 3 audio clips, 12 files; audio can never be the only reference. References are labelled in order. -- Music3's `audio_duration` is a ceiling: ask for more than needed and trim - with `templates/audio-trim-fade`. -- Write the prompt for the length generated: timestamps should span the - duration. +- Write the prompt for the length generated: timestamps span the duration. ## Prompts @@ -179,16 +183,12 @@ itself - or every shot inherits the portrait's composition. a chain for drift sharpening to noise, and `get_output_audio` for a voice-over without affect. `get_output_audio` returns sound, not text; to confirm a line rendered, use its docstring's transcription route - (`validate_workflow` on `templates/transcribe-audio`, `run_workflow` with - `acknowledged_cost` bound, `get_output_text`, `delete_output`) - `basis: - "unknown"`, quote seconds not minutes. + (`templates/transcribe-audio`, then `get_output_text`). A `shot_dead_air` finding: bed with `slice_audio`->`loop_audio`->`mix_audio` - room tone, not silence. - Then `get_gallery_metadata` for duration/audio presence, - and hand the user the gallery `url` - (`list_gallery`). `get_output_image` works only on image steps - the - Z-Image portraits and boards of `dialogue-short`, `storyboard`, - `generated-subject-reference` and `music-video`. + room tone, not silence. Then `get_gallery_metadata` for duration/audio + presence, and hand the user the gallery `url` + (`list_gallery`). `get_output_image` works only on image steps (the + Z-Image portraits and boards). 5. After a run worth keeping, `get_job_workflow` and `save_workflow` it; `export_job` bundles it on the server. `auth_required: false` - fetch `open_url` into `exports/` (never a temp dir). `true` - hand `open_url` diff --git a/tests/test_plugin_skills.py b/tests/test_plugin_skills.py index 41b4d1e0..d3a52a7e 100644 --- a/tests/test_plugin_skills.py +++ b/tests/test_plugin_skills.py @@ -115,7 +115,10 @@ def test_every_catalog_name_a_skill_quotes_resolves(path): # The task commands a skill sends an agent to by name. A skill cannot be # swept for these the way catalog names are - a backticked identifier may be # a tool, a field or a task - so each one is listed here. -SKILL_TASKS = {"minimax-music3": ["attribute_voices"]} +SKILL_TASKS = { + "minimax-music3": ["attribute_voices"], + "minimax-h3": ["join_into_song", "slice_audio", "normalize_audio", "pair_audio"], +} @pytest.mark.parametrize( @@ -342,6 +345,54 @@ def test_the_768p_ref2va_tradeoff_points_at_the_recipes_guide(self): assert "175" in guide and "17n+5" in guide assert "31 minutes" in guide + def test_the_dialogue_into_a_song_recipe_is_the_task_s_and_the_guide_s(self): + """#514: the skill states the recipe in brief and points at the + `workflows` guide for the worked tail. The parameter names are + join_into_song's own, the -3 dBFS is music-video's `balanced` step, + and the guide's tail validates as a workflow, so a rename or a + re-level fails here rather than in a session following the text.""" + from dw.server.guides import get_guide + from dw.tasks.join_into_song import join_into_song + from dw.workflow import Workflow + + text = skill_text(H3_SKILL) + parameters = inspect.signature(join_into_song).parameters + for name in ("cue_seconds", "song_shots", "song"): + assert name in parameters + assert f"`{name}`" in text + + path = os.path.join( + REPO_ROOT, "workflows", "templates", "minimax", "music-video.json" + ) + steps = json.load(open(path, encoding="utf-8"))["steps"] + (balanced,) = [step for step in steps if step["name"] == "balanced"] + assert balanced["task"]["arguments"]["peak_dbfs"] == -3.0 + assert "-3 dBFS" in text + + section = "A spoken scene breaking into a song" + assert f'"{section}"' in text + guide = get_guide("workflows", section=section)["content"] + for word in ( + "`join_into_song`", + "`cue_seconds`", + "`slice_audio`", + "`normalize_audio`", + "`pair_audio`", + "-3 dBFS", + ): + assert word in guide + + (tail,) = re.findall(r"```json\n(.*?)```", guide, re.DOTALL) + definition = {"id": "number", "steps": json.loads(f"[{tail}]")} + (normalize,) = [ + step + for step in definition["steps"] + if step["task"]["command"] == "normalize_audio" + ] + assert normalize["task"]["arguments"]["peak_dbfs"] == -3.0 + workflow = Workflow(definition, REPO_ROOT, os.path.join(REPO_ROOT, "n.json")) + assert workflow.validation_errors() == [] + def test_the_1344x768_reference_ceilings_are_the_templates_vram_estimate(self): """#479: `gb_per_reference` and the per-step projection moved the Ref2VA ceiling from one number (175 frames, unstated reference count) From 81ea49510d57443226fa865e1469d44cc1af4711 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 19:41:20 -0500 Subject: [PATCH 018/251] docs(audio): #540 - correct normalize_audio(limit=true)'s gain_db contract get_task("normalize_audio") quoted the docstring's "the gain is target_lufs's alone (the limiter never adds any)", but the limiter path searches for a static pre-limiter gain that reaches target_lufs after limiting - which runs past the plain target_lufs-only gain on dense material, since limiting itself takes back some loudness. That search is intended behavior (docs/TASKS.md's `limit` row already describes it correctly, and CLAUDE.md documents the search), so the fix corrects the two inaccurate docstrings rather than the algorithm, and pins the behavior with a new test. Co-Authored-By: Claude Sonnet 5 --- dw/tasks/audio_utils.py | 22 ++++++++++++++-------- tests/test_audio_utils.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/dw/tasks/audio_utils.py b/dw/tasks/audio_utils.py index a4722893..0e8e32ec 100644 --- a/dw/tasks/audio_utils.py +++ b/dw/tasks/audio_utils.py @@ -1424,10 +1424,13 @@ def normalize_audio( limit: Hold peak_dbfs with a look-ahead limiter instead of capping the gain, so target_lufs can be reached past a transient that sets the peak. peak_dbfs becomes a true-peak (4x oversampled, BS.1770) - ceiling; the gain is target_lufs's alone (the limiter never adds - any), and limiting stops at 12 dB of reduction, past which - target_lufs is warned as capped. False (the default) leaves - behavior exactly as without it + ceiling; the limiter itself never adds gain, but the static gain + applied before it is searched for - not read off target_lufs + directly - since limiting takes back some of the loudness a + plain gain would have reached, more on dense material, so the + reported gain_db can run past target_lufs's own gain. Limiting + stops at 12 dB of reduction, past which target_lufs is warned as + capped. False (the default) leaves behavior exactly as without it sample_rate: Sample rate of a waveform passed directly Returns: @@ -1666,10 +1669,13 @@ def miss(result): def _normalize_limited(waveform, sample_rate, peak_dbfs, target_lufs): - """normalize_audio(limit=True): the target's gain, applied uncapped, with - a true-peak limiter holding peak_dbfs. The limiter is never a gain stage - - makeup comes from target_lufs alone - and it stops at - LIMITER_MAX_REDUCTION_DB, past which the gain is what stops instead.""" + """normalize_audio(limit=True): a static gain, applied uncapped, with a + true-peak limiter holding peak_dbfs. The limiter is never a gain stage - + it only tames what the static gain sends it - but that static gain is + searched for so the *limited* output lands on target_lufs, since + limiting takes back some of the loudness a plain target_lufs gain would + have reached; the search stops at LIMITER_MAX_REDUCTION_DB, past which + the gain is what stops instead.""" ceiling = 10 ** (peak_dbfs / 20) envelope = _true_peak_envelope(waveform) input_peak_db = 20 * numpy.log10(float(envelope.max())) diff --git a/tests/test_audio_utils.py b/tests/test_audio_utils.py index 59ccec6b..b3e0b93b 100644 --- a/tests/test_audio_utils.py +++ b/tests/test_audio_utils.py @@ -897,6 +897,21 @@ def test_dense_material_still_reaches_a_target_inside_the_cap(self): assert 0 < logs[0]["max_gain_reduction_db"] < 12.0 assert logs[0]["limited_fraction"] > 0.3 + def test_gain_db_is_the_searched_static_gain_not_the_plain_lufs_gain(self): + # #540: the limiter takes back loudness on dense material, so the + # static gain the search settles on to still land on target_lufs + # runs past the plain (unlimited) gain a naive target_lufs - measured + # would give - gain_db reports that searched gain, not the plain one + from dw.loudness import integrated_lufs + + waveform = self._dense() + measured_lufs = integrated_lufs(waveform.T, self.RATE) + plain_gain_db = -12.0 - measured_lufs + track, logs, warnings = self._run(waveform, peak_dbfs=-3.0, target_lufs=-12.0) + + assert "target_lufs_capped" not in warnings + assert logs[0]["gain_db"] > plain_gain_db + 0.1 + def test_a_target_the_cap_stops_short_of_says_by_how_much(self): # Past the cap the track lands short; the warning's shortfall is the # one the output actually has, not the gain's From 2f6f22326b9eed70c01aab477476da62feec0382 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 19:52:29 -0500 Subject: [PATCH 019/251] fix(docs): #551 - name the resize route for a join size mismatch, fix resize_center_crop table row "No reconciliation" in the concat_videos/dissolve_videos videos docstrings and video_size_errors.py read as "impossible" - it only meant the join tasks themselves won't resize. Name the actual route: video_frames -> resize_rescale/resize_center_crop -> pair_audio(fit="video"). Also fix TASKS.md's resize_center_crop row, which said "Resize with center crop" while the task summary (correctly) says it crops to a centered square then stretches - a trap for a non-square target. Co-Authored-By: Claude Sonnet 5 --- docs/TASKS.md | 6 +++--- dw/tasks/concat_videos.py | 7 +++++-- dw/tasks/dissolve_videos.py | 11 +++++++---- dw/video_size_errors.py | 10 +++++++--- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 2b3be59d..c9b06324 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -92,7 +92,7 @@ All accept an `image` argument with processing parameters: | Command | Description | Extra Arguments | | ------- | ----------- | --------------- | | `remove_background` | Remove image background | | -| `resize_center_crop` | Resize with center crop | `width`, `height` | +| `resize_center_crop` | Crop to a centered square, then stretch to width x height - distorts a non-square target | `width`, `height` | | `resize_resample` | Resample to nearest 64px multiple | | | `resize_rescale` | Resize to exact dimensions | `width`, `height` | | `resize_bucket` | Snap to closest model-native aspect ratio | `resolution`, `ratios`, `alignment` | @@ -212,7 +212,7 @@ video generation" in the workflow guide): | Argument | Required | Description | | -------- | -------- | ----------- | -| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, which is read with the audio muxed into it; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, there is no reconciliation for a size mismatch, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518) | +| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, which is read with the audio muxed into it; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, this task does not resize one for you, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518). To fit the odd video: `video_frames` to get its frames, `resize_rescale`/`resize_center_crop` to the target size, then `pair_audio(fit="video")` to put its soundtrack back before passing it here (#551) | | `trim_frames` | No | Frames dropped from the head of every video after the first (default: 0) | | `crossfade_ms` | No | Equal-power crossfade at each audio seam, drawn from the trimmed material - no effect when `trim_frames` is 0, and validation warns when one is written there (default: 75) | | `audio_bleed_ms` | No | How long the outgoing video's tail rings on over the head of the next one, at seams with nothing trimmed to crossfade (default: 0, off) | @@ -358,7 +358,7 @@ montage cut to a score wants: | Argument | Required | Description | | -------- | -------- | ----------- | -| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, one entry per video as with `concat_videos`; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, there is no reconciliation for a size mismatch, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518) | +| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, one entry per video as with `concat_videos`; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, this task does not resize one for you, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518). To fit the odd video: `video_frames` to get its frames, `resize_rescale`/`resize_center_crop` to the target size, then `pair_audio(fit="video")` to put its soundtrack back before passing it here (#551) | | `dissolve_frames` | No | Frames of overlap at each seam, blended linearly (default: 12). 0 is a hard cut | | `fade_in_frames` | No | Frames over which the first video rises out of `fade_color` (default: 0) | | `fade_out_frames` | No | Frames over which the last video sinks into it (default: 0) | diff --git a/dw/tasks/concat_videos.py b/dw/tasks/concat_videos.py index c0a7f8e6..cbeda36f 100644 --- a/dw/tasks/concat_videos.py +++ b/dw/tasks/concat_videos.py @@ -79,10 +79,13 @@ def concat_videos( step that produced several videos fans this step out over them, one concatenation per video, rather than joining them. Every video must be the same frame size - unlike a sample-rate - mismatch, there is no reconciliation for a size mismatch, so a + mismatch, this task does not resize one for you, so a statically-resolvable (asset:/output:/literal path) size disagreement is refused at validate; one only known at run time - still fails there (#504) + still fails there (#504). To fit the odd video: `video_frames` + to get its frames, `resize_rescale`/`resize_center_crop` to the + target size, then `pair_audio(fit="video")` to put its + soundtrack back before passing it here (#551) trim_frames: Frames dropped from the head of every video after the first - the trim used when each video was generated from the previous one's last frame diff --git a/dw/tasks/dissolve_videos.py b/dw/tasks/dissolve_videos.py index 46e64a15..1eb93a84 100644 --- a/dw/tasks/dissolve_videos.py +++ b/dw/tasks/dissolve_videos.py @@ -58,12 +58,15 @@ def dissolve_videos( AudioVideos, the path or URL of a video file, or a {"location": ...} dict wrapping either (#510). Give each video its own entry, as with concat_videos. Every video must be the - same frame size - unlike a sample-rate mismatch, there is no - reconciliation for a size mismatch, so a statically-resolvable + same frame size - unlike a sample-rate mismatch, this task does + not resize one for you, so a statically-resolvable (asset:/output:/literal path) size disagreement is refused at validate; one only known at run time still fails there (#504). - Soundtracks at different sample rates are not a constraint - see - `sample_rate` below + To fit the odd video: `video_frames` to get its frames, + `resize_rescale`/`resize_center_crop` to the target size, then + `pair_audio(fit="video")` to put its soundtrack back before + passing it here (#551). Soundtracks at different sample rates + are not a constraint - see `sample_rate` below dissolve_frames: Frames of overlap at each seam. 0 is a hard cut fade_in_frames: Frames over which the first video rises out of `fade_color` diff --git a/dw/video_size_errors.py b/dw/video_size_errors.py index 30c3fd42..71d4bd90 100644 --- a/dw/video_size_errors.py +++ b/dw/video_size_errors.py @@ -4,9 +4,13 @@ Both tasks join clips frame-by-frame (`check_same_frame_size`, `dw/tasks/video_utils.py`) and raise once every input has been decoded: "video N is WxH, video M is WxH". Unlike a sample-rate mismatch (#108/#287), which is -auto-resampled with a warning, there is no reconciliation for a size mismatch -- Don declined a resize/fit argument on the join tasks (#504, #512) - so the -only thing to move earlier is the refusal itself. +auto-resampled with a warning, neither join task resizes a mismatched video +for you - Don declined a resize/fit argument on the join tasks themselves +(#504, #512) - so the fix is upstream of the join: `video_frames` to get the +odd video's frames, `resize_rescale`/`resize_center_crop` to the target size, +then `pair_audio(fit="video")` to put its soundtrack back before passing it +to `dissolve_videos`/`concat_videos` (#551). The only thing to move earlier +is the refusal itself. Moved here, into `validation_errors`, for exactly the cases a size is knowable without running anything: an `asset:`/`output:` reference, a From ff31a3fac33401b664926472e9e6ea5a080d99f2 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 20:03:59 -0500 Subject: [PATCH 020/251] feat(audio): #544 - find_loop_bed ranks room-tone loop windows, measured looped Adds the find_loop_bed task (dw/tasks/loop_bed.py, returns json, not an assessment probe), its argument domains, an audio-only float soundtrack decode for video sources (media_audio.decode_soundtrack) and an FFT autocorrelation in _harmonicity. Co-Authored-By: Claude Opus 5.5 --- dw/media_audio.py | 38 ++++ dw/task_domains.py | 13 ++ dw/tasks/audio_utils.py | 12 +- dw/tasks/loop_bed.py | 480 ++++++++++++++++++++++++++++++++++++++++ dw/tasks/task.py | 11 + 5 files changed, 551 insertions(+), 3 deletions(-) create mode 100644 dw/tasks/loop_bed.py diff --git a/dw/media_audio.py b/dw/media_audio.py index 74f96b05..dcf94c96 100644 --- a/dw/media_audio.py +++ b/dw/media_audio.py @@ -215,3 +215,41 @@ def extract_audio(path, start=None, duration=None): "start": start, "excerpt": excerpt, } + + +def decode_soundtrack(path): + """The whole soundtrack of `path` as a float32 (channels, samples) array + and its sample rate, decoded from the audio stream alone. + + `load_audio` on a video decodes every picture frame to get at the track + (`load_audio_video`), which on a minutes-long cut is most of the work and + all of the memory. A caller that only measures the sound - find_loop_bed + (#218) - reads it here instead. Float rather than extract_audio's s16: + room tone sits at -70 to -85 dBFS, close enough to 16-bit's floor that + requantizing would add to what is being measured. + """ + with av.open(path) as container: + if not container.streams.audio: + raise NoSoundtrack(f"{path} has no soundtrack") + stream = container.streams.audio[0] + rate = int(stream.rate) + channels = int(stream.channels) + layout = ( + "stereo" + if channels == 2 + else ("mono" if channels == 1 else stream.layout.name) + ) + resampler = AudioResampler(format="flt", layout=layout, rate=rate) + pieces = [] + for frame in container.decode(stream): + for chunk in resampler.resample(frame): + pieces.append(chunk.to_ndarray().reshape(-1, channels)) + for chunk in resampler.resample(None): + pieces.append(chunk.to_ndarray().reshape(-1, channels)) + + samples = ( + numpy.concatenate(pieces) + if pieces + else numpy.zeros((0, channels), numpy.float32) + ) + return numpy.ascontiguousarray(samples.T, dtype=numpy.float32), rate diff --git a/dw/task_domains.py b/dw/task_domains.py index a0ddfdf2..e0518ef1 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -105,6 +105,19 @@ "crossfade_ms": NON_NEGATIVE, "sample_rate": POSITIVE, }, + "find_loop_bed": { + "start_seconds": NON_NEGATIVE, + "end_seconds": POSITIVE, + "min_seconds": POSITIVE, + "max_seconds": POSITIVE, + "max_bin_dbfs": NON_POSITIVE, + "max_mean_dbfs": NON_POSITIVE, + "max_spike_db": NON_NEGATIVE, + "crossfade_ms": NON_NEGATIVE, + "loop_seconds": POSITIVE, + "target_bed_dbfs": NON_POSITIVE, + "max_candidates": POSITIVE, + }, "fade_audio": { "fade_in_ms": NON_NEGATIVE, "fade_out_ms": NON_NEGATIVE, diff --git a/dw/tasks/audio_utils.py b/dw/tasks/audio_utils.py index 0e8e32ec..6bca5ba0 100644 --- a/dw/tasks/audio_utils.py +++ b/dw/tasks/audio_utils.py @@ -249,9 +249,15 @@ def _harmonicity(waveform, sample_rate): energy = float(numpy.dot(centered, centered)) if energy <= 1e-12: continue - correlation = numpy.correlate(centered, centered, mode="full") - zero_lag = correlation.shape[0] // 2 - window = correlation[zero_lag + min_lag : zero_lag + max_lag + 1] + # The autocorrelation through an FFT rather than numpy.correlate, + # which is O(n^2): on a 2 s window at 48 kHz that is about 10^10 + # operations, and find_loop_bed measures many windows (#218). + # Padding to at least 2n - 1 keeps the circular correlation from + # wrapping, so every lag equals the direct sum + size = 1 << int(2 * centered.shape[0] - 1).bit_length() + spectrum = numpy.fft.rfft(centered, size) + correlation = numpy.fft.irfft(spectrum * numpy.conj(spectrum), size) + window = correlation[min_lag : max_lag + 1] if window.size == 0: continue scores.append(float(numpy.max(window) / energy)) diff --git a/dw/tasks/loop_bed.py b/dw/tasks/loop_bed.py new file mode 100644 index 00000000..7e381eb2 --- /dev/null +++ b/dw/tasks/loop_bed.py @@ -0,0 +1,480 @@ +"""find_loop_bed: rank the windows of a recording worth looping into a bed (#218). + +A dialogue cut needs a bed of room tone under it, and the bed is made by +looping a short stretch of the programme's own quiet (`slice_audio` -> +`loop_audio` -> `mix_audio`). Picking that stretch has three traps a level +check misses, each found on a real episode: + +1. near-programme material - faint speech attenuated ~30 dB reads as quiet, + and is audible once it repeats every lap; +2. lap-rate modulation - the loop's repeat rate beats against the source's + own level movement, which one pass through the source cannot show; +3. ticks - a 1 ms transient is invisible to 50 ms RMS and recurs once per lap. + +This searches a time range for windows that pass all three and measures each +survivor *looped*, the way `loop_audio` will make it, so the ranking is of the +bed rather than of the source. It picks a window and builds nothing: the +answer's `start_seconds`/`duration_seconds` are `slice_audio`'s arguments and +`gain` is `mix_audio`'s multiplier, so the remedy is a copy. + +Like the assessment probes it answers JSON and decides nothing, but it is a +search rather than a check of a finished cut, so it is not listed among them. +""" + +import logging +import math +import os + +import numpy + +from ..task_domains import check_arguments +from .audio_utils import ( + HARMONICITY_THRESHOLD, + TONAL_FLATNESS_THRESHOLD, + _as_number, + _harmonicity, + _spectral_flatness, + _waveform_and_rate, + crossfade_concat, +) + +logger = logging.getLogger("dw") + +COMMAND = "find_loop_bed" + +# The envelope's resolution - the field method's, and the bin every level +# reading here is taken over +BIN_SECONDS = 0.05 +# A tick's resolution: a 1 ms transient is what 50 ms RMS cannot see +PEAKS_PER_BIN = 50 +# A looped candidate whose strongest envelope component sits above this +# (relative to the envelope's mean) carries `lap_modulation`. The two field +# winners read -21 and -19 dB +LAP_MODULATION_WARN_DB = -15.0 +# How many survivors, per candidate asked for, are looped and measured +LOOPED_PER_CANDIDATE = 4 +# The ripple spread, as percentiles of the 50 ms bins +RIPPLE_PERCENTILES = (5.0, 95.0) +# Tonality is only measured for windows reaching it in pre-rank order, and +# this bounds how many are measured however much of the range is tonal +MAX_TONAL_CHECKS = 400 + +REJECTION_RULES = ("too_loud", "silent", "spike", "tonal") + + +def _db(value): + return 20.0 * math.log10(value) if value > 0 else None + + +def _round(value, places=2): + return None if value is None else round(float(value), places) + + +def _read_source(audio): + """(waveform, rate) for the `audio` argument. A local video file is read + audio-only through media_audio - load_audio would decode every frame of + a cut to reach its soundtrack.""" + if isinstance(audio, str) and not audio.startswith(("http://", "https://")): + from ..locations import validate_media_path + from ..security import ALLOWED_VIDEO_EXTENSIONS + + extension = os.path.splitext(audio)[1].lower() + if extension in ALLOWED_VIDEO_EXTENSIONS: + from ..media_audio import NoSoundtrack, decode_soundtrack + + path = validate_media_path(audio, None, "an audio argument") + try: + return decode_soundtrack(path) + except NoSoundtrack: + raise ValueError( + f"{COMMAND}: {audio} carries no audio track - a bed is " + "found in a soundtrack" + ) from None + return _waveform_and_rate(audio, None, COMMAND) + + +def _sliding(array, length): + """Every run of `length` consecutive rows of `array`, one per start.""" + return numpy.lib.stride_tricks.sliding_window_view(array, length, axis=0) + + +def _envelope(mono, sample_rate): + """50 ms mean-square bins and each bin's 1 ms peaks, over whole bins. + + A bin is split into PEAKS_PER_BIN sub-blocks, the last padded with zeros + when the bin does not divide evenly, so no sample escapes the peak search + - a tick in a bin's last few samples is still a tick. + """ + bin_length = max(int(round(sample_rate * BIN_SECONDS)), PEAKS_PER_BIN) + count = mono.shape[0] // bin_length + bins = mono[: count * bin_length].reshape(count, bin_length) + power = numpy.mean(numpy.square(bins), axis=1) + block = -(-bin_length // PEAKS_PER_BIN) + padded = numpy.zeros((count, block * PEAKS_PER_BIN)) + padded[:, :bin_length] = numpy.abs(bins) + peaks = padded.reshape(count, PEAKS_PER_BIN, block).max(axis=2) + return bin_length, power, peaks + + +def _bin_db(power): + return 10.0 * numpy.log10(numpy.maximum(power, 1e-20)) + + +def _looped(segment, sample_rate, crossfade_ms, loop_seconds): + """`segment` looped to `loop_seconds` exactly as loop_audio loops it, and + the readings of the result: its 50 ms ripple and its envelope spectrum.""" + samples = segment.shape[0] + window = min(int(crossfade_ms / 1000.0 * sample_rate), samples // 2) + length = int(round(loop_seconds * sample_rate)) + lap = samples - window + laps = 1 + max(0, -(-(length - samples) // lap)) + bed = crossfade_concat( + [segment[numpy.newaxis, :]] * laps, + sample_rate, + window / sample_rate * 1000.0, + )[0, :length] + + _, power, _ = _envelope(bed, sample_rate) + low, high = numpy.percentile(_bin_db(power), RIPPLE_PERCENTILES) + envelope = numpy.sqrt(power) + mean = float(envelope.mean()) + lap_hz = sample_rate / lap + if mean <= 0 or envelope.shape[0] < 4: + return { + "ripple_db": _round(high - low), + "envelope_peak_db": None, + "envelope_peak_hz": None, + "lap_hz": _round(lap_hz, 3), + "lap_component_db": None, + } + + # Each component as the amplitude of the sinusoid it stands for, over + # the envelope's mean: a level wobbling by a fraction m reads 20*log10(m) + varying = envelope - mean + count = varying.shape[0] + spectrum = numpy.abs(numpy.fft.rfft(varying)) * 2.0 / count + frequencies = numpy.fft.rfftfreq(count, BIN_SECONDS) + strongest = 1 + int(numpy.argmax(spectrum[1:])) + # The lap rate rarely lands on an FFT bin, so it is measured directly + times = numpy.arange(count) * BIN_SECONDS + at_lap = abs(numpy.sum(varying * numpy.exp(-2j * numpy.pi * lap_hz * times))) + at_lap = at_lap * 2.0 / count + return { + "ripple_db": _round(high - low), + "envelope_peak_db": _round(_db(spectrum[strongest] / mean)), + "envelope_peak_hz": _round(frequencies[strongest], 3), + "lap_hz": _round(lap_hz, 3), + "lap_component_db": _round(_db(at_lap / mean)), + } + + +def _empty_finding(rejected, criteria): + """The one finding for a search that found nothing, naming the rule that + threw the most out and the argument that relaxes it.""" + if not any(rejected.values()): + message = "no window fit in the range; widen it or lower min_seconds" + rule = None + else: + rule = max(REJECTION_RULES, key=lambda name: rejected[name]) + message = { + "too_loud": ( + f"nothing below {criteria['max_bin_dbfs']:g} dBFS per 50 ms " + f"bin and {criteria['max_mean_dbfs']:g} dBFS mean in the " + "range; widen it or raise max_bin_dbfs/max_mean_dbfs" + ), + "silent": ( + "the quiet stretches in the range are digital silence, which " + "fills nothing; widen the range to reach real room tone" + ), + "spike": ( + f"every quiet window holds a 1 ms peak more than " + f"{criteria['max_spike_db']:g} dB over its median - a tick " + "that would recur once per lap; widen the range or raise " + "max_spike_db" + ), + "tonal": ( + "every quiet window is tonal or speech-like - faint " + "programme that would be audible once it repeats; widen " + "the range" + ), + }[rule] + return {"rule": "no_loop_bed", "rejected_by": rule, "message": message} + + +def find_loop_bed( + audio, + start_seconds=None, + end_seconds=None, + min_seconds=0.5, + max_seconds=2.0, + max_bin_dbfs=-55.0, + max_mean_dbfs=-60.0, + max_spike_db=12.0, + crossfade_ms=250, + loop_seconds=10.0, + target_bed_dbfs=-60.0, + max_candidates=5, +): + """Task command: rank the windows of a recording worth looping into a + room-tone bed, measured as they will sound looped. + + Every window on a 50 ms grid, from min_seconds to max_seconds long, is + kept only if it is quiet (every 50 ms bin at or below max_bin_dbfs, its + mean at or below max_mean_dbfs), not digital silence, free of ticks (its + largest 1 ms peak at most max_spike_db over its median 1 ms peak) and not + tonal or speech-like (bleed_join's flatness and harmonicity test). + Overlapping survivors are thinned to the steadiest, and each remaining + one is looped with loop_audio's own crossfade to loop_seconds and + measured: `ripple_db` (the 5-95 % spread of the looped 50 ms bins), + `envelope_peak_db`/`envelope_peak_hz` (the strongest level wobble) and + `lap_component_db` (the wobble at the lap rate). Candidates are ranked by + looped ripple, lowest first; one whose envelope peak is above -15 dB + carries the `lap_modulation` warning. + + A candidate's start_seconds/duration_seconds are slice_audio's arguments + and its gain is mix_audio's multiplier for reaching target_bed_dbfs. + Finding nothing is an answer: `candidates` is empty, `rejected` counts + the windows each rule threw out, and one finding says which to relax. + The result must be saved as application/json. + + Args: + audio: Path of an audio or video file (a video is read audio-only, + never frame-decoded), or a track or video from an earlier step + start_seconds: Where the search starts; the head of the file if omitted + end_seconds: Where the search ends; the end of the file if omitted + min_seconds: Shortest window tried + max_seconds: Longest window tried + max_bin_dbfs: Every 50 ms bin of a window must be at or below this + max_mean_dbfs: A window's mean (RMS) level must be at or below this + max_spike_db: Most a window's largest 1 ms peak may sit above its + median 1 ms peak, in dB + crossfade_ms: The crossfade candidates are looped with - loop_audio's + default, so what is measured is what loop_audio will make + loop_seconds: Length of the looped result that is measured + target_bed_dbfs: The level each candidate's gain is computed to reach + max_candidates: How many ranked candidates to return + + Returns: + A JSON-safe dict: source, criteria, candidates, rejected, findings + """ + start_seconds = _as_number(start_seconds, float, "start_seconds", COMMAND) + end_seconds = _as_number(end_seconds, float, "end_seconds", COMMAND) + min_seconds = _as_number(min_seconds, float, "min_seconds", COMMAND) + max_seconds = _as_number(max_seconds, float, "max_seconds", COMMAND) + max_bin_dbfs = _as_number(max_bin_dbfs, float, "max_bin_dbfs", COMMAND) + max_mean_dbfs = _as_number(max_mean_dbfs, float, "max_mean_dbfs", COMMAND) + max_spike_db = _as_number(max_spike_db, float, "max_spike_db", COMMAND) + crossfade_ms = _as_number(crossfade_ms, float, "crossfade_ms", COMMAND) + loop_seconds = _as_number(loop_seconds, float, "loop_seconds", COMMAND) + target_bed_dbfs = _as_number(target_bed_dbfs, float, "target_bed_dbfs", COMMAND) + max_candidates = _as_number(max_candidates, int, "max_candidates", COMMAND) + check_arguments( + COMMAND, + start_seconds=start_seconds, + end_seconds=end_seconds, + min_seconds=min_seconds, + max_seconds=max_seconds, + max_bin_dbfs=max_bin_dbfs, + max_mean_dbfs=max_mean_dbfs, + max_spike_db=max_spike_db, + crossfade_ms=crossfade_ms, + loop_seconds=loop_seconds, + target_bed_dbfs=target_bed_dbfs, + max_candidates=max_candidates, + ) + if min_seconds > max_seconds: + raise ValueError( + f"{COMMAND} needs 'min_seconds' ({min_seconds:g}) at or below " + f"'max_seconds' ({max_seconds:g})" + ) + + waveform, sample_rate = _read_source(audio) + mono = numpy.asarray(waveform, dtype=numpy.float64) + mono = mono.mean(axis=0) if mono.ndim == 2 else mono + duration = mono.shape[0] / sample_rate + if duration < min_seconds: + raise ValueError( + f"{COMMAND}: the source is {duration:.2f} s long, shorter than " + f"'min_seconds' ({min_seconds:g}) - there is no window to find" + ) + + first = 0.0 if start_seconds is None else start_seconds + last = duration if end_seconds is None else end_seconds + # Half a sample of slack: an end_seconds copied from a rounded duration + # is the end of the file, not past it + slack = 0.5 / sample_rate + if first >= duration: + raise ValueError( + f"{COMMAND}: 'start_seconds' ({first:g}) is past the end of the " + f"{duration:.2f} s source" + ) + if last > duration + slack: + raise ValueError( + f"{COMMAND}: 'end_seconds' ({last:g}) is past the end of the " + f"{duration:.2f} s source" + ) + if last <= first: + raise ValueError( + f"{COMMAND} needs 'end_seconds' ({last:g}) after 'start_seconds' " + f"({first:g})" + ) + if last - first < min_seconds: + raise ValueError( + f"{COMMAND}: the range {first:g}-{last:g} s is shorter than " + f"'min_seconds' ({min_seconds:g}) - there is no window to find" + ) + + offset = int(round(first * sample_rate)) + stop = min(int(round(last * sample_rate)), mono.shape[0]) + searched = mono[offset:stop] + bin_length, power, peaks = _envelope(searched, sample_rate) + bin_seconds = bin_length / sample_rate + count = power.shape[0] + + shortest = max(1, int(round(min_seconds / bin_seconds))) + longest = min(count, int(round(max_seconds / bin_seconds))) + rejected = dict.fromkeys(REJECTION_RULES, 0) + bin_threshold = 10.0 ** (max_bin_dbfs / 10.0) + mean_threshold = 10.0 ** (max_mean_dbfs / 10.0) + spike_ratio = 10.0 ** (max_spike_db / 20.0) + + # Step 2 and 3, vectorised per window length: every (start, length) pair + # is judged at once for loudness, silence and ticks, and each survivor + # carries its source ripple for the pre-rank + survivors = [] # (source ripple, start bin, length in bins, readings) + for length in range(shortest, longest + 1): + windows = _sliding(power, length) # (starts, length) + loudest = windows.max(axis=1) + mean = windows.mean(axis=1) + loud = (loudest > bin_threshold) | (mean > mean_threshold) + silent = ~loud & (mean <= 0.0) + rejected["too_loud"] += int(loud.sum()) + rejected["silent"] += int(silent.sum()) + quiet = numpy.flatnonzero(~loud & ~silent) + if quiet.size == 0: + continue + + peak_windows = _sliding(peaks, length)[quiet].reshape(quiet.size, -1) + largest = peak_windows.max(axis=1) + median = numpy.median(peak_windows, axis=1) + ticked = largest > spike_ratio * median + rejected["spike"] += int(ticked.sum()) + kept = quiet[~ticked] + if kept.size == 0: + continue + + spikes = 20.0 * numpy.log10(largest[~ticked] / median[~ticked]) + in_db = _bin_db(windows[kept]) + low, high = numpy.percentile(in_db, RIPPLE_PERCENTILES, axis=1) + for position, start in enumerate(kept): + survivors.append( + ( + float(high[position] - low[position]), + int(start), + length, + { + "mean_dbfs": 10.0 * math.log10(float(mean[start])), + "max_bin_dbfs": 10.0 * math.log10(float(loudest[start])), + "spike_db": float(spikes[position]), + }, + ) + ) + + # Steps 4 and 5: steadiest first, thinned so no two overlap. Tonality is + # measured only as a window is reached, and a window mostly inside one + # already found tonal inherits the verdict rather than being measured + # again - otherwise one tonal stretch costs a measurement per 50 ms shift + survivors.sort(key=lambda entry: (entry[0], -entry[2], entry[1])) + kept = [] + tonal_spans = [] + checks = 0 + wanted = LOOPED_PER_CANDIDATE * max_candidates + for ripple, start, length, readings in survivors: + if len(kept) >= wanted: + break + end = start + length + if any(start < k_end and k_start < end for k_start, k_end, _ in kept): + continue + if any( + min(end, t_end) - max(start, t_start) > length / 2 + for t_start, t_end in tonal_spans + ): + rejected["tonal"] += 1 + continue + if checks >= MAX_TONAL_CHECKS: + break + checks += 1 + segment = searched[start * bin_length : end * bin_length][numpy.newaxis, :] + flatness = _spectral_flatness(segment) + harmonicity = _harmonicity(segment, sample_rate) + if flatness < TONAL_FLATNESS_THRESHOLD or harmonicity >= HARMONICITY_THRESHOLD: + rejected["tonal"] += 1 + tonal_spans.append((start, end)) + continue + readings = {**readings, "flatness": flatness, "harmonicity": harmonicity} + kept.append((start, end, readings)) + + # Steps 6 and 7: loop each, measure the bed, rank by its ripple + candidates = [] + for start, end, readings in kept: + segment = searched[start * bin_length : end * bin_length] + looped = _looped(segment, sample_rate, crossfade_ms, loop_seconds) + start_at = (offset + start * bin_length) / sample_rate + seconds = (end - start) * bin_length / sample_rate + gain_db = target_bed_dbfs - readings["mean_dbfs"] + warnings = [] + peak = looped["envelope_peak_db"] + if peak is not None and peak > LAP_MODULATION_WARN_DB: + warnings.append("lap_modulation") + candidates.append( + { + "start_seconds": round(start_at, 3), + "duration_seconds": round(seconds, 3), + "end_seconds": round(start_at + seconds, 3), + "shot": None, + "mean_dbfs": _round(readings["mean_dbfs"]), + "max_bin_dbfs": _round(readings["max_bin_dbfs"]), + "spike_db": _round(readings["spike_db"]), + "flatness": _round(readings["flatness"], 3), + "harmonicity": _round(readings["harmonicity"], 3), + "looped": looped, + "gain_db": _round(gain_db), + "gain": _round(10.0 ** (gain_db / 20.0), 3), + "warnings": warnings, + } + ) + candidates.sort(key=lambda c: (c["looped"]["ripple_db"], c["start_seconds"])) + candidates = [ + {"rank": rank, **candidate} + for rank, candidate in enumerate(candidates[:max_candidates], start=1) + ] + + criteria = { + "max_bin_dbfs": max_bin_dbfs, + "max_mean_dbfs": max_mean_dbfs, + "max_spike_db": max_spike_db, + "tonal_flatness": TONAL_FLATNESS_THRESHOLD, + "harmonicity": HARMONICITY_THRESHOLD, + "crossfade_ms": crossfade_ms, + "loop_seconds": loop_seconds, + "min_seconds": min_seconds, + "max_seconds": max_seconds, + "target_bed_dbfs": target_bed_dbfs, + } + findings = [] if candidates else [_empty_finding(rejected, criteria)] + logger.info( + f"{COMMAND}: {len(survivors)} windows passed level and tick tests, " + f"{len(kept)} looped, {len(candidates)} returned; rejected {rejected}" + ) + return { + "source": { + "duration_seconds": round(duration, 3), + "sample_rate": int(sample_rate), + "searched": [round(first, 3), round(last, 3)], + "shots_source": None, + }, + "criteria": criteria, + "candidates": candidates, + "rejected": rejected, + "findings": findings, + } diff --git a/dw/tasks/task.py b/dw/tasks/task.py index 0f6a075e..e95389cc 100644 --- a/dw/tasks/task.py +++ b/dw/tasks/task.py @@ -310,6 +310,17 @@ def _handle_loop_audio(task, arguments, previous_pipelines): return loop_audio(**arguments) +@register_command( + "find_loop_bed", implementation="dw.tasks.loop_bed.find_loop_bed", returns="json" +) +def _handle_find_loop_bed(task, arguments, previous_pipelines): + """Rank the quiet windows of a recording worth looping into a room-tone bed""" + logger.debug("Searching for a loop bed") + from .loop_bed import find_loop_bed + + return find_loop_bed(**arguments) + + @register_command( "stabilize_video", implementation="dw.tasks.stabilize.stabilize_video" ) From fd66a0d14ea5869e8e2e099565bc10ad4d8f3b49 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 20:39:10 -0500 Subject: [PATCH 021/251] fix(vram): #552 - check vram_estimate even when a workflow declares no cost vram_estimate_errors bailed out entirely when a workflow had no top-level cost array, so a caller-authored workflow that copied a template's vram_estimate but dropped its cost block got no VRAM check at all. Treat a missing/non-list cost as empty so _entries_for's device-capacity fallback still applies - the ceiling in that case is the device's own capacity, not a curated cost entry. Co-Authored-By: Claude Sonnet 5 --- dw/vram_estimate.py | 3 +-- tests/test_vram_estimate.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dw/vram_estimate.py b/dw/vram_estimate.py index 635b7165..17bb8cea 100644 --- a/dw/vram_estimate.py +++ b/dw/vram_estimate.py @@ -313,8 +313,7 @@ def vram_estimate_errors( if estimate is None: return [] cost = definition.get("cost") - if not isinstance(cost, list): - return [] + cost = cost if isinstance(cost, list) else [] entries = _entries_for(cost, device_type, capacity_gb) capacities = [ entry.get("vram_gb") for entry in entries if entry.get("vram_gb") is not None diff --git a/tests/test_vram_estimate.py b/tests/test_vram_estimate.py index 3d4d165c..0a32f4eb 100644 --- a/tests/test_vram_estimate.py +++ b/tests/test_vram_estimate.py @@ -71,6 +71,23 @@ def test_an_indexed_cost_device_matches_its_backend(): assert vram_estimate_errors(d, device_type="cuda", capacity_gb=24) == [] +def test_a_workflow_with_no_cost_block_still_checks_the_device_capacity(): + # #552: a caller-authored workflow that copies a template's vram_estimate + # but drops its cost block must not go unchecked - the ceiling here is + # the device's own capacity, not a curated cost entry. + d = definition() + del d["cost"] + errors = vram_estimate_errors(d, device_type="mps", capacity_gb=16) + assert len(errors) == 1 + assert "mps" in errors[0]["message"] + + +def test_a_workflow_with_no_cost_block_and_room_is_not_refused(): + d = definition() + del d["cost"] + assert vram_estimate_errors(d, device_type="mps", capacity_gb=62) == [] + + def test_run_time_backstop_takes_the_device_too(): d = definition() apply_vram_estimate(d, d["variables"], device_type="mps", capacity_gb=62) From 2a334ab2aaa5e22f857c49bc0082f6f96d7b229b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 20:53:33 -0500 Subject: [PATCH 022/251] test(audio): #544 - find_loop_bed rules, refusals, audio-only mp4 read, _harmonicity equality; docs Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 125 ++++++++- docs/WORKFLOW_GUIDE.md | 4 +- tests/test_find_loop_bed.py | 546 ++++++++++++++++++++++++++++++++++++ 3 files changed, 669 insertions(+), 6 deletions(-) create mode 100644 tests/test_find_loop_bed.py diff --git a/docs/TASKS.md b/docs/TASKS.md index c9b06324..b8b9c9c7 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -842,10 +842,127 @@ The bed is laid under the cut with `mix_audio` and attached to the picture with "result": { "content_type": "video/mp4", "fps": 24 } } ``` -Where the bed itself comes from is the open question: a few seconds of a -generated shot's own ambience, cut out with `slice_audio` from a stretch with -nothing tonal in it, is the material that matches — the room the shots were -generated in. +The bed's source: [`find_loop_bed`](#find_loop_bed) picks the stretch. Run it +against the cut (or a stem of it) over the range that should hold room tone, +copy the top candidate's `start_seconds`/`duration_seconds` into `slice_audio`, +`loop_audio` the slice to the cut's length, and `mix_audio` it in at the +candidate's `gain` — the room the shots were generated in, picked by +measurement rather than by ear. + +### find_loop_bed + +Pick the window of a recording worth looping into the room-tone bed above, +measured as it will sound looped rather than as it sits in the source. A +level check alone misses three things, each found on a real episode: + +- near-programme material — faint speech attenuated ~30 dB reads as quiet, + and is audible once it repeats every lap. +- lap-rate modulation — the loop's repeat rate beats against the source's own + level movement, invisible in one pass through the source. +- ticks — a 1 ms transient is invisible to 50 ms RMS and recurs once per lap. + +```json +{ + "task": { + "command": "find_loop_bed", + "arguments": { + "audio": "output:episode/latest/final/cut.mp4", + "start_seconds": 30.0, + "end_seconds": 90.0 + } + }, + "result": { "content_type": "application/json" } +} +``` + +| Argument | Required | Description | +| -------- | -------- | ----------- | +| `audio` | Yes | Path, `asset:`/`output:` reference of an audio or video file (a video is read audio-only - frames are never decoded), or a track or video from `previous_result:` | +| `start_seconds` / `end_seconds` | No | The range to search; the whole file if omitted | +| `min_seconds` / `max_seconds` | No | Shortest/longest window tried (default `0.5` / `2.0`) | +| `max_bin_dbfs` | No | Every 50 ms bin of a window must be at or below this (default `-55`) | +| `max_mean_dbfs` | No | A window's mean (RMS) level must be at or below this (default `-60`) | +| `max_spike_db` | No | Most a window's largest 1 ms peak may sit above its median 1 ms peak (default `12`) | +| `crossfade_ms` | No | The crossfade candidates are looped with - `loop_audio`'s own default, so what is measured is what `loop_audio` will make (default `250`) | +| `loop_seconds` | No | Length of the looped result that is measured (default `10.0`) | +| `target_bed_dbfs` | No | The level each candidate's `gain` is computed to reach (default `-60`) | +| `max_candidates` | No | How many ranked candidates to return (default `5`) | + +Every window on a 50 ms grid, from `min_seconds` to `max_seconds` long, is +kept only if it is quiet, not digital silence, free of ticks and not tonal or +speech-like (the same flatness/harmonicity test `bleed_join` uses). +Overlapping survivors are thinned to the steadiest, rejecting one mostly +inside a span already found tonal without re-measuring it. Each survivor is +then looped with `loop_audio`'s own crossfade to `loop_seconds` and measured: +`ripple_db` (the 5-95% spread of the looped 50 ms bins), `envelope_peak_db`/ +`envelope_peak_hz` (the strongest level wobble) and `lap_component_db` (the +wobble at the lap rate). Candidates are ranked by looped `ripple_db`, lowest +first; one whose `envelope_peak_db` is above -15 dB carries the +`lap_modulation` warning. + +```json +{ + "source": { "duration_seconds": 620.4, "sample_rate": 44100, "searched": [30.0, 90.0], "shots_source": null }, + "criteria": { "min_seconds": 0.5, "max_seconds": 2.0, "target_bed_dbfs": -60.0 }, + "candidates": [ + { + "rank": 1, + "start_seconds": 41.28, + "duration_seconds": 1.35, + "end_seconds": 42.63, + "shot": null, + "mean_dbfs": -63.1, + "max_bin_dbfs": -57.4, + "spike_db": 4.2, + "flatness": 0.61, + "harmonicity": 0.08, + "looped": { + "ripple_db": 1.1, + "envelope_peak_db": -24.0, + "envelope_peak_hz": 0.4, + "lap_hz": 0.096, + "lap_component_db": -26.0 + }, + "gain_db": 3.1, + "gain": 1.43, + "warnings": [] + } + ], + "rejected": { "too_loud": 812, "silent": 0, "spike": 14, "tonal": 203 }, + "findings": [] +} +``` + +`start_seconds`/`duration_seconds` are `slice_audio`'s arguments and `gain` +is `mix_audio`'s multiplier for reaching `target_bed_dbfs` — the answer picks +a window and builds nothing, so the remedy is a copy. Finding nothing is an +answer, not an error: `candidates` is `[]`, `rejected` counts the windows +each rule threw out, and one `no_loop_bed` finding says which rule to relax. + +Like the assessment probes it answers JSON and decides nothing, but it +searches rather than checking a finished cut, so it is not one of them — it +stays out of `list_tasks`' `assessment` list. The result must be saved as +`application/json`. + +```json +{ + "variables": { "audio": "output:episode/latest/final/cut.mp4" }, + "steps": [ + { "name": "bed", "task": { "command": "find_loop_bed", + "arguments": { "audio": "variable:audio" } }, + "result": { "content_type": "application/json" } } + ] +} +``` + +run against the finished cut once it exists, then read back over MCP with +`get_output_text` (JSON results are text) rather than `get_output_audio`. + +Out-of-range literals (`min_seconds`/`max_seconds`/`loop_seconds` at or below +zero, `max_candidates` at or below zero, a negative `crossfade_ms`, and the +like) are refused at validation. `end_seconds` at or before `start_seconds`, +`min_seconds` above `max_seconds`, a range past the end of the file, and a +source shorter than `min_seconds` fail at run time — they depend on the file. ### resample_audio diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index c7c2642f..75d4d0a5 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -1042,8 +1042,8 @@ Supported content types: `image/jpeg`, `image/png`, `image/webp`, `image/gif`, ` A task command's implementation declares what it hands back - most answer an `artifact` (a file `result` saves in one of the media content types above), some (`judge`) answer a bare `scalar` that cannot be saved at all, and some -(the assessment probes in [TASKS.md](TASKS.md), and `attribute_voices`) answer -a `json` document - every measurement taken, in one dict. A step on a `json` command must set +(the assessment probes in [TASKS.md](TASKS.md), `find_loop_bed`, and +`attribute_voices`) answer a `json` document - every measurement taken, in one dict. A step on a `json` command must set `content_type` to `application/json`, which saves it as one document; a step on a `scalar` command may not carry a `result` at all. Both are checked in validation, by the command's own declared kind rather than a name match. diff --git a/tests/test_find_loop_bed.py b/tests/test_find_loop_bed.py new file mode 100644 index 00000000..fd29cf8f --- /dev/null +++ b/tests/test_find_loop_bed.py @@ -0,0 +1,546 @@ +"""Tests for find_loop_bed (#218): ranking the quiet windows of a recording +worth looping into a room-tone bed, measured as they will sound looped. + +Fixtures are synthetic noise built with a seeded numpy.random.Generator at +48 kHz, so a run is deterministic; each dBFS target is converted to an +amplitude via 10**(dbfs/20) (root-mean-square for noise, amplitude/sqrt(2) +for a sine), matching the engine's own `10*log10(mean square)` bin reading. +""" + +import math + +import numpy +import pytest +import torch + +from dw.introspection import list_tasks +from dw.media_audio import decode_soundtrack +from dw.result import AudioTrack +from dw.scalar_result_validation import scalar_result_errors +from dw.task_domains import task_argument_errors +from dw.tasks.audio_utils import _PERIODICITY_MAX_HZ, _PERIODICITY_MIN_HZ, _harmonicity +from dw.tasks.loop_bed import find_loop_bed +from dw.tasks.task import task_command_info +from dw.workflow import Workflow + +SR = 48000 +BIN_SECONDS = 0.05 +BIN_LENGTH = 2400 # round(SR * BIN_SECONDS) + + +def rms_amplitude(dbfs): + """The RMS a noise segment needs to read `dbfs` on the engine's own + 10*log10(mean square) scale.""" + return 10.0 ** (dbfs / 20.0) + + +def track(waveform, sample_rate=SR): + return AudioTrack(waveform[None, :].astype(numpy.float32), sample_rate) + + +def noise(rng, num_samples, dbfs): + return rng.normal(0.0, rms_amplitude(dbfs), num_samples) + + +CANDIDATE_KEYS = { + "rank", + "start_seconds", + "duration_seconds", + "end_seconds", + "shot", + "mean_dbfs", + "max_bin_dbfs", + "spike_db", + "flatness", + "harmonicity", + "looped", + "gain_db", + "gain", + "warnings", +} +LOOPED_KEYS = { + "ripple_db", + "envelope_peak_db", + "envelope_peak_hz", + "lap_hz", + "lap_component_db", +} + + +class TestCleanNoiseWins: + """A quiet stretch inside a loud track is the only place a candidate can + come from, and the answer's shape is exactly what the plan promises.""" + + @pytest.fixture + def result(self): + rng = numpy.random.default_rng(1) + total = 12 * SR # bin-aligned: 240 bins + waveform = noise(rng, total, -50.0) + quiet_start, quiet_end = 3 * SR, 8 * SR # bins 60..160 + waveform[quiet_start:quiet_end] = noise(rng, quiet_end - quiet_start, -78.0) + return find_loop_bed(track(waveform)) + + def test_top_level_keys(self, result): + assert set(result.keys()) == { + "source", + "criteria", + "candidates", + "rejected", + "findings", + } + + def test_source_carries_no_shots(self, result): + assert result["source"]["shots_source"] is None + + def test_no_findings_when_something_was_found(self, result): + assert result["findings"] == [] + + def test_every_candidate_lies_inside_the_quiet_stretch(self, result): + candidates = result["candidates"] + assert candidates + for candidate in candidates: + assert candidate["start_seconds"] >= 3.0 - 1e-6 + assert candidate["end_seconds"] <= 8.0 + 1e-6 + + def test_candidates_are_sorted_by_looped_ripple_ascending(self, result): + ripples = [c["looped"]["ripple_db"] for c in result["candidates"]] + assert ripples == sorted(ripples) + + def test_ranks_are_1_through_n(self, result): + assert [c["rank"] for c in result["candidates"]] == list( + range(1, len(result["candidates"]) + 1) + ) + + def test_every_candidate_has_every_plan_key(self, result): + for candidate in result["candidates"]: + assert set(candidate.keys()) == CANDIDATE_KEYS + assert set(candidate["looped"].keys()) == LOOPED_KEYS + + def test_gain_matches_gain_db(self, result): + for candidate in result["candidates"]: + expected = 10.0 ** (candidate["gain_db"] / 20.0) + assert math.isclose(candidate["gain"], expected, rel_tol=1e-3) + + def test_gain_db_targets_the_default_bed_level(self, result): + for candidate in result["candidates"]: + expected = -60.0 - candidate["mean_dbfs"] + assert math.isclose(candidate["gain_db"], expected, abs_tol=0.05) + + +class TestATick: + """A single-sample spike is invisible to 50 ms RMS but not to the 1 ms + peak search, and only a window that covers it should be rejected.""" + + def _fixture(self): + # A loud (-50 dBFS) recording with one quiet (-78 dBFS) stretch from + # 5s to 15s: the loud material bounds how many non-overlapping quiet + # windows there are to tile around the tick with, so a window that + # actually covers the tick is eventually reached once the spike + # check stops rejecting it. The tick itself is loud enough to + # dominate its own 1 ms peak (spike_db well above the 12 dB default) + # but small enough - spread over one 2400 sample bin's mean square - + # that it barely moves that bin's own level. + rng = numpy.random.default_rng(2) + duration = 20 * SR + waveform = noise(rng, duration, -50.0) + quiet_start, quiet_end = 5 * SR, 15 * SR + waveform[quiet_start:quiet_end] = noise(rng, quiet_end - quiet_start, -78.0) + tick_sample = 10 * SR + waveform[tick_sample] = 10.0 * rms_amplitude(-78.0) + return waveform, tick_sample / SR + + def test_no_candidate_covers_the_tick(self): + waveform, tick_time = self._fixture() + result = find_loop_bed(track(waveform)) + + for candidate in result["candidates"]: + assert not ( + candidate["start_seconds"] <= tick_time < candidate["end_seconds"] + ) + assert result["rejected"]["spike"] >= 1 + + def test_raising_max_spike_db_lets_the_tick_survive(self): + waveform, tick_time = self._fixture() + result = find_loop_bed(track(waveform), max_spike_db=30.0, max_candidates=10) + + assert any( + candidate["start_seconds"] <= tick_time < candidate["end_seconds"] + for candidate in result["candidates"] + ) + assert result["rejected"]["spike"] == 0 + + def test_lowering_max_bin_dbfs_below_the_bed_gives_no_candidates(self): + waveform, _tick_time = self._fixture() + result = find_loop_bed(track(waveform), max_bin_dbfs=-90.0) + + assert result["candidates"] == [] + assert result["rejected"]["too_loud"] > 0 + + +class TestTonal: + """A faint pure tone is as quiet as the noise around it, but its + periodicity should still get it thrown out.""" + + def test_no_candidate_covers_the_tone(self): + rng = numpy.random.default_rng(3) + duration = 15 * SR + waveform = noise(rng, duration, -78.0) + + tone_start, tone_end = 6 * SR, 9 * SR + t = numpy.arange(tone_end - tone_start) / SR + amplitude = rms_amplitude(-78.0) * math.sqrt(2) + waveform[tone_start:tone_end] = amplitude * numpy.sin(2 * math.pi * 220.0 * t) + + result = find_loop_bed(track(waveform)) + + for candidate in result["candidates"]: + assert ( + candidate["end_seconds"] <= 6.0 + 1e-6 + or candidate["start_seconds"] >= 9.0 - 1e-6 + ) + assert result["rejected"]["tonal"] >= 1 + + +class TestLapModulation: + """A candidate whose level ramps across the window becomes a sawtooth + once looped, and that sawtooth should be visible in the measurement.""" + + @staticmethod + def _ramped(rng, num_bins=40, low_dbfs=-73.0, high_dbfs=-65.0): + waveform = numpy.empty(num_bins * BIN_LENGTH) + for i in range(num_bins): + dbfs = low_dbfs + (high_dbfs - low_dbfs) * i / (num_bins - 1) + waveform[i * BIN_LENGTH : (i + 1) * BIN_LENGTH] = noise( + rng, BIN_LENGTH, dbfs + ) + return waveform + + @staticmethod + def _flat(rng, num_bins=40, dbfs=-70.0): + return noise(rng, num_bins * BIN_LENGTH, dbfs) + + def test_a_ramp_reads_as_lap_modulation(self): + rng = numpy.random.default_rng(4) + waveform = self._ramped(rng) + + result = find_loop_bed(track(waveform), min_seconds=2.0, max_seconds=2.0) + + assert len(result["candidates"]) == 1 + looped = result["candidates"][0]["looped"] + assert looped["envelope_peak_db"] > -15.0 + assert "lap_modulation" in result["candidates"][0]["warnings"] + + def test_a_flat_stretch_does_not(self): + rng = numpy.random.default_rng(5) + waveform = self._flat(rng) + + result = find_loop_bed(track(waveform), min_seconds=2.0, max_seconds=2.0) + + assert len(result["candidates"]) == 1 + candidate = result["candidates"][0] + assert candidate["warnings"] == [] + + ramped = find_loop_bed( + track(self._ramped(numpy.random.default_rng(4))), + min_seconds=2.0, + max_seconds=2.0, + )["candidates"][0] + assert ( + candidate["looped"]["lap_component_db"] + < ramped["looped"]["lap_component_db"] + ) + + +class TestNoCandidates: + def test_an_all_loud_track_reports_why(self): + rng = numpy.random.default_rng(6) + waveform = noise(rng, 5 * SR, -20.0) + + result = find_loop_bed(track(waveform)) + + assert result["candidates"] == [] + assert result["rejected"]["too_loud"] > 0 + assert len(result["findings"]) == 1 + assert result["findings"][0]["rule"] == "no_loop_bed" + assert result["findings"][0]["rejected_by"] == "too_loud" + assert "max_bin_dbfs" in result["findings"][0]["message"] + + def test_digital_silence_is_not_a_bed(self): + waveform = numpy.zeros(5 * SR) + + result = find_loop_bed(track(waveform)) + + assert result["candidates"] == [] + assert result["rejected"]["silent"] > 0 + + +class TestCandidatesDoNotOverlap: + def test_pairwise_disjoint(self): + rng = numpy.random.default_rng(7) + waveform = noise(rng, 20 * SR, -78.0) + + result = find_loop_bed(track(waveform), max_candidates=5) + candidates = result["candidates"] + assert len(candidates) > 1 + + for i, a in enumerate(candidates): + for b in candidates[i + 1 :]: + assert ( + a["end_seconds"] <= b["start_seconds"] + or b["end_seconds"] <= a["start_seconds"] + ) + + +class TestRunTimeRefusals: + @staticmethod + def _quiet(seconds=5.0, seed=8, dbfs=-78.0): + rng = numpy.random.default_rng(seed) + return track(noise(rng, int(seconds * SR), dbfs)) + + def test_end_at_or_before_start(self): + with pytest.raises(ValueError, match="after 'start_seconds'"): + find_loop_bed(self._quiet(), start_seconds=3.0, end_seconds=2.0) + + def test_min_above_max(self): + with pytest.raises(ValueError, match="at or below 'max_seconds'"): + find_loop_bed(self._quiet(), min_seconds=2.0, max_seconds=1.0) + + def test_start_past_the_end(self): + with pytest.raises(ValueError, match="'start_seconds'.*past the end"): + find_loop_bed(self._quiet(seconds=2.0), start_seconds=5.0) + + def test_end_past_the_end(self): + with pytest.raises(ValueError, match="'end_seconds'.*past the end"): + find_loop_bed(self._quiet(seconds=2.0), end_seconds=5.0) + + def test_source_shorter_than_min_seconds(self): + with pytest.raises(ValueError, match="shorter than 'min_seconds'"): + find_loop_bed(self._quiet(seconds=0.2), min_seconds=0.5) + + def test_range_shorter_than_min_seconds(self): + with pytest.raises(ValueError, match="the range .* is shorter than"): + find_loop_bed( + self._quiet(seconds=5.0), + start_seconds=1.0, + end_seconds=1.2, + min_seconds=0.5, + ) + + def test_min_seconds_zero_is_a_domain_violation(self): + with pytest.raises(ValueError, match="min_seconds"): + find_loop_bed(self._quiet(), min_seconds=0) + + def test_max_candidates_zero_is_a_domain_violation(self): + with pytest.raises(ValueError, match="max_candidates"): + find_loop_bed(self._quiet(), max_candidates=0) + + def test_negative_crossfade_ms_is_a_domain_violation(self): + with pytest.raises(ValueError, match="crossfade_ms"): + find_loop_bed(self._quiet(), crossfade_ms=-1) + + def test_string_numerics_are_coerced(self): + result = find_loop_bed(self._quiet(), min_seconds="0.5") + + assert "candidates" in result + + +def _write_video(path, num_frames=8, fps=4, sample_rate=8000, audio=True): + from diffusers.utils.export_utils import encode_video + from PIL import Image + + frames = [Image.new("RGB", (16, 16), (i, 0, 0)) for i in range(num_frames)] + clip = torch.full((2, int(num_frames / fps * sample_rate)), 0.25) if audio else None + encode_video( + frames, + fps=fps, + output_path=str(path), + audio=clip, + audio_sample_rate=sample_rate if audio else None, + ) + return str(path) + + +class TestVideoSource: + """A video file's soundtrack is read without decoding its pictures.""" + + def test_an_mp4_soundtrack_is_read_without_the_frame_decoder( + self, tmp_path, monkeypatch + ): + def _explode(*args, **kwargs): + raise AssertionError("load_audio_video should not run for find_loop_bed") + + monkeypatch.setattr("dw.tasks.video_utils.load_audio_video", _explode) + path = _write_video(tmp_path / "cut.mp4") + + result = find_loop_bed(path) + + assert "candidates" in result + + def test_a_silent_video_is_an_error(self, tmp_path): + path = _write_video(tmp_path / "mute.mp4", audio=False) + + with pytest.raises(ValueError, match="no audio track"): + find_loop_bed(path) + + +class TestDecodeSoundtrack: + def test_a_wav(self, tmp_path): + import soundfile + + path = tmp_path / "tone.wav" + soundfile.write(path, numpy.zeros((100, 2), dtype=numpy.float32), 16000) + + waveform, sample_rate = decode_soundtrack(str(path)) + + assert waveform.shape == (2, 100) + assert waveform.dtype == numpy.float32 + assert sample_rate == 16000 + + def test_an_mp4(self, tmp_path): + path = _write_video(tmp_path / "cut.mp4") + + waveform, sample_rate = decode_soundtrack(path) + + assert waveform.shape[0] == 2 + assert waveform.dtype == numpy.float32 + assert sample_rate == 8000 + + +def _old_harmonicity(waveform, sample_rate): + """The pre-#218 O(n^2) implementation, kept here only to pin the FFT + replacement's numbers against it.""" + min_lag = max(int(sample_rate / _PERIODICITY_MAX_HZ), 1) + max_lag = min(int(sample_rate / _PERIODICITY_MIN_HZ), waveform.shape[1] - 1) + if max_lag <= min_lag: + return 0.0 + + scores = [] + for channel in waveform: + centered = channel - channel.mean() + energy = float(numpy.dot(centered, centered)) + if energy <= 1e-12: + continue + n = centered.shape[0] + correlation = numpy.correlate(centered, centered, "full") + zero_lag = n - 1 + window = correlation[zero_lag + min_lag : zero_lag + max_lag + 1] + if window.size == 0: + continue + scores.append(float(numpy.max(window) / energy)) + return max(scores) if scores else 0.0 + + +class TestHarmonicityMatchesTheOldImplementation: + def _assert_matches(self, waveform, sample_rate=SR): + old = _old_harmonicity(waveform, sample_rate) + new = _harmonicity(waveform, sample_rate) + assert numpy.isclose(old, new, rtol=1e-9, atol=1e-12) + + def test_noise(self): + rng = numpy.random.default_rng(9) + self._assert_matches(rng.normal(0, 1.0, (1, 4096))) + + def test_a_sine_tone(self): + t = numpy.arange(4096) / SR + waveform = numpy.sin(2 * math.pi * 220.0 * t)[numpy.newaxis, :] + self._assert_matches(waveform) + + def test_two_channels_with_different_content(self): + rng = numpy.random.default_rng(10) + t = numpy.arange(4096) / SR + tone = numpy.sin(2 * math.pi * 300.0 * t) + waveform = numpy.stack([tone, rng.normal(0, 1.0, 4096)]) + self._assert_matches(waveform) + + def test_a_very_short_waveform(self): + rng = numpy.random.default_rng(11) + self._assert_matches(rng.normal(0, 1.0, (1, 50))) + + def test_silence(self): + self._assert_matches(numpy.zeros((1, 4096))) + + +class TestRegistry: + def test_returns_json(self): + assert task_command_info("find_loop_bed")["returns"] == "json" + + def test_is_a_registered_command(self): + assert "find_loop_bed" in list_tasks()["commands"] + + def test_is_not_an_assessment_probe(self): + assert "find_loop_bed" not in list_tasks()["assessment"] + + +def _step(name, command, result=None, arguments=None): + step = {"name": name, "task": {"command": command, "arguments": arguments or {}}} + if result is not None: + step["result"] = result + return step + + +class TestContentType: + def test_application_json_is_fine(self): + definition = { + "steps": [ + _step( + "bed", + "find_loop_bed", + result={"content_type": "application/json"}, + arguments={"audio": "asset:bed.wav"}, + ) + ] + } + + assert scalar_result_errors(definition, source_indices=[0]) == [] + + def test_audio_wav_is_an_error_naming_application_json(self): + definition = { + "steps": [ + _step( + "bed", + "find_loop_bed", + result={"content_type": "audio/wav"}, + arguments={"audio": "asset:bed.wav"}, + ) + ] + } + + errors = scalar_result_errors(definition, source_indices=[0]) + + assert len(errors) == 1 + assert errors[0]["path"] == "steps[0].result" + assert "application/json" in errors[0]["message"] + assert "find_loop_bed" in errors[0]["message"] + + +class TestDomainInAWorkflow: + def _definition(self, arguments): + return { + "id": "loop-bed", + "steps": [ + _step( + "bed", + "find_loop_bed", + result={"content_type": "application/json"}, + arguments=arguments, + ) + ], + } + + def test_task_argument_errors_reports_a_literal_min_seconds_zero(self): + errors = task_argument_errors( + self._definition({"audio": "asset:bed.wav", "min_seconds": 0}) + ) + + assert [e["path"] for e in errors] == ["steps[0].task.arguments.min_seconds"] + + def test_validation_errors_reports_it_too(self): + workflow = Workflow( + self._definition({"audio": "asset:bed.wav", "min_seconds": 0}), + "outputs", + None, + ) + + errors = workflow.validation_errors() + + assert any(e["path"] == "steps[0].task.arguments.min_seconds" for e in errors) From 52706b1bb9807f3946af2f3f5c4cae7e7bb31410 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 21:02:14 -0500 Subject: [PATCH 023/251] docs(proposals): #486 - join_into_song design record Co-Authored-By: Claude Opus 5.5 --- .../complete/join-into-song-complete.md | 157 ++++++++++++++++++ docs/proposals/todo.md | 4 + 2 files changed, 161 insertions(+) create mode 100644 docs/proposals/complete/join-into-song-complete.md diff --git a/docs/proposals/complete/join-into-song-complete.md b/docs/proposals/complete/join-into-song-complete.md new file mode 100644 index 00000000..18aaf396 --- /dev/null +++ b/docs/proposals/complete/join-into-song-complete.md @@ -0,0 +1,157 @@ +# `join_into_song`: a spoken scene breaking into a song (#486) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out session, +2026-09-27). Designed on #486 from a field report (*La Vela del Pueblo*: 6 +dialogue shots breaking into 18 lip-synced song shots, assembled locally in +~130 lines of ffmpeg). Two design sessions each posted a plan v1; Don chose +the 23:54 one ("B"), approved as **build smaller** with its defaults to Q1-Q4. +Built as stages #513 (A) and #514 (B). + +## The problem + +A spoken scene - shots that carry their own dialogue audio - has to break +into a song whose shots were lip-synced against frame-exact `slice_audio` +slices of one track: +- each dialogue shot loudness-matched; +- the song entering under the last spoken line, so that song time + `cue_seconds` lands exactly on the first song shot's frame 0; +- the dialogue ducked after a delay; +- the song shots playing over the **unbroken** song, their own audio dropped. + +`assemble-and-score` lays one score under everything, and composing the join +from existing tasks was possible but not viable: `mix_audio` has no per-track +offset, `gain_audio` has no ramp, and the song offset (`dialogue length - +cue`) would be a hand-computed literal that goes stale the moment a dialogue +shot is regenerated. The second half of the report was provenance: the film +built outside dw was not in the gallery, not reachable by `export_job`, and +had no record of its inputs. + +## Verdict and what was decided + +**Build smaller**: one bespoke task that owns the timing arithmetic. +- Q1: the name is `join_into_song`. +- Q2: a bespoke task rather than generic primitives (`mix_audio.offsets` + + a `gain_audio` ramp + a silence task), because only a task can measure the + dialogue length at run time. +- Q3: the task does not normalize its output; `normalize_audio` downstream + decides the level, as in `music-video`. +- Q4: the sweep findings are filed separately (see *Deferred*). +- Cut: a "register external output" path. A task's output already gets the + manifest, the realized `workflow.json` (which records every input + reference) and `export_job`, so doing the join in dw closes the provenance + half too. +- Cut: a template. The inputs are finished shots from earlier runs; the join + is one inline step. + +## What was built + +### Stage A (#513): the task + +`join_into_song(dialogue, song_shots, song, cue_seconds=0, +dialogue_target_lufs=None, duck_delay_ms=0, duck_db=-12, duck_ramp_ms=250, +fps=None)` in `dw/tasks/join_into_song.py`, registered in `dw/tasks/task.py`. +- **Picture:** the dialogue frames, then the song-shot frames. A mismatched + frame size or fps is refused, naming each input and its rate. +- **Placement:** each dialogue shot's track is fitted to its own frame span + (trimmed or padded; a fit of a frame or more warns + `dialogue_fitted_to_frames`), so `D` is the sample the first song-shot + frame sits at. The song goes in at `D - round(cue_seconds * sr)` and runs + to the end of the picture. A cue longer than the dialogue is a directed + error naming both. A song too short is padded with silence and warns + `song_short`. +- **Loudness:** `dialogue_target_lufs` applies one static gain per shot, + measured then applied (the report's ffmpeg `loudnorm` trap avoided). A + shot under 400 ms or unmeasurable keeps its level and warns + `dialogue_unmatched`. A silent dialogue shot becomes silence of its own + length, so nothing after it shifts. +- **Duck:** from `song_entry + duck_delay_ms`, a linear ramp of + `duck_ramp_ms` down to `duck_db`. +- **Channels:** the output takes the song's channel count; mono dialogue is + spread, wider dialogue cut down. +- **Shots:** one record per input, named `shot@` from `dialogue` then + `song_shots` references (`dw.shots.shot_references`, which `dw/workflow.py` + now uses instead of reading `videos` alone), samples measured off the built + waveform, `hard_cut` on every shot after the first. `EXPECTED_SITES` entry + "populates". +- **Domains** (`dw/task_domains.py`): `cue_seconds`, `duck_delay_ms`, + `duck_ramp_ms` non-negative, `duck_db` non-positive, `fps` positive. Empty + lists are refused at run time. +- **Tests:** `tests/test_join_into_song.py` on synthetic clips. Docs: a + `docs/TASKS.md` section. + +Deviations from the plan, all recorded on #513: +1. **An optional `fps`.** Pipeline frames carry no rate, and the song is + placed in time, so a join of pipeline results needs one. It is refused + when it contradicts a rate the videos carry. +2. **`D` measured per shot, fitted to frames**, not off the joined dialogue + track: otherwise a dialogue track overrunning its frames would push the + song off the song shots' picture. +3. **Shot naming** generalized through `shot_references`. +4. **Channel handling**, which the plan left open. + +The bounce fix reached the engine: `Result.save` (`dw/result.py`, +`save_audio_video`) now writes a *declared* `result.fps` back onto the +in-memory artifact. Before, a fresh run handed the next step the pre-save +rate while a cache hit or `output:` reload read the written one, so +`concat_videos`/`dissolve_videos` could silently re-time a re-rated shot. +They now refuse the mismatch, as this task does. + +### Stage B (#514): the recipe + +- `plugins/dw/skills/minimax-h3/SKILL.md`: a "Which shape" bullet, + **Dialogue into a song** - `slice_audio` (first slice at `cue_seconds`, + each next where the last ended) -> `join_into_song` (same `cue_seconds`) + -> `normalize_audio` at -3 dBFS -> `pair_audio`. +- `docs/WORKFLOW_GUIDE.md`: `### A spoken scene breaking into a song`, with + the slice arithmetic (`start_seconds = cue_seconds + earlier sung shots' + num_frames / fps`) and a JSON tail that `tests/test_plugin_skills.py` + validates. +- Deviation: the full recipe lives in the guide, the skill carries it in + brief. The H3 skill is at its 12 KB limit (12280/12288 bytes); room was made + by moving and rewording, with no rule dropped. + +## Measured on the fixture + +Recipe followed verbatim (C-F158, job `115b1c8941fd`) with the C-F152 +fixtures and a -60 dB marker hole at song time 3.0 s, `cue_seconds: 3.0`: +- the hole lands on the seam (frames [249,259) at -79.2 dBFS); +- the song is audible under the last line (15 dB above the dialogue's own + level there); +- 496 frames, 4 shots at [0,124) [124,248) [248,372) [372,496); +- the film peaks at -3.07 dBFS (true peak -3.05) with no `audio_clipped` or + `audio_no_headroom`; +- `analyze_sync_drift`: no findings, max offset -0.66 ms. + +Stage A's C-F153 measured a -12.03 dB duck and a monotone ramp; C-F154 +landed two shots 11 LU apart at -22.98 and -22.99 LUFS. + +## Deferred, and why + +- **The plan sweep's two latent bugs** (Q4): `concat_videos` skips a silent + input's audio rather than filling it with silence, so every later shot's + audio lands early; and `pair_audio` loads a string `audio` but not a string + `video`. Filed as one bug for the implementer at close-out (#553). The third + finding (`docs/TASKS.md` missing `normalize_audio`'s `target_lufs`) was + fixed by #474. +- **Naming a re-rated input in the fps refusal.** The error calls a + `previous_result:` video `video 3` where the shot map calls it after its + step (tester's nit on #513). Not filed. +- **Song into dialogue, more than one song, a dissolve at the seam, a + limiter**: non-goals. A limiter now exists (`normalize_audio(limit=true)`, + #474) and is the downstream step's to use. +- **A template and register-external-output**: cut (above). A second musical + field report asking for the shape, or a second deliverable assembled + outside dw, would bring either back. + +## Cost and bounces + +The stage comments record no usage figures, so no cost is given here. The +plan estimated about $5 for stage A and $2 for stage B. + +| stage | bounces | why | +|---|---|---| +| #513 | 1 | C-F156 arm 5: a 12 fps `previous_result:` shot was joined as 24 fps. The cause was the engine: `Result.save` didn't write a declared `result.fps` back onto the artifact. Fixed there; an `fps` contradicting the carried rate is refused too. | +| #514 | 0 | | + +Cases: C-F151-C-F157 (stage A), C-F158 (stage B). C-F151 has a suite +amendment pending on harnest for the added `fps` parameter. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 8fad1581..79fe3f49 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -60,6 +60,10 @@ remaining deferred fix is recorded in (holding the ceiling on the encoded film, true-peak in the post-encode check, `compress_audio`'s limit mode): `complete/normalize-audio-limiter-complete.md`. +- **`join_into_song`, a spoken scene breaking into a song** (#486, stages + #513-#514), shipped 2026-09-27. Record, including what was deferred (a + template, register-external-output, the `concat_videos` silent-input + desync filed as #553): `complete/join-into-song-complete.md`. ## Declined From 16326b755e5e2f5688ddcdb8a84ae1eb8347302e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 21:26:30 -0500 Subject: [PATCH 024/251] fix(mcp): #550 #551 - name the video-fit route in image/join error text #550: the run-time "Image file extension not allowed" error from load_image_arg now tells the caller a video must go through video_frames first, and TASKS.md's "Videos in image tasks" section says the same about a file path/asset:/output: reference. No validate rule, per Don's comment on #550 declining a general file-type validation framework. #551: the served dissolve_videos/concat_videos size-mismatch message now names the fix route (video_frames -> resize_rescale(width=, height=) -> pair_audio(fit="video")) instead of leaving it undiscoverable in a module docstring. Dropped resize_center_crop from that route everywhere it was named as an equal alternative to resize_rescale (the two docstrings, TASKS.md's two videos rows, and the module docstring in video_size_errors.py) since it squares the frame before stretching, which distorts a non-square join target - the exact trap #551 asked not to walk a reader into. Co-Authored-By: Claude Sonnet 5 --- docs/TASKS.md | 9 +++++++-- dw/arguments.py | 5 ++++- dw/tasks/concat_videos.py | 7 ++++--- dw/tasks/dissolve_videos.py | 7 ++++--- dw/video_size_errors.py | 16 +++++++++++----- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index b8b9c9c7..47adba77 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -212,7 +212,7 @@ video generation" in the workflow guide): | Argument | Required | Description | | -------- | -------- | ----------- | -| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, which is read with the audio muxed into it; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, this task does not resize one for you, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518). To fit the odd video: `video_frames` to get its frames, `resize_rescale`/`resize_center_crop` to the target size, then `pair_audio(fit="video")` to put its soundtrack back before passing it here (#551) | +| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, which is read with the audio muxed into it; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, this task does not resize one for you, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518). To fit the odd video: `video_frames` to get its frames, `resize_rescale` to the target size (`resize_center_crop` squares the frame first and then stretches it, distorting a non-square target), then `pair_audio(fit="video")` to put its soundtrack back before passing it here (#551) | | `trim_frames` | No | Frames dropped from the head of every video after the first (default: 0) | | `crossfade_ms` | No | Equal-power crossfade at each audio seam, drawn from the trimmed material - no effect when `trim_frames` is 0, and validation warns when one is written there (default: 75) | | `audio_bleed_ms` | No | How long the outgoing video's tail rings on over the head of the next one, at seams with nothing trimmed to crossfade (default: 0, off) | @@ -358,7 +358,7 @@ montage cut to a score wants: | Argument | Required | Description | | -------- | -------- | ----------- | -| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, one entry per video as with `concat_videos`; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, this task does not resize one for you, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518). To fit the odd video: `video_frames` to get its frames, `resize_rescale`/`resize_center_crop` to the target size, then `pair_audio(fit="video")` to put its soundtrack back before passing it here (#551) | +| `videos` | Yes | The videos to join, in order - `previous_result` references, or the path or URL of a video file an earlier run wrote, one entry per video as with `concat_videos`; an entry may also be a `{"location": ...}` dict wrapping either. Every video must be the same frame size - unlike a sample-rate mismatch, this task does not resize one for you, so a statically-resolvable (`asset:`/`output:`/literal path, wrapped in a `{"location": ...}` dict or not) size disagreement is refused at validate; a `previous_result:` or other reference not yet resolved still fails only at run time (#504, #518). To fit the odd video: `video_frames` to get its frames, `resize_rescale` to the target size (`resize_center_crop` squares the frame first and then stretches it, distorting a non-square target), then `pair_audio(fit="video")` to put its soundtrack back before passing it here (#551) | | `dissolve_frames` | No | Frames of overlap at each seam, blended linearly (default: 12). 0 is a hard cut | | `fade_in_frames` | No | Frames over which the first video rises out of `fade_color` (default: 0) | | `fade_out_frames` | No | Frames over which the last video sinks into it (default: 0) | @@ -1388,6 +1388,11 @@ can be upscaled without losing what was generated alongside it: Captioning (`image_to_text`) is the exception - describe a frame, taken with `get_first_frame`, rather than a video. +A file path or `asset:`/`output:` reference to a video is not accepted here, +even though the route above takes a video from a step - route it through +`video_frames` first: an image command's `image` argument must be an +`AudioVideo`, a frame array, or a still image. + ## Image Upscaling Upscale images using spandrel-compatible super-resolution models (ESRGAN, SwinIR, HAT, DAT, and 40+ other architectures). Models are auto-detected from weight files. diff --git a/dw/arguments.py b/dw/arguments.py index d7667fa8..2ab69662 100644 --- a/dw/arguments.py +++ b/dw/arguments.py @@ -1025,7 +1025,10 @@ def fetch_image(img_spec, base_dir=None): # Validate file extension ext = os.path.splitext(validated_path)[1].lower() if ext not in ALLOWED_IMAGE_EXTENSIONS: - raise SecurityError(f"Image file extension not allowed: {ext}") + raise SecurityError( + f"Image file extension not allowed: {ext} - a video file " + "must go through video_frames first" + ) return load_image(validated_path) except SecurityError: diff --git a/dw/tasks/concat_videos.py b/dw/tasks/concat_videos.py index cbeda36f..b49dd94d 100644 --- a/dw/tasks/concat_videos.py +++ b/dw/tasks/concat_videos.py @@ -83,9 +83,10 @@ def concat_videos( statically-resolvable (asset:/output:/literal path) size disagreement is refused at validate; one only known at run time still fails there (#504). To fit the odd video: `video_frames` - to get its frames, `resize_rescale`/`resize_center_crop` to the - target size, then `pair_audio(fit="video")` to put its - soundtrack back before passing it here (#551) + to get its frames, `resize_rescale` to the target size + (`resize_center_crop` squares the frame first and then stretches + it, distorting a non-square target), then `pair_audio(fit="video")` + to put its soundtrack back before passing it here (#551) trim_frames: Frames dropped from the head of every video after the first - the trim used when each video was generated from the previous one's last frame diff --git a/dw/tasks/dissolve_videos.py b/dw/tasks/dissolve_videos.py index 1eb93a84..30b4d213 100644 --- a/dw/tasks/dissolve_videos.py +++ b/dw/tasks/dissolve_videos.py @@ -63,9 +63,10 @@ def dissolve_videos( (asset:/output:/literal path) size disagreement is refused at validate; one only known at run time still fails there (#504). To fit the odd video: `video_frames` to get its frames, - `resize_rescale`/`resize_center_crop` to the target size, then - `pair_audio(fit="video")` to put its soundtrack back before - passing it here (#551). Soundtracks at different sample rates + `resize_rescale` to the target size (`resize_center_crop` squares + the frame first and then stretches it, distorting a non-square + target), then `pair_audio(fit="video")` to put its soundtrack + back before passing it here (#551). Soundtracks at different sample rates are not a constraint - see `sample_rate` below dissolve_frames: Frames of overlap at each seam. 0 is a hard cut fade_in_frames: Frames over which the first video rises out of diff --git a/dw/video_size_errors.py b/dw/video_size_errors.py index 71d4bd90..59a2c597 100644 --- a/dw/video_size_errors.py +++ b/dw/video_size_errors.py @@ -7,10 +7,13 @@ auto-resampled with a warning, neither join task resizes a mismatched video for you - Don declined a resize/fit argument on the join tasks themselves (#504, #512) - so the fix is upstream of the join: `video_frames` to get the -odd video's frames, `resize_rescale`/`resize_center_crop` to the target size, -then `pair_audio(fit="video")` to put its soundtrack back before passing it -to `dissolve_videos`/`concat_videos` (#551). The only thing to move earlier -is the refusal itself. +odd video's frames, `resize_rescale` to the target size (`resize_center_crop` +squares the frame first and then stretches it, distorting a non-square +target), then `pair_audio(fit="video")` to put its soundtrack back before +passing it to `dissolve_videos`/`concat_videos` (#551). The served error +message names the route too, since a consumer runs into this at validate, +not by reading this module. The only thing to move earlier is the refusal +itself. Moved here, into `validation_errors`, for exactly the cases a size is knowable without running anything: an `asset:`/`output:` reference, a @@ -102,11 +105,14 @@ def video_size_errors(workflow_definition, source_indices=None, base_dir=None): if isinstance(name, str) and MEMBER_SEPARATOR in name else "" ) + fit_width, fit_height = first_size errors.append( { "path": render_path(("steps", source, "task", "arguments", "videos")), "message": f"{command} needs every video at one size: " - f"{', '.join(problems)}{where}", + f"{', '.join(problems)}{where} - fit the odd one with " + f"video_frames → resize_rescale(width={fit_width}, " + f'height={fit_height}) → pair_audio(fit="video")', } ) return errors From 3ca4de36cc03e350778969bfcac38caef40bfdea Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 21:44:23 -0500 Subject: [PATCH 025/251] fix(mcp): #553 - fill silent concat_videos inputs, load a string pair_audio video concat_videos was skipping a silent input among videos where at least one other carried audio, contributing zero duration to the joined track and shifting every later shot's audio early. It now fills that input's own length with silence, matching join_into_song's rule (#513) that a shot with no track is silence for its length rather than a gap. Requires fps to size the silence, as every other length-dependent path here already does; every catalog template calling concat_videos already passes it. When no input carries audio, output audio still stays None. pair_audio already loaded a string 'audio' path via load_audio, but not a string 'video' - it now loads a video path/URL/{"location": ...} dict the same way concat_videos loads one of its own inputs, confined the same way audio's own path already is. Co-Authored-By: Claude Sonnet 5 --- dw/tasks/concat_videos.py | 42 ++++++++++++++++++++++--- dw/tasks/pair_audio.py | 17 ++++++++-- tests/test_concat_videos.py | 62 +++++++++++++++++++++++++++++++++++-- tests/test_pair_audio.py | 39 +++++++++++++++++++++++ 4 files changed, 150 insertions(+), 10 deletions(-) diff --git a/dw/tasks/concat_videos.py b/dw/tasks/concat_videos.py index b49dd94d..c4d6f879 100644 --- a/dw/tasks/concat_videos.py +++ b/dw/tasks/concat_videos.py @@ -11,6 +11,8 @@ import logging import os +import numpy + from ..events import emit_log, emit_warning from ..result import AudioVideo from ..shots import measured_num_samples, nested_shots, shot_record, trimmed_shots @@ -226,6 +228,18 @@ def concat_videos( # long shows up here as the samples it actually took (#378) shots = [] + # Whether the joined track carries anything at all - decides whether a + # silent input among these gets silence of its own length or is skipped + # outright, matching join_into_song's rule that a shot with no track is + # silence for its length rather than a gap that shifts everything after + # it early (#513, #553) + has_audio = any(waveform is not None for waveform in waveforms) + silence_channels = ( + next(waveform.shape[0] for waveform in waveforms if waveform is not None) + if has_audio + else None + ) + for index, (video, clip) in enumerate(zip(videos, clips)): head_trim = trim_frames if index > 0 else 0 start_frame = len(frames) @@ -236,7 +250,7 @@ def concat_videos( video_shots = nested_shots( trimmed_shots(inner, head_trim), start_frame, - start_sample if waveforms[index] is not None else None, + start_sample if has_audio else None, getattr(video, "sample_rate", None), sample_rate, ) @@ -258,13 +272,31 @@ def concat_videos( shot["source_index"] = index shots.extend(video_shots) - if waveforms[index] is None: + if not has_audio: continue - waveform = waveforms[index] + if waveforms[index] is None: + if fps is None: + raise ValueError( + "concat_videos needs 'fps' to fill silence for a video " + "with no audio track of its own" + ) + waveform = numpy.zeros( + (silence_channels, frames_to_samples(len(clip), fps, sample_rate)), + dtype=numpy.float32, + ) + emit_log( + f"concat_videos: {names[index]} carries no audio - filled with " + "silence", + command="concat_videos", + video=names[index], + ) + else: + waveform = waveforms[index] + if audio is None: audio = waveform - audio_native_rate = video.sample_rate + audio_native_rate = getattr(video, "sample_rate", None) continue if head_trim > 0 and fps is None: @@ -294,7 +326,7 @@ def concat_videos( crossfade_ms, seam_fade_ms, ) - audio_native_rate = video.sample_rate + audio_native_rate = getattr(video, "sample_rate", None) # The rate the caller declared, else the rate the first input carries - # either beats the result's 8 fps default (#84) diff --git a/dw/tasks/pair_audio.py b/dw/tasks/pair_audio.py index 44579d4f..623ba3f1 100644 --- a/dw/tasks/pair_audio.py +++ b/dw/tasks/pair_audio.py @@ -214,8 +214,13 @@ def pair_audio(video, audio, sample_rate=None, fps=None, fit=None): Args: video: The frames - a frame list, a frame array or tensor (or a - pipeline's batch of one of those, unwrapped), or an - AudioVideo whose own soundtrack is replaced by this one; the + pipeline's batch of one of those, unwrapped), an AudioVideo whose + own soundtrack is replaced by this one, or the path or URL of a + video file (or a {"location": ...} dict wrapping either - the + same idiom `get_last_frame(video=...)` and a pipeline's `image` + argument accept, #510) - loaded the same way `concat_videos` + loads one of its inputs, confined the same way `audio`'s own + path is (#553); the frames' own rate is carried through to the output, so set `result.fps` only to override it (a loaded file brings its rate along; frames that carry none are written at 8 fps) @@ -268,6 +273,14 @@ def pair_audio(video, audio, sample_rate=None, fps=None, fit=None): audio = _Loaded(*load_audio(audio)) waveform = getattr(audio, "audio", audio) + + from .video_utils import is_video_location, load_audio_video + + if is_video_location(video): + # A path/URL naming a video an earlier run wrote - confined the same + # way concat_videos loads one of its own inputs (#553) + video = load_audio_video(video) + if waveform is None: raise ValueError( "pair_audio needs an audio track - the video it was given carries none" diff --git a/tests/test_concat_videos.py b/tests/test_concat_videos.py index 6c70889a..9293dfa8 100644 --- a/tests/test_concat_videos.py +++ b/tests/test_concat_videos.py @@ -66,12 +66,15 @@ def test_untrimmed_audio_concatenates_whole(self): assert result.audio.shape == (2, 400) def test_mixed_inputs_keep_the_audio_that_exists(self): + # #553: the silent first video now fills its own length with + # silence rather than being skipped, which is why it needs fps - + # the same requirement trimming or crossfading audio already has videos = [frames(4), audio_video(8, 1)] - result = concat_videos(videos) + result = concat_videos(videos, fps=4) assert len(result.frames) == 12 - assert result.audio.shape == (2, 200) + assert result.audio.shape == (2, 100 + 200) def test_mismatched_sample_rates_are_resampled_to_the_highest(self, caplog): """#108: a 24 kHz voice clip paired onto a 32 kHz generation used to @@ -417,6 +420,56 @@ def test_a_location_dict_is_loaded_with_its_audio(self, tmp_path): assert result.audio.shape == (2, 8 / 4 * 8000) +class TestSilentInputAmongSoundedOnes: + """#553: a silent input contributed zero duration to the joined track, + shifting every later shot's audio early instead of the gap it actually + left in the picture.""" + + def test_a_silent_middle_shot_does_not_shift_the_third_shots_audio(self): + sound, silent, more_sound = ( + audio_video(8, 0.5), + frames(8), + audio_video(8, 0.25), + ) + + result = concat_videos([sound, silent, more_sound], fps=4) + + # Each shot is 8 frames @ 4fps @ 100Hz = 200 samples; the silent shot + # must still occupy its own 200 samples rather than contributing none + assert result.audio.shape == (2, 600) + third = result.shots[-1] + assert third["start_sample"] == 400 + assert third["start_frame"] == 16 + + def test_shots_stay_contiguous_across_the_silent_gap(self): + sound, silent, more_sound = ( + audio_video(8, 0.5), + frames(8), + audio_video(8, 0.25), + ) + + result = concat_videos([sound, silent, more_sound], fps=4) + + for earlier, later in zip(result.shots, result.shots[1:]): + assert ( + earlier["start_sample"] + earlier["num_samples"] + == later["start_sample"] + ) + + def test_a_silent_first_shot_is_also_filled(self): + silent, sound = frames(8), audio_video(8, 0.5) + + result = concat_videos([silent, sound], fps=4) + + assert result.audio.shape == (2, 400) + assert result.shots[-1]["start_sample"] == 200 + + def test_no_input_carrying_audio_still_leaves_audio_none(self): + result = concat_videos([frames(4), frames(4)], fps=4) + + assert result.audio is None + + class TestLevelMatching: """Independently generated shots land at whatever level the model chose, and cutting a -2.6 dBFS shot against a -12.6 dBFS one is audible as a @@ -470,7 +523,10 @@ def test_a_silent_shot_is_left_alone(self): assert float(numpy.abs(result.audio[:, :100]).max()) == 0.0 def test_a_shot_with_no_soundtrack_joins_as_before(self): - result = concat_videos([frames(4), audio_video(4, 0.5)], match_levels="rms") + # #553: fps is now required to fill the silent shot's own length + result = concat_videos( + [frames(4), audio_video(4, 0.5)], match_levels="rms", fps=4 + ) assert result.audio is not None assert len(result.frames) == 8 diff --git a/tests/test_pair_audio.py b/tests/test_pair_audio.py index 6cf1e824..b7c23259 100644 --- a/tests/test_pair_audio.py +++ b/tests/test_pair_audio.py @@ -11,6 +11,7 @@ import numpy import pytest import torch +from PIL import Image sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -99,6 +100,44 @@ def test_raises_when_no_sample_rate_can_be_established(): pair_audio(["frame"], _waveform()) +def test_loads_a_string_video_path_the_same_way_concat_videos_does(tmp_path): + """#553: 'audio' already loaded a string path via load_audio; 'video' did + not, so a workflow naming its video by an asset:/output: path (resolved + to a plain string by the time the task runs) failed rather than loading + it the way concat_videos loads one of its own inputs.""" + from diffusers.utils.export_utils import encode_video + + path = tmp_path / "shot.mp4" + encode_video( + [Image.new("RGB", (16, 16)) for _ in range(4)], + fps=4, + output_path=str(path), + ) + + paired = pair_audio(str(path), _waveform(samples=40), sample_rate=24000) + + assert isinstance(paired, AudioVideo) + assert len(paired.frames) == 4 + assert paired.audio.shape == (2, 40) + + +def test_loads_a_location_dict_video(tmp_path): + """The same {"location": ...} idiom concat_videos and get_last_frame accept.""" + from diffusers.utils.export_utils import encode_video + + path = tmp_path / "shot.mp4" + encode_video( + [Image.new("RGB", (16, 16)) for _ in range(4)], + fps=4, + output_path=str(path), + ) + + paired = pair_audio({"location": str(path)}, _waveform(samples=40), sample_rate=24000) + + assert isinstance(paired, AudioVideo) + assert len(paired.frames) == 4 + + def test_registered_as_a_task_command(): from dw.tasks.task import _COMMAND_REGISTRY From 2b5610a4775a344543113031aebd6c8084308105 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 21:45:02 -0500 Subject: [PATCH 026/251] fix(mcp): #553 - satisfy ruff format on the concat_videos/pair_audio fix Co-Authored-By: Claude Sonnet 5 --- dw/tasks/concat_videos.py | 3 +-- tests/test_pair_audio.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dw/tasks/concat_videos.py b/dw/tasks/concat_videos.py index c4d6f879..47a570b2 100644 --- a/dw/tasks/concat_videos.py +++ b/dw/tasks/concat_videos.py @@ -286,8 +286,7 @@ def concat_videos( dtype=numpy.float32, ) emit_log( - f"concat_videos: {names[index]} carries no audio - filled with " - "silence", + f"concat_videos: {names[index]} carries no audio - filled with silence", command="concat_videos", video=names[index], ) diff --git a/tests/test_pair_audio.py b/tests/test_pair_audio.py index b7c23259..f9375859 100644 --- a/tests/test_pair_audio.py +++ b/tests/test_pair_audio.py @@ -132,7 +132,9 @@ def test_loads_a_location_dict_video(tmp_path): output_path=str(path), ) - paired = pair_audio({"location": str(path)}, _waveform(samples=40), sample_rate=24000) + paired = pair_audio( + {"location": str(path)}, _waveform(samples=40), sample_rate=24000 + ) assert isinstance(paired, AudioVideo) assert len(paired.frames) == 4 From 0c2858121ffd9311eaa8f2b8c740c9c6d7583204 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 22:00:05 -0500 Subject: [PATCH 027/251] feat(tasks): #544 - find_loop_bed: fixed looped pool, per-window tally, per-block tonal test Answers the bounce on #544: - max_candidates only truncates the looped ranking; the looped pool is a fixed 200 non-overlapping survivors, so max_candidates=1 is the default run's rank 1 (C-F163 arm 3). - rejected counts every window on the grid under the first rule it fails, so the counts are comparable between two sources (C-F161). - the tonal test runs over every 0.2 s block inside a window, and a window is tonal when any block is: intermittent speech no longer averages below the threshold over a whole window (C-F161). - a window whose median 1 ms peak is zero is silent, not a spike. - a tick in the 5 ms just outside either end of a window counts against it, so a window no longer ends on a click (C-F163 side note). Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 29 ++++++-- dw/tasks/loop_bed.py | 138 +++++++++++++++++++++++------------- tests/test_find_loop_bed.py | 121 +++++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 53 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 47adba77..c4423f35 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -889,11 +889,30 @@ level check alone misses three things, each found on a real episode: | `max_candidates` | No | How many ranked candidates to return (default `5`) | Every window on a 50 ms grid, from `min_seconds` to `max_seconds` long, is -kept only if it is quiet, not digital silence, free of ticks and not tonal or -speech-like (the same flatness/harmonicity test `bleed_join` uses). -Overlapping survivors are thinned to the steadiest, rejecting one mostly -inside a span already found tonal without re-measuring it. Each survivor is -then looped with `loop_audio`'s own crossfade to `loop_seconds` and measured: +judged against four rules in order and counted in `rejected` under the first +one it fails, so `rejected` is a tally of the whole grid: + +- `too_loud` — a 50 ms bin above `max_bin_dbfs`, or a mean above `max_mean_dbfs`. +- `silent` — digital silence: a mean of zero, or a median 1 ms peak of zero + (more than half the window is exact zeros, whatever sits in the rest). +- `spike` — its largest 1 ms peak more than `max_spike_db` above its median + 1 ms peak. The largest peak is also looked for in the 5 ms just outside + each end, unless that neighbouring bin is itself too loud, so a window + ending on a click is thrown out rather than putting the click's onset at + the loop's seam. +- `tonal` — the flatness/harmonicity test `bleed_join` uses, taken over every + 0.2 s block inside the window (one per 50 ms step; the whole window when + `min_seconds` is shorter). A window is tonal when any block in it is. Faint + speech comes and goes, and over a whole window the pauses dilute a + syllable below the threshold; in the block it sits in, it is not diluted. + A candidate's `flatness` and `harmonicity` are the readings of its blocks + closest to failing (lowest flatness, highest harmonicity). + +The survivors are thinned so no two overlap, steadiest source first, to a +pool of up to 200 (`LOOPED_POOL`). The pool does not depend on +`max_candidates`, which only cuts the final ranking, so asking for one +candidate returns the default run's first. Each one in the pool is then +looped with `loop_audio`'s own crossfade to `loop_seconds` and measured: `ripple_db` (the 5-95% spread of the looped 50 ms bins), `envelope_peak_db`/ `envelope_peak_hz` (the strongest level wobble) and `lap_component_db` (the wobble at the lap rate). Candidates are ranked by looped `ripple_db`, lowest diff --git a/dw/tasks/loop_bed.py b/dw/tasks/loop_bed.py index 7e381eb2..5c978fbd 100644 --- a/dw/tasks/loop_bed.py +++ b/dw/tasks/loop_bed.py @@ -51,13 +51,20 @@ # (relative to the envelope's mean) carries `lap_modulation`. The two field # winners read -21 and -19 dB LAP_MODULATION_WARN_DB = -15.0 -# How many survivors, per candidate asked for, are looped and measured -LOOPED_PER_CANDIDATE = 4 +# How many non-overlapping survivors are looped and measured. Fixed rather +# than scaled by max_candidates, so asking for fewer candidates returns the +# head of the same ranking rather than a ranking of a smaller pool (#544) +LOOPED_POOL = 200 # The ripple spread, as percentiles of the 50 ms bins RIPPLE_PERCENTILES = (5.0, 95.0) -# Tonality is only measured for windows reaching it in pre-rank order, and -# this bounds how many are measured however much of the range is tonal -MAX_TONAL_CHECKS = 400 +# Tonality is judged per block of this length, sliding on the 50 ms grid, and +# a window is tonal when any block inside it is: faint speech comes and goes, +# and a whole-window measurement averages a syllable into the pauses around +# it (#544). Steady material reads the same either way +TONAL_BLOCK_SECONDS = 0.2 +# A tick this close outside a window still counts against it: a window +# ending on a click would put the click's edge at the loop's seam +TICK_GUARD_PEAKS = 5 REJECTION_RULES = ("too_loud", "silent", "spike", "tonal") @@ -120,6 +127,27 @@ def _bin_db(power): return 10.0 * numpy.log10(numpy.maximum(power, 1e-20)) +def _tonal_blocks(searched, bin_length, sample_rate, block_bins, needed): + """bleed_join's flatness/harmonicity test over every block of + `block_bins` bins, one per starting bin: (flatness, harmonicity, tonal) + arrays. Only blocks marked `needed` are measured - a block holding a loud + or silent bin lies only inside windows already thrown out - and the rest + read as noise (flatness 1, harmonicity 0), never as tonal.""" + count = needed.shape[0] + flatness = numpy.ones(count) + harmonicity = numpy.zeros(count) + for block in numpy.flatnonzero(needed): + segment = searched[block * bin_length : (block + block_bins) * bin_length][ + numpy.newaxis, : + ] + flatness[block] = _spectral_flatness(segment) + harmonicity[block] = _harmonicity(segment, sample_rate) + tonal = (flatness < TONAL_FLATNESS_THRESHOLD) | ( + harmonicity >= HARMONICITY_THRESHOLD + ) + return flatness, harmonicity, tonal + + def _looped(segment, sample_rate, crossfade_ms, loop_seconds): """`segment` looped to `loop_seconds` exactly as loop_audio loops it, and the readings of the result: its 50 ms ripple and its envelope spectrum.""" @@ -222,9 +250,11 @@ def find_loop_bed( kept only if it is quiet (every 50 ms bin at or below max_bin_dbfs, its mean at or below max_mean_dbfs), not digital silence, free of ticks (its largest 1 ms peak at most max_spike_db over its median 1 ms peak) and not - tonal or speech-like (bleed_join's flatness and harmonicity test). - Overlapping survivors are thinned to the steadiest, and each remaining - one is looped with loop_audio's own crossfade to loop_seconds and + tonal or speech-like (bleed_join's flatness and harmonicity test, over + every 0.2 s block inside it); `rejected` counts each window under the + first of those it fails. Overlapping survivors are thinned to the + steadiest, up to a pool that max_candidates does not shrink, and each + remaining one is looped with loop_audio's own crossfade to loop_seconds and measured: `ripple_db` (the 5-95 % spread of the looped 50 ms bins), `envelope_peak_db`/`envelope_peak_hz` (the strongest level wobble) and `lap_component_db` (the wobble at the lap rate). Candidates are ranked by @@ -338,35 +368,63 @@ def find_loop_bed( mean_threshold = 10.0 ** (max_mean_dbfs / 10.0) spike_ratio = 10.0 ** (max_spike_db / 20.0) - # Step 2 and 3, vectorised per window length: every (start, length) pair - # is judged at once for loudness, silence and ticks, and each survivor - # carries its source ripple for the pre-rank + # The tonal test's blocks, measured once over the range: a block is only + # worth measuring when every bin in it is quiet and not silent, since any + # window holding one of the others is thrown out before tonality + block_bins = max(1, min(shortest, int(round(TONAL_BLOCK_SECONDS / bin_seconds)))) + usable = (power <= bin_threshold) & (power > 0.0) + needed = numpy.zeros(count, dtype=bool) + if count >= block_bins: + needed[: count - block_bins + 1] = _sliding(usable, block_bins).all(axis=1) + block_flatness, block_harmonicity, block_tonal = _tonal_blocks( + searched, bin_length, sample_rate, block_bins, needed + ) + + # A tick just outside a window: the loudest few 1 ms peaks at the tail of + # the bin before each start and the head of the bin after each end. A + # neighbour loud enough to be thrown out on its own is louder material, + # not a tick, and the window stops short of it + edge_quiet = power <= bin_threshold + tails = numpy.where(edge_quiet, peaks[:, -TICK_GUARD_PEAKS:].max(axis=1), 0.0) + heads = numpy.where(edge_quiet, peaks[:, :TICK_GUARD_PEAKS].max(axis=1), 0.0) + before = numpy.concatenate(([0.0], tails[:-1])) + after = numpy.concatenate((heads[1:], [0.0])) + + # Steps 2 to 4, vectorised per window length: every (start, length) pair + # is judged at once and counted under the first rule it fails - loudness, + # silence, ticks, tonality - so `rejected` is a tally of every window on + # the grid, and each survivor carries its source ripple for the pre-rank survivors = [] # (source ripple, start bin, length in bins, readings) for length in range(shortest, longest + 1): windows = _sliding(power, length) # (starts, length) loudest = windows.max(axis=1) mean = windows.mean(axis=1) loud = (loudest > bin_threshold) | (mean > mean_threshold) - silent = ~loud & (mean <= 0.0) + peak_windows = _sliding(peaks, length).reshape(windows.shape[0], -1) + median = numpy.median(peak_windows, axis=1) + # More than half the window's 1 ms peaks at zero is digital silence + # with something in it, not room tone - and would read every + # sample of that something as a tick + silent = ~loud & ((mean <= 0.0) | (median <= 0.0)) + largest = numpy.maximum( + peak_windows.max(axis=1), + numpy.maximum(before[: windows.shape[0]], after[length - 1 :]), + ) + ticked = ~loud & ~silent & (largest > spike_ratio * median) + blocks = _sliding(block_tonal, length - block_bins + 1)[: windows.shape[0]] + tonal = ~loud & ~silent & ~ticked & blocks.any(axis=1) rejected["too_loud"] += int(loud.sum()) rejected["silent"] += int(silent.sum()) - quiet = numpy.flatnonzero(~loud & ~silent) - if quiet.size == 0: - continue - - peak_windows = _sliding(peaks, length)[quiet].reshape(quiet.size, -1) - largest = peak_windows.max(axis=1) - median = numpy.median(peak_windows, axis=1) - ticked = largest > spike_ratio * median rejected["spike"] += int(ticked.sum()) - kept = quiet[~ticked] + rejected["tonal"] += int(tonal.sum()) + kept = numpy.flatnonzero(~loud & ~silent & ~ticked & ~tonal) if kept.size == 0: continue - spikes = 20.0 * numpy.log10(largest[~ticked] / median[~ticked]) in_db = _bin_db(windows[kept]) low, high = numpy.percentile(in_db, RIPPLE_PERCENTILES, axis=1) for position, start in enumerate(kept): + inside = slice(start, start + length - block_bins + 1) survivors.append( ( float(high[position] - low[position]), @@ -375,43 +433,27 @@ def find_loop_bed( { "mean_dbfs": 10.0 * math.log10(float(mean[start])), "max_bin_dbfs": 10.0 * math.log10(float(loudest[start])), - "spike_db": float(spikes[position]), + "spike_db": 20.0 + * math.log10(float(largest[start] / median[start])), + # The block nearest to failing, which is what the + # test judged + "flatness": float(block_flatness[inside].min()), + "harmonicity": float(block_harmonicity[inside].max()), }, ) ) - # Steps 4 and 5: steadiest first, thinned so no two overlap. Tonality is - # measured only as a window is reached, and a window mostly inside one - # already found tonal inherits the verdict rather than being measured - # again - otherwise one tonal stretch costs a measurement per 50 ms shift + # Step 5: steadiest first, thinned so no two overlap, up to a pool that + # does not depend on how many candidates were asked for survivors.sort(key=lambda entry: (entry[0], -entry[2], entry[1])) kept = [] - tonal_spans = [] - checks = 0 - wanted = LOOPED_PER_CANDIDATE * max_candidates + pool = max(LOOPED_POOL, max_candidates) for ripple, start, length, readings in survivors: - if len(kept) >= wanted: + if len(kept) >= pool: break end = start + length if any(start < k_end and k_start < end for k_start, k_end, _ in kept): continue - if any( - min(end, t_end) - max(start, t_start) > length / 2 - for t_start, t_end in tonal_spans - ): - rejected["tonal"] += 1 - continue - if checks >= MAX_TONAL_CHECKS: - break - checks += 1 - segment = searched[start * bin_length : end * bin_length][numpy.newaxis, :] - flatness = _spectral_flatness(segment) - harmonicity = _harmonicity(segment, sample_rate) - if flatness < TONAL_FLATNESS_THRESHOLD or harmonicity >= HARMONICITY_THRESHOLD: - rejected["tonal"] += 1 - tonal_spans.append((start, end)) - continue - readings = {**readings, "flatness": flatness, "harmonicity": harmonicity} kept.append((start, end, readings)) # Steps 6 and 7: loop each, measure the bed, rank by its ripple diff --git a/tests/test_find_loop_bed.py b/tests/test_find_loop_bed.py index fd29cf8f..2412d40a 100644 --- a/tests/test_find_loop_bed.py +++ b/tests/test_find_loop_bed.py @@ -201,6 +201,127 @@ def test_no_candidate_covers_the_tone(self): assert result["rejected"]["tonal"] >= 1 +class TestIntermittentVoice: + """Speech under a bed comes and goes: 250 ms harmonic syllables every + 400 ms, at the bed's own level. Measured over a whole 0.5-2 s window the + pauses dilute the voice below the harmonicity threshold (about 0.37-0.40 + here, against 0.45); each syllable is tonal in the block it sits in, so + the voiced stretch yields no candidate and the tonal tally rises against + the same bed without it (#544).""" + + VOICED = (5.0, 9.65) # first onset, last syllable's end + + @staticmethod + def _bed(voiced): + rng = numpy.random.default_rng(11) + waveform = noise(rng, 15 * SR, -78.0) + if voiced: + t = numpy.arange(int(0.25 * SR)) / SR + syllable = sum( + numpy.sin(2 * math.pi * 150.0 * harmonic * t) / harmonic + for harmonic in range(1, 6) + ) + syllable *= rms_amplitude(-78.0) / numpy.sqrt(numpy.mean(syllable**2)) + for onset in numpy.arange(5.0, 9.5, 0.4): + at = int(round(onset * SR)) + waveform[at : at + syllable.size] += syllable + return waveform + + def test_no_candidate_covers_the_voice(self): + result = find_loop_bed(track(self._bed(voiced=True)), max_candidates=10) + + first, last = self.VOICED + assert result["candidates"] + for candidate in result["candidates"]: + assert ( + candidate["end_seconds"] <= first + 1e-6 + or candidate["start_seconds"] >= last - 1e-6 + ) + + def test_the_voice_raises_the_tonal_tally(self): + voiced = find_loop_bed(track(self._bed(voiced=True)))["rejected"] + control = find_loop_bed(track(self._bed(voiced=False)))["rejected"] + + assert voiced["tonal"] > control["tonal"] + + +class TestRejectedIsATallyOfEveryWindow: + """Every window on the grid is counted once, under the first rule it + fails, so the tally and the survivors together cover the grid.""" + + def test_the_counts_cover_the_grid(self): + rng = numpy.random.default_rng(12) + waveform = noise(rng, 4 * SR, -78.0) + waveform[: 2 * SR] = noise(rng, 2 * SR, -40.0) + + result = find_loop_bed(track(waveform), min_seconds=0.5, max_seconds=1.0) + + bins = 4 * SR // BIN_LENGTH + windows = sum(bins - length + 1 for length in range(10, 21)) + rejected = sum(result["rejected"].values()) + assert 0 < rejected < windows + assert result["rejected"]["too_loud"] > 0 + + +class TestDigitalSilenceIsSilent: + """A window that is mostly exact zeros is silence with something in it, + not a tick in room tone.""" + + def test_a_mostly_zero_window_counts_as_silent(self): + waveform = numpy.zeros(4 * SR) + rng = numpy.random.default_rng(13) + # a little noise in the first 10 ms of every bin, zeros after it + for start in range(0, waveform.size, BIN_LENGTH): + waveform[start : start + 480] = noise(rng, 480, -78.0) + + result = find_loop_bed(track(waveform)) + + assert result["rejected"]["silent"] > 0 + assert result["rejected"]["spike"] == 0 + assert result["candidates"] == [] + + +class TestATickAtTheEdge: + """A window ending exactly where a click starts would put the click's + onset at the loop's seam, so the click counts against it.""" + + def test_no_candidate_ends_on_the_click(self): + rng = numpy.random.default_rng(14) + waveform = noise(rng, 10 * SR, -78.0) + click = 5 * SR # the first sample of a bin + waveform[click : click + 24] = 30.0 * rms_amplitude(-78.0) + + result = find_loop_bed(track(waveform), max_candidates=50) + + assert result["candidates"] + for candidate in result["candidates"]: + assert not (candidate["start_seconds"] <= 5.0 <= candidate["end_seconds"]) + + +class TestMaxCandidatesOnlyTruncates: + """Asking for fewer candidates returns the head of the same ranking, + not the ranking of a smaller pool (#544).""" + + @staticmethod + def _unranked(candidate): + return {key: value for key, value in candidate.items() if key != "rank"} + + def test_fewer_candidates_are_the_head_of_the_default_run(self): + rng = numpy.random.default_rng(15) + waveform = noise(rng, 30 * SR, -78.0) + # uneven level second to second, so the pre-rank and the looped + # ranking disagree + for second in range(30): + waveform[second * SR : (second + 1) * SR] *= 1.0 + 0.4 * rng.random() + + full = find_loop_bed(track(waveform), max_candidates=10)["candidates"] + for count in (1, 3): + head = find_loop_bed(track(waveform), max_candidates=count)["candidates"] + assert [self._unranked(c) for c in head] == [ + self._unranked(c) for c in full[:count] + ] + + class TestLapModulation: """A candidate whose level ramps across the window becomes a sawtooth once looped, and that sawtooth should be visible in the measurement.""" From a584b4b51fe248e9f8dbff7d76eaad059018786a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 22:24:04 -0500 Subject: [PATCH 028/251] feat(tasks): #544 - find_loop_bed: flatness over the occupied band, 0.1 s and 0.2 s tonal blocks A bed resampled up to a mix's rate has an empty band above its own Nyquist that spectral flatness reads as tonal (#198): the C-F161 control read tonal in every window and a voice under it filled the band and read as noise. The occupied band is measured off the source's long-term spectrum and handed to _spectral_flatness as its native rate. Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 13 ++-- dw/tasks/loop_bed.py | 119 ++++++++++++++++++++++++++++-------- tests/test_find_loop_bed.py | 37 ++++++++++- 3 files changed, 139 insertions(+), 30 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index c4423f35..e87db343 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -901,12 +901,17 @@ one it fails, so `rejected` is a tally of the whole grid: ending on a click is thrown out rather than putting the click's onset at the loop's seam. - `tonal` — the flatness/harmonicity test `bleed_join` uses, taken over every - 0.2 s block inside the window (one per 50 ms step; the whole window when - `min_seconds` is shorter). A window is tonal when any block in it is. Faint + 0.1 s and every 0.2 s block inside the window (one per 50 ms step; no + longer than `min_seconds`). A window is tonal when any block in it is. Faint speech comes and goes, and over a whole window the pauses dilute a syllable below the threshold; in the block it sits in, it is not diluted. - A candidate's `flatness` and `harmonicity` are the readings of its blocks - closest to failing (lowest flatness, highest harmonicity). + Two lengths, because 0.1 s sits inside one syllable and 0.2 s holds enough + periods of a low hum. Flatness is measured over the band the source + actually occupies: a source resampled up (a 16 kHz bed mixed at 24 kHz) + has an empty band above its own Nyquist that reads as tonal whatever the + material is, as `bleed_join`'s tail does (#198). A candidate's `flatness` + and `harmonicity` are the readings of its blocks closest to failing + (lowest flatness, highest harmonicity). The survivors are thinned so no two overlap, steadiest source first, to a pool of up to 200 (`LOOPED_POOL`). The pool does not depend on diff --git a/dw/tasks/loop_bed.py b/dw/tasks/loop_bed.py index 5c978fbd..4ec3acc6 100644 --- a/dw/tasks/loop_bed.py +++ b/dw/tasks/loop_bed.py @@ -57,11 +57,20 @@ LOOPED_POOL = 200 # The ripple spread, as percentiles of the 50 ms bins RIPPLE_PERCENTILES = (5.0, 95.0) -# Tonality is judged per block of this length, sliding on the 50 ms grid, and -# a window is tonal when any block inside it is: faint speech comes and goes, -# and a whole-window measurement averages a syllable into the pauses around -# it (#544). Steady material reads the same either way -TONAL_BLOCK_SECONDS = 0.2 +# The band flatness is measured over: the energy an upsampled source's empty +# tail holds (band-limited interpolation leaves it ~60 dB and more down), and +# how much of the band must be occupied before the whole of it is used +OCCUPIED_BAND_FLOOR = 1e-6 +OCCUPIED_BAND_FULL = 0.95 +OCCUPIED_FRAME = 4096 +# Tonality is judged per block, sliding on the 50 ms grid, and a window is +# tonal when any block inside it is: faint speech comes and goes, and a +# whole-window measurement averages a syllable into the pauses around it +# (#544). Two block lengths, because neither catches both: 0.1 s sits inside +# one syllable, where a longer block averages the pause beside it back under +# the threshold, and 0.2 s holds enough periods of a low hum that the +# autocorrelation's shrinking overlap does not read it below the threshold +TONAL_BLOCK_SECONDS = (0.1, 0.2) # A tick this close outside a window still counts against it: a window # ending on a click would put the click's edge at the loop's seam TICK_GUARD_PEAKS = 5 @@ -127,7 +136,46 @@ def _bin_db(power): return 10.0 * numpy.log10(numpy.maximum(power, 1e-20)) -def _tonal_blocks(searched, bin_length, sample_rate, block_bins, needed): +def _occupied_rate(mono, sample_rate): + """The rate the material actually fills, read off its long-term spectrum: + twice the frequency below which all but OCCUPIED_BAND_FLOOR of its energy + sits, or None when that is the whole band. + + Resampling up leaves the band above the original Nyquist empty, and an + empty band reads as tonal to spectral flatness whatever the material is + (#198) - a 16 kHz bed mixed at 24 kHz read tonal in every window, and a + voice under it filled the band and read as noise (#544). The source + carries no record of its native rate once mixed, so it is measured. + """ + frame = min(OCCUPIED_FRAME, mono.shape[0]) + frames = mono.shape[0] // frame + if frames < 1: + return None + spectrum = numpy.mean( + numpy.square( + numpy.abs( + numpy.fft.rfft( + mono[: frames * frame].reshape(frames, frame) + * numpy.hanning(frame), + axis=1, + ) + ) + ), + axis=0, + ) + total = spectrum.sum() + if total <= 0.0: + return None + above = numpy.cumsum(spectrum[::-1])[::-1] / total + occupied = int(numpy.flatnonzero(above > OCCUPIED_BAND_FLOOR)[-1]) + 1 + if occupied >= OCCUPIED_BAND_FULL * spectrum.shape[0]: + return None + return sample_rate * occupied / (spectrum.shape[0] - 1) + + +def _tonal_blocks( + searched, bin_length, sample_rate, block_bins, needed, native_rate=None +): """bleed_join's flatness/harmonicity test over every block of `block_bins` bins, one per starting bin: (flatness, harmonicity, tonal) arrays. Only blocks marked `needed` are measured - a block holding a loud @@ -140,7 +188,7 @@ def _tonal_blocks(searched, bin_length, sample_rate, block_bins, needed): segment = searched[block * bin_length : (block + block_bins) * bin_length][ numpy.newaxis, : ] - flatness[block] = _spectral_flatness(segment) + flatness[block] = _spectral_flatness(segment, sample_rate, native_rate) harmonicity[block] = _harmonicity(segment, sample_rate) tonal = (flatness < TONAL_FLATNESS_THRESHOLD) | ( harmonicity >= HARMONICITY_THRESHOLD @@ -251,7 +299,7 @@ def find_loop_bed( mean at or below max_mean_dbfs), not digital silence, free of ticks (its largest 1 ms peak at most max_spike_db over its median 1 ms peak) and not tonal or speech-like (bleed_join's flatness and harmonicity test, over - every 0.2 s block inside it); `rejected` counts each window under the + every 0.1 s and 0.2 s block inside it); `rejected` counts each window under the first of those it fails. Overlapping survivors are thinned to the steadiest, up to a pool that max_candidates does not shrink, and each remaining one is looped with loop_audio's own crossfade to loop_seconds and @@ -368,17 +416,28 @@ def find_loop_bed( mean_threshold = 10.0 ** (max_mean_dbfs / 10.0) spike_ratio = 10.0 ** (max_spike_db / 20.0) - # The tonal test's blocks, measured once over the range: a block is only - # worth measuring when every bin in it is quiet and not silent, since any - # window holding one of the others is thrown out before tonality - block_bins = max(1, min(shortest, int(round(TONAL_BLOCK_SECONDS / bin_seconds)))) + # The tonal test's blocks, measured once over the range at each block + # length: a block is only worth measuring when every bin in it is quiet + # and not silent, since any window holding one of the others is thrown + # out before tonality usable = (power <= bin_threshold) & (power > 0.0) - needed = numpy.zeros(count, dtype=bool) - if count >= block_bins: - needed[: count - block_bins + 1] = _sliding(usable, block_bins).all(axis=1) - block_flatness, block_harmonicity, block_tonal = _tonal_blocks( - searched, bin_length, sample_rate, block_bins, needed - ) + native_rate = _occupied_rate(searched, sample_rate) + scales = [] # (block length in bins, flatness, harmonicity, tonal) + for block_seconds in TONAL_BLOCK_SECONDS: + block_bins = max(1, min(shortest, int(round(block_seconds / bin_seconds)))) + if any(block_bins == known[0] for known in scales): + continue + needed = numpy.zeros(count, dtype=bool) + if count >= block_bins: + needed[: count - block_bins + 1] = _sliding(usable, block_bins).all(axis=1) + scales.append( + ( + block_bins, + *_tonal_blocks( + searched, bin_length, sample_rate, block_bins, needed, native_rate + ), + ) + ) # A tick just outside a window: the loudest few 1 ms peaks at the tail of # the bin before each start and the head of the bin after each end. A @@ -411,8 +470,11 @@ def find_loop_bed( numpy.maximum(before[: windows.shape[0]], after[length - 1 :]), ) ticked = ~loud & ~silent & (largest > spike_ratio * median) - blocks = _sliding(block_tonal, length - block_bins + 1)[: windows.shape[0]] - tonal = ~loud & ~silent & ~ticked & blocks.any(axis=1) + any_tonal = numpy.zeros(windows.shape[0], dtype=bool) + for block_bins, _, _, block_tonal in scales: + inside = _sliding(block_tonal, length - block_bins + 1) + any_tonal |= inside[: windows.shape[0]].any(axis=1) + tonal = ~loud & ~silent & ~ticked & any_tonal rejected["too_loud"] += int(loud.sum()) rejected["silent"] += int(silent.sum()) rejected["spike"] += int(ticked.sum()) @@ -424,7 +486,16 @@ def find_loop_bed( in_db = _bin_db(windows[kept]) low, high = numpy.percentile(in_db, RIPPLE_PERCENTILES, axis=1) for position, start in enumerate(kept): - inside = slice(start, start + length - block_bins + 1) + # The block nearest to failing, at either length, which is what + # the test judged + flatness = min( + float(flat[start : start + length - bins + 1].min()) + for bins, flat, _, _ in scales + ) + harmonicity = max( + float(harm[start : start + length - bins + 1].max()) + for bins, _, harm, _ in scales + ) survivors.append( ( float(high[position] - low[position]), @@ -435,10 +506,8 @@ def find_loop_bed( "max_bin_dbfs": 10.0 * math.log10(float(loudest[start])), "spike_db": 20.0 * math.log10(float(largest[start] / median[start])), - # The block nearest to failing, which is what the - # test judged - "flatness": float(block_flatness[inside].min()), - "harmonicity": float(block_harmonicity[inside].max()), + "flatness": flatness, + "harmonicity": harmonicity, }, ) ) diff --git a/tests/test_find_loop_bed.py b/tests/test_find_loop_bed.py index 2412d40a..2b6ef881 100644 --- a/tests/test_find_loop_bed.py +++ b/tests/test_find_loop_bed.py @@ -19,7 +19,7 @@ from dw.scalar_result_validation import scalar_result_errors from dw.task_domains import task_argument_errors from dw.tasks.audio_utils import _PERIODICITY_MAX_HZ, _PERIODICITY_MIN_HZ, _harmonicity -from dw.tasks.loop_bed import find_loop_bed +from dw.tasks.loop_bed import _occupied_rate, find_loop_bed from dw.tasks.task import task_command_info from dw.workflow import Workflow @@ -245,6 +245,41 @@ def test_the_voice_raises_the_tonal_tally(self): assert voiced["tonal"] > control["tonal"] +class TestAnUpsampledBed: + """A bed resampled up to a mix's rate has an empty band above its own + Nyquist, which spectral flatness reads as tonal whatever the material: + a 16 kHz bed mixed at 24 kHz read tonal in every window, and a voice + under it filled the band and read as noise (#544). Flatness is measured + over the band the material occupies, as bleed_join's is (#198).""" + + @staticmethod + def _upsampled(seed=21, seconds=4.0, native=SR // 2, dbfs=-78.0): + """Noise made at `native` and band-limited up to SR by zero-padding + its spectrum - what a resampler leaves above the old Nyquist.""" + rng = numpy.random.default_rng(seed) + low = noise(rng, int(seconds * native), dbfs) + spectrum = numpy.fft.rfft(low) + length = int(seconds * SR) + padded = numpy.zeros(length // 2 + 1, dtype=complex) + padded[: spectrum.shape[0]] = spectrum + return numpy.fft.irfft(padded, n=length) * (SR / native) + + def test_the_occupied_band_is_measured(self): + rate = _occupied_rate(self._upsampled(), SR) + assert rate is not None + assert SR // 2 <= rate < 0.6 * SR + + def test_a_full_band_source_is_left_alone(self): + rng = numpy.random.default_rng(22) + assert _occupied_rate(noise(rng, 4 * SR, -78.0), SR) is None + + def test_upsampled_noise_is_still_a_bed(self): + result = find_loop_bed(audio=track(self._upsampled())) + assert result["rejected"]["tonal"] == 0 + assert result["candidates"] + assert all(c["flatness"] >= 0.3 for c in result["candidates"]) + + class TestRejectedIsATallyOfEveryWindow: """Every window on the grid is counted once, under the first rule it fails, so the tally and the survivors together cover the grid.""" From ba6d499ad5d328ab4f19568337007955155eb78c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 22:45:55 -0500 Subject: [PATCH 029/251] feat(tasks): #545 - shot-aware find_loop_bed, and the skills that point at it shots argument (+fps), resolved argument > carried > run manifest > none; no candidate crosses a boundary, each names its shot, source.shots_source says where the shots came from. shot_dead_air names find_loop_bed; minimax-h3 and series-episodes point at it. Co-Authored-By: Claude Opus 5.5 --- docs/TASKS.md | 17 ++ docs/WORKFLOW_GUIDE.md | 2 +- dw/assessment_rules.py | 5 +- dw/media_audio.py | 11 + dw/task_domains.py | 1 + dw/tasks/loop_bed.py | 199 ++++++++++++++++-- plugins/dw/skills/minimax-h3/SKILL.md | 14 +- plugins/dw/skills/series-episodes/SKILL.md | 8 +- tests/test_assessment_rules.py | 10 +- tests/test_find_loop_bed.py | 222 ++++++++++++++++++++- 10 files changed, 455 insertions(+), 34 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index e87db343..064cccc5 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -887,6 +887,8 @@ level check alone misses three things, each found on a real episode: | `loop_seconds` | No | Length of the looped result that is measured (default `10.0`) | | `target_bed_dbfs` | No | The level each candidate's `gain` is computed to reach (default `-60`) | | `max_candidates` | No | How many ranked candidates to return (default `5`) | +| `shots` | No | Shot boundaries, the assessment probes' shape: `[{name, start_frame, num_frames}]`, optionally with `start_sample`/`num_samples`; overrides the shots a video carries or its run records | +| `fps` | No | Frame rate the shots' frames count at, for a source with none of its own (an audio file); a video's own rate otherwise | Every window on a 50 ms grid, from `min_seconds` to `max_seconds` long, is judged against four rules in order and counted in `rejected` under the first @@ -913,6 +915,21 @@ one it fails, so `rejected` is a tally of the whole grid: and `harmonicity` are the readings of its blocks closest to failing (lowest flatness, highest harmonicity). +On a cut, a bed must come from inside one shot: a window across a cut +loops the seam's change of room as a once-per-lap step. Shots resolve in the +probes' order - the `shots` argument (`shots_source: "argument"`), else the +shots a video from an earlier step carries (`"artifact"`), else the ones the +run manifest beside the file records (`"manifest"`, which is why +`output:` the joined file finds them unasked), else none (`null`, the search +above unchanged). With shots, a window that crosses a boundary, or lies where +no shot covers, is counted under `rejected.shot_boundary` before the four +rules (so theirs count only in-shot windows), each candidate names its +`shot` (`shot@` from a manifest), and `source.shots` lists each shot's +`{name, start_seconds, end_seconds}` as placed in the soundtrack. A shot with +both frames and recorded samples is placed at their overlap - the picture's +cut and the audio's can sit a few samples apart. A shot placed by frames on +a source with no frame rate is refused; pass `fps`. + The survivors are thinned so no two overlap, steadiest source first, to a pool of up to 200 (`LOOPED_POOL`). The pool does not depend on `max_candidates`, which only cuts the final ranking, so asking for one diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 75d4d0a5..54184bde 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -1022,7 +1022,7 @@ already made: each entry in `videos` is `output:` + the run's | `seam_click` | a longer `crossfade_ms` on the join | recut | | `seam_hole` | `audio_bleed_ms` on the join, so the outgoing tail rings on across the seam | recut | | `seam_frame_jump` | a `dissolve_videos` join, or regenerate the incoming shot from the outgoing shot's last frame. If the cut was meant, leave it alone | recut, or regenerate | -| `shot_dead_air` | cut a room-tone bed from the take with `slice_audio`, `loop_audio` it to the gap's length, and `mix_audio` it under the line rather than leaving the drop silent | recut | +| `shot_dead_air` | `find_loop_bed` on the cut names a quiet stretch inside one shot and its gain; cut that bed with `slice_audio`, `loop_audio` it to the gap's length, and `mix_audio` it under the line rather than leaving the drop silent | recut | | `sync_drift` | regenerate the shot. Drift inside a shot is the model's, not the join's | regenerate | | `sync_length` | rerun the mux through `pair_audio` with `fit: "video"`, which cuts or pads the track to the picture | recut | diff --git a/dw/assessment_rules.py b/dw/assessment_rules.py index f8a6acca..11a63fe1 100644 --- a/dw/assessment_rules.py +++ b/dw/assessment_rules.py @@ -112,8 +112,9 @@ "severity": "warn", "says": ( "the shot holds a gap this long, well below the floor a line's own room" - " tone sits at - cut a room-tone bed from the take with slice_audio, loop" - " it to the gap's length with loop_audio, and mix it under the line with" + " tone sits at - find_loop_bed on the cut names a quiet stretch inside" + " one shot and the gain to lay it at; cut it with slice_audio, loop it" + " to the gap's length with loop_audio, and mix it under the line with" " mix_audio rather than leaving the drop silent" ), "unless": f"the shot's own rms is at or below {HOLE_VOICED_DBFS} dBFS", diff --git a/dw/media_audio.py b/dw/media_audio.py index dcf94c96..34768e84 100644 --- a/dw/media_audio.py +++ b/dw/media_audio.py @@ -50,6 +50,17 @@ def media_duration(path): return _container_duration(container) +def container_fps(path): + """The picture stream's frame rate from the container's headers alone, or + None for a file with no picture stream (or none it declares) - what turns + a shot's frames into seconds without decoding a frame.""" + with av.open(path) as container: + if not container.streams.video: + return None + rate = container.streams.video[0].average_rate + return float(rate) if rate else None + + def audio_shape(path): """The soundtrack's duration (seconds), sample rate and channel count from the container's headers alone - what projecting the size of a diff --git a/dw/task_domains.py b/dw/task_domains.py index e0518ef1..1b241ac3 100644 --- a/dw/task_domains.py +++ b/dw/task_domains.py @@ -117,6 +117,7 @@ "loop_seconds": POSITIVE, "target_bed_dbfs": NON_POSITIVE, "max_candidates": POSITIVE, + "fps": POSITIVE, }, "fade_audio": { "fade_in_ms": NON_NEGATIVE, diff --git a/dw/tasks/loop_bed.py b/dw/tasks/loop_bed.py index 4ec3acc6..ea33363f 100644 --- a/dw/tasks/loop_bed.py +++ b/dw/tasks/loop_bed.py @@ -87,26 +87,138 @@ def _round(value, places=2): def _read_source(audio): - """(waveform, rate) for the `audio` argument. A local video file is read - audio-only through media_audio - load_audio would decode every frame of - a cut to reach its soundtrack.""" + """(waveform, rate, fps, carried shots, file path) for the `audio` + argument. A local video file is read audio-only through media_audio - + load_audio would decode every frame of a cut to reach its soundtrack - + and its frame rate comes off the container's headers. `fps` and the + carried shots are an in-memory video's own; `path` is the validated file + whose run manifest may record shots, None for anything in memory or + remote.""" + from .video_utils import VideoFileReference + + if isinstance(audio, VideoFileReference): + audio = audio.path if isinstance(audio, str) and not audio.startswith(("http://", "https://")): from ..locations import validate_media_path from ..security import ALLOWED_VIDEO_EXTENSIONS + path = validate_media_path(audio, None, "an audio argument") extension = os.path.splitext(audio)[1].lower() if extension in ALLOWED_VIDEO_EXTENSIONS: - from ..media_audio import NoSoundtrack, decode_soundtrack + from ..media_audio import NoSoundtrack, container_fps, decode_soundtrack - path = validate_media_path(audio, None, "an audio argument") try: - return decode_soundtrack(path) + waveform, rate = decode_soundtrack(path) except NoSoundtrack: raise ValueError( f"{COMMAND}: {audio} carries no audio track - a bed is " "found in a soundtrack" ) from None - return _waveform_and_rate(audio, None, COMMAND) + return waveform, rate, container_fps(path), None, path + waveform, rate = _waveform_and_rate(audio, None, COMMAND) + return waveform, rate, None, None, path + waveform, rate = _waveform_and_rate(audio, None, COMMAND) + return ( + waveform, + rate, + getattr(audio, "fps", None), + getattr(audio, "shots", None), + None, + ) + + +def _resolve_shots(shots, carried, path): + """The shots to keep candidates inside, and where they came from - the + assessment probes' order (`resolve_shots`, dw/tasks/assess.py): the + argument, then the shots the video carries, then the run manifest beside + the file, else (None, None).""" + if shots: + return [dict(shot) for shot in shots], "argument" + if carried: + return [dict(shot) for shot in carried], "artifact" + if path is not None: + from ..runs import shots_beside + + recorded = shots_beside(path) + if recorded: + return [dict(shot) for shot in recorded], "manifest" + return None, None + + +def _check_shots(shots): + """Refuse a `shots` argument that is not a list of shot records.""" + if not isinstance(shots, (list, tuple)): + raise ValueError( + f"{COMMAND}: 'shots' is a list of {{name, start_frame, num_frames}} " + f"records, not {type(shots).__name__}" + ) + for index, shot in enumerate(shots): + if not isinstance(shot, dict) or not shot.get("name"): + raise ValueError( + f"{COMMAND}: shots[{index}] needs a 'name' - a candidate names " + "the shot it lies in" + ) + for field in ("start_frame", "num_frames", "start_sample", "num_samples"): + value = shot.get(field) + if value is None: + continue + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError( + f"{COMMAND}: shots[{index}].{field} must be a number, not {value!r}" + ) + if value < 0 or (field.startswith("num_") and value <= 0): + raise ValueError( + f"{COMMAND}: shots[{index}].{field} must be " + f"{'above' if field.startswith('num_') else 'at or above'} " + f"zero, not {value!r}" + ) + + +def _shot_spans(shots, fps, sample_rate): + """Each shot's (name, first sample, end sample) in the source's samples. + + A shot is placed by its frames (at `fps`) and by its recorded samples, + and where it has both it is the overlap of the two: the picture's cut + and the soundtrack's can sit a few samples apart (a shot's audio + overrun, dw/shots.py), and a window inside both crosses neither. + """ + spans = [] + for index, shot in enumerate(shots): + found = [] + start_frame, num_frames = shot.get("start_frame"), shot.get("num_frames") + if fps and start_frame is not None and num_frames is not None: + found.append( + ( + start_frame / fps * sample_rate, + (start_frame + num_frames) / fps * sample_rate, + ) + ) + start_sample, num_samples = shot.get("start_sample"), shot.get("num_samples") + if start_sample is not None and num_samples is not None: + found.append((start_sample, start_sample + num_samples)) + if not found: + raise ValueError( + f"{COMMAND}: shot '{shot.get('name')}' (shots[{index}]) is placed " + "by its frames and the source has no frame rate - pass 'fps', " + "or give the shot start_sample/num_samples" + ) + first = max(span[0] for span in found) + end = min(span[1] for span in found) + spans.append((str(shot["name"]), int(math.ceil(first)), int(end))) + return spans + + +def _bin_shots(spans, offset, bin_length, count): + """Which shot each 50 ms bin of the searched range lies wholly inside, as + an index into `spans`; -1 for a bin a boundary cuts through or no shot + covers.""" + owner = numpy.full(count, -1, dtype=numpy.int64) + starts = offset + numpy.arange(count) * bin_length + ends = starts + bin_length + for index, (_, first, end) in enumerate(spans): + inside = (starts >= first) & (ends <= end) & (owner < 0) + owner[inside] = index + return owner def _sliding(array, length): @@ -251,7 +363,7 @@ def _empty_finding(rejected, criteria): message = "no window fit in the range; widen it or lower min_seconds" rule = None else: - rule = max(REJECTION_RULES, key=lambda name: rejected[name]) + rule = max(rejected, key=lambda name: rejected[name]) message = { "too_loud": ( f"nothing below {criteria['max_bin_dbfs']:g} dBFS per 50 ms " @@ -268,6 +380,11 @@ def _empty_finding(rejected, criteria): "that would recur once per lap; widen the range or raise " "max_spike_db" ), + "shot_boundary": ( + "every window that fits crosses a shot boundary or lies " + "outside every shot - the shots in the range are shorter " + "than min_seconds; lower it or search another shot" + ), "tonal": ( "every quiet window is tonal or speech-like - faint " "programme that would be audible once it repeats; widen " @@ -290,6 +407,8 @@ def find_loop_bed( loop_seconds=10.0, target_bed_dbfs=-60.0, max_candidates=5, + shots=None, + fps=None, ): """Task command: rank the windows of a recording worth looping into a room-tone bed, measured as they will sound looped. @@ -311,6 +430,14 @@ def find_loop_bed( A candidate's start_seconds/duration_seconds are slice_audio's arguments and its gain is mix_audio's multiplier for reaching target_bed_dbfs. + With shot boundaries - the `shots` argument, else the shots a video from + an earlier step carries, else the ones the run manifest beside the file + records - no candidate crosses a boundary, each names the `shot` it lies + in, `source.shots_source` says which of the three the shots came from + (`argument`, `artifact`, `manifest`; null with none) and `rejected` + counts the windows a boundary cut through under `shot_boundary`. A + stretch no shot covers is not searched. + Finding nothing is an answer: `candidates` is empty, `rejected` counts the windows each rule threw out, and one finding says which to relax. The result must be saved as application/json. @@ -331,6 +458,11 @@ def find_loop_bed( loop_seconds: Length of the looped result that is measured target_bed_dbfs: The level each candidate's gain is computed to reach max_candidates: How many ranked candidates to return + shots: Shot boundaries, the assessment probes' shape: a list of + {name, start_frame, num_frames}, optionally with start_sample and + num_samples; overrides the shots a video carries or its run records + fps: Frame rate the shots' frames are counted at, for a source that + has none of its own (an audio file); a video's own rate otherwise Returns: A JSON-safe dict: source, criteria, candidates, rejected, findings @@ -346,6 +478,9 @@ def find_loop_bed( loop_seconds = _as_number(loop_seconds, float, "loop_seconds", COMMAND) target_bed_dbfs = _as_number(target_bed_dbfs, float, "target_bed_dbfs", COMMAND) max_candidates = _as_number(max_candidates, int, "max_candidates", COMMAND) + fps = _as_number(fps, float, "fps", COMMAND) + if shots is not None: + _check_shots(shots) check_arguments( COMMAND, start_seconds=start_seconds, @@ -359,6 +494,7 @@ def find_loop_bed( loop_seconds=loop_seconds, target_bed_dbfs=target_bed_dbfs, max_candidates=max_candidates, + fps=fps, ) if min_seconds > max_seconds: raise ValueError( @@ -366,7 +502,8 @@ def find_loop_bed( f"'max_seconds' ({max_seconds:g})" ) - waveform, sample_rate = _read_source(audio) + waveform, sample_rate, source_fps, carried, path = _read_source(audio) + shots, shots_source = _resolve_shots(shots, carried, path) mono = numpy.asarray(waveform, dtype=numpy.float64) mono = mono.mean(axis=0) if mono.ndim == 2 else mono duration = mono.shape[0] / sample_rate @@ -408,10 +545,16 @@ def find_loop_bed( bin_length, power, peaks = _envelope(searched, sample_rate) bin_seconds = bin_length / sample_rate count = power.shape[0] + spans = _shot_spans(shots, fps or source_fps, sample_rate) if shots else None + bin_shot = ( + _bin_shots(spans, offset, bin_length, count) if spans is not None else None + ) shortest = max(1, int(round(min_seconds / bin_seconds))) longest = min(count, int(round(max_seconds / bin_seconds))) rejected = dict.fromkeys(REJECTION_RULES, 0) + if spans is not None: + rejected["shot_boundary"] = 0 bin_threshold = 10.0 ** (max_bin_dbfs / 10.0) mean_threshold = 10.0 ** (max_mean_dbfs / 10.0) spike_ratio = 10.0 ** (max_spike_db / 20.0) @@ -458,28 +601,38 @@ def find_loop_bed( windows = _sliding(power, length) # (starts, length) loudest = windows.max(axis=1) mean = windows.mean(axis=1) - loud = (loudest > bin_threshold) | (mean > mean_threshold) + if bin_shot is None: + crossing = numpy.zeros(windows.shape[0], dtype=bool) + else: + # A window is inside one shot when every bin it holds is: the + # same shot throughout, and none a boundary cuts or no shot covers + owners = _sliding(bin_shot, length) + crossing = (owners.min(axis=1) != owners.max(axis=1)) | ( + owners.min(axis=1) < 0 + ) + rejected["shot_boundary"] += int(crossing.sum()) + loud = ~crossing & ((loudest > bin_threshold) | (mean > mean_threshold)) peak_windows = _sliding(peaks, length).reshape(windows.shape[0], -1) median = numpy.median(peak_windows, axis=1) # More than half the window's 1 ms peaks at zero is digital silence # with something in it, not room tone - and would read every # sample of that something as a tick - silent = ~loud & ((mean <= 0.0) | (median <= 0.0)) + silent = ~crossing & ~loud & ((mean <= 0.0) | (median <= 0.0)) largest = numpy.maximum( peak_windows.max(axis=1), numpy.maximum(before[: windows.shape[0]], after[length - 1 :]), ) - ticked = ~loud & ~silent & (largest > spike_ratio * median) + ticked = ~crossing & ~loud & ~silent & (largest > spike_ratio * median) any_tonal = numpy.zeros(windows.shape[0], dtype=bool) for block_bins, _, _, block_tonal in scales: inside = _sliding(block_tonal, length - block_bins + 1) any_tonal |= inside[: windows.shape[0]].any(axis=1) - tonal = ~loud & ~silent & ~ticked & any_tonal + tonal = ~crossing & ~loud & ~silent & ~ticked & any_tonal rejected["too_loud"] += int(loud.sum()) rejected["silent"] += int(silent.sum()) rejected["spike"] += int(ticked.sum()) rejected["tonal"] += int(tonal.sum()) - kept = numpy.flatnonzero(~loud & ~silent & ~ticked & ~tonal) + kept = numpy.flatnonzero(~crossing & ~loud & ~silent & ~ticked & ~tonal) if kept.size == 0: continue @@ -542,7 +695,7 @@ def find_loop_bed( "start_seconds": round(start_at, 3), "duration_seconds": round(seconds, 3), "end_seconds": round(start_at + seconds, 3), - "shot": None, + "shot": None if bin_shot is None else spans[bin_shot[start]][0], "mean_dbfs": _round(readings["mean_dbfs"]), "max_bin_dbfs": _round(readings["max_bin_dbfs"]), "spike_db": _round(readings["spike_db"]), @@ -582,7 +735,21 @@ def find_loop_bed( "duration_seconds": round(duration, 3), "sample_rate": int(sample_rate), "searched": [round(first, 3), round(last, 3)], - "shots_source": None, + "shots_source": shots_source, + **( + { + "shots": [ + { + "name": name, + "start_seconds": round(first / sample_rate, 3), + "end_seconds": round(end / sample_rate, 3), + } + for name, first, end in spans + ] + } + if spans is not None + else {} + ), }, "criteria": criteria, "candidates": candidates, diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index a72bdc93..4d2d006b 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -182,13 +182,13 @@ itself - or every shot inherits the portrait's composition. everywhere), a portrait imposing its framing on every shot - `at` late in a chain for drift sharpening to noise, and `get_output_audio` for a voice-over without affect. `get_output_audio` returns sound, not text; to - confirm a line rendered, use its docstring's transcription route - (`templates/transcribe-audio`, then `get_output_text`). - A `shot_dead_air` finding: bed with `slice_audio`->`loop_audio`->`mix_audio` - room tone, not silence. Then `get_gallery_metadata` for duration/audio - presence, and hand the user the gallery `url` - (`list_gallery`). `get_output_image` works only on image steps (the - Z-Image portraits and boards). + confirm a line, transcribe (`templates/transcribe-audio`, + `get_output_text`). Dialogue gaps (`shot_dead_air`) need room tone: + `find_loop_bed` the `output:` cut; + `slice_audio`->`loop_audio`->`mix_audio` its pick at its `gain`. + Then `get_gallery_metadata` for duration/audio presence, and hand the + user the gallery `url` (`list_gallery`). `get_output_image` works only on + image steps (the Z-Image portraits and boards). 5. After a run worth keeping, `get_job_workflow` and `save_workflow` it; `export_job` bundles it on the server. `auth_required: false` - fetch `open_url` into `exports/` (never a temp dir). `true` - hand `open_url` diff --git a/plugins/dw/skills/series-episodes/SKILL.md b/plugins/dw/skills/series-episodes/SKILL.md index 4f4092fb..403b909a 100644 --- a/plugins/dw/skills/series-episodes/SKILL.md +++ b/plugins/dw/skills/series-episodes/SKILL.md @@ -82,7 +82,10 @@ composes into, not something to re-derive: - **bed**: if the episode's score is shorter than the cut, stretch it first with the `loop_audio` task (`target_frames` = the cut's `total_frames`, `fps` matching) rather than letting `assemble-and-score` pad the tail - with silence. + with silence. A dialogue episode also needs room tone under its line + gaps: `find_loop_bed` on the recut (`output:` it, so the run's shots keep + every candidate inside one shot) names the stretch to `slice_audio` and + the `gain` to `mix_audio` it at. - **match_levels**: shots generated independently drift in loudness - `assemble-and-score`'s `match_levels` (`"rms"` or `"peak"`) evens them before the cut; leaving it null only warns on a wide spread instead of @@ -126,7 +129,8 @@ Judge a finished episode with `assess_output(name)` before listening end to end: it measures every seam and the shots' levels, and says where to look. A `shot_dead_air` finding is an H3 dialogue gap (0.5-2s of near-silence between lines) inside one shot, not a seam problem - see `minimax-h3`'s -room-tone bed recipe (`slice_audio` -> `loop_audio` -> `mix_audio`) rather +room-tone bed recipe (`find_loop_bed`, then `slice_audio` -> `loop_audio` -> +`mix_audio`) rather than treating it as a cut to fix. `output:` a shot straight from its generation run rather than downloading diff --git a/tests/test_assessment_rules.py b/tests/test_assessment_rules.py index a06fafdc..c8fe7071 100644 --- a/tests/test_assessment_rules.py +++ b/tests/test_assessment_rules.py @@ -279,13 +279,15 @@ class TestFindingSaysTheRemedy: def test_shot_dead_air_names_the_room_tone_recipe(self): """#491: the finding an agent actually reads, not just the docs, has to carry the remedy - lay a room-tone bed with slice_audio -> - loop_audio -> mix_audio, using the take's own quiet stretch.""" + loop_audio -> mix_audio, using the take's own quiet stretch, which + find_loop_bed finds (#545).""" says = RULES_BY_NAME["shot_dead_air"]["says"] assert says == ( "the shot holds a gap this long, well below the floor a line's own room" - " tone sits at - cut a room-tone bed from the take with slice_audio, loop" - " it to the gap's length with loop_audio, and mix it under the line with" + " tone sits at - find_loop_bed on the cut names a quiet stretch inside" + " one shot and the gain to lay it at; cut it with slice_audio, loop it" + " to the gap's length with loop_audio, and mix it under the line with" " mix_audio rather than leaving the drop silent" ) - for command in ("slice_audio", "loop_audio", "mix_audio"): + for command in ("find_loop_bed", "slice_audio", "loop_audio", "mix_audio"): assert command in says diff --git a/tests/test_find_loop_bed.py b/tests/test_find_loop_bed.py index 2b6ef881..2294c803 100644 --- a/tests/test_find_loop_bed.py +++ b/tests/test_find_loop_bed.py @@ -7,6 +7,7 @@ for a sine), matching the engine's own `10*log10(mean square)` bin reading. """ +import json import math import numpy @@ -500,12 +501,13 @@ def test_string_numerics_are_coerced(self): assert "candidates" in result -def _write_video(path, num_frames=8, fps=4, sample_rate=8000, audio=True): +def _write_video(path, num_frames=8, fps=4, sample_rate=8000, audio=True, clip=None): from diffusers.utils.export_utils import encode_video from PIL import Image frames = [Image.new("RGB", (16, 16), (i, 0, 0)) for i in range(num_frames)] - clip = torch.full((2, int(num_frames / fps * sample_rate)), 0.25) if audio else None + if clip is None and audio: + clip = torch.full((2, int(num_frames / fps * sample_rate)), 0.25) encode_video( frames, fps=fps, @@ -700,3 +702,219 @@ def test_validation_errors_reports_it_too(self): errors = workflow.validation_errors() assert any(e["path"] == "steps[0].task.arguments.min_seconds" for e in errors) + + +# --------------------------------------------------------------------------- +# Stage B (#545): shot boundaries +# --------------------------------------------------------------------------- + + +def _quiet_track(seconds=6.0, seed=11, dbfs=-72.0): + rng = numpy.random.default_rng(seed) + return track(noise(rng, int(seconds * SR), dbfs)) + + +# Three 2 s shots at 10 fps over a 6 s track +SHOTS = [ + {"name": "shot@a", "start_frame": 0, "num_frames": 20}, + {"name": "shot@b", "start_frame": 20, "num_frames": 20}, + {"name": "shot@c", "start_frame": 40, "num_frames": 20}, +] +BOUNDS = {"shot@a": (0.0, 2.0), "shot@b": (2.0, 4.0), "shot@c": (4.0, 6.0)} + + +def _assert_inside(result, bounds): + assert result["candidates"] + for candidate in result["candidates"]: + first, end = bounds[candidate["shot"]] + assert candidate["start_seconds"] >= first - 1e-6 + assert candidate["end_seconds"] <= end + 1e-6 + + +class TestShotBoundaries: + def test_no_candidate_straddles_a_boundary(self): + result = find_loop_bed(_quiet_track(), shots=SHOTS, fps=10, max_candidates=20) + + assert result["source"]["shots_source"] == "argument" + assert result["rejected"]["shot_boundary"] > 0 + _assert_inside(result, BOUNDS) + assert [shot["name"] for shot in result["source"]["shots"]] == list(BOUNDS) + + def test_without_shots_windows_do_cross_where_the_boundaries_would_be(self): + result = find_loop_bed(_quiet_track(), max_candidates=20) + + assert result["source"]["shots_source"] is None + assert "shot_boundary" not in result["rejected"] + assert "shots" not in result["source"] + assert all(c["shot"] is None for c in result["candidates"]) + + def test_shots_shorter_than_min_seconds_find_nothing_and_say_why(self): + tiny = [ + {"name": f"s{i}", "start_frame": i * 4, "num_frames": 4} for i in range(15) + ] + result = find_loop_bed(_quiet_track(), shots=tiny, fps=10) + + assert result["candidates"] == [] + assert result["findings"][0]["rejected_by"] == "shot_boundary" + + def test_a_stretch_no_shot_covers_is_not_searched(self): + result = find_loop_bed( + _quiet_track(), + shots=[{"name": "only", "start_frame": 20, "num_frames": 20}], + fps=10, + max_candidates=20, + ) + + _assert_inside(result, {"only": (2.0, 4.0)}) + + def test_recorded_samples_narrow_a_shot(self): + shots = [ + { + "name": "a", + "start_frame": 0, + "num_frames": 30, + "start_sample": 0, + "num_samples": 2 * SR, + }, + { + "name": "b", + "start_frame": 30, + "num_frames": 30, + "start_sample": 3 * SR, + "num_samples": 3 * SR, + }, + ] + result = find_loop_bed(_quiet_track(), shots=shots, fps=10, max_candidates=20) + + _assert_inside(result, {"a": (0.0, 2.0), "b": (3.0, 6.0)}) + + def test_frames_without_a_frame_rate_are_refused(self): + with pytest.raises(ValueError, match="'fps'"): + find_loop_bed(_quiet_track(), shots=SHOTS) + + def test_sample_placed_shots_need_no_frame_rate(self): + shots = [ + {"name": "a", "start_sample": 0, "num_samples": 3 * SR}, + {"name": "b", "start_sample": 3 * SR, "num_samples": 3 * SR}, + ] + result = find_loop_bed(_quiet_track(), shots=shots, max_candidates=20) + + _assert_inside(result, {"a": (0.0, 3.0), "b": (3.0, 6.0)}) + + @pytest.mark.parametrize( + "shots, match", + [ + ("shot@a", "list"), + ([{"start_frame": 0, "num_frames": 10}], "name"), + ([{"name": "a", "start_frame": -1, "num_frames": 10}], "start_frame"), + ([{"name": "a", "start_frame": 0, "num_frames": 0}], "num_frames"), + ([{"name": "a", "start_frame": 0, "num_frames": "ten"}], "number"), + ], + ) + def test_malformed_shots_are_refused(self, shots, match): + with pytest.raises(ValueError, match=match): + find_loop_bed(_quiet_track(), shots=shots, fps=10) + + def test_fps_domain(self): + with pytest.raises(ValueError, match="'fps' above zero"): + find_loop_bed(_quiet_track(), shots=SHOTS, fps=0) + + def test_fps_is_in_the_domain_table(self): + definition = { + "steps": [ + _step( + "bed", + "find_loop_bed", + arguments={"audio": "x.wav", "fps": -1}, + ) + ] + } + errors = task_argument_errors(definition) + assert any(e["path"].endswith("fps") for e in errors) + + +class TestShotsResolution: + """argument > the video's carried shots > the run manifest > none.""" + + def _carried(self): + from dw.result import AudioVideo + + rng = numpy.random.default_rng(11) + audio = noise(rng, 6 * SR, -72.0)[None, :].astype(numpy.float32) + frames = [numpy.zeros((8, 8, 3), dtype=numpy.uint8)] * 60 + return AudioVideo(frames, audio, SR, fps=10, shots=SHOTS) + + def test_carried_shots_are_used_with_the_videos_own_rate(self): + result = find_loop_bed(self._carried(), max_candidates=20) + + assert result["source"]["shots_source"] == "artifact" + _assert_inside(result, BOUNDS) + + def test_the_argument_beats_carried_shots(self): + mine = [{"name": "mine", "start_frame": 0, "num_frames": 60}] + result = find_loop_bed(self._carried(), shots=mine, max_candidates=20) + + assert result["source"]["shots_source"] == "argument" + assert {c["shot"] for c in result["candidates"]} == {"mine"} + + def _run_with_manifest(self, tmp_path, monkeypatch): + from dw.runs import MANIFEST_FILE_NAME + from dw.shots import shot_record + + monkeypatch.setenv("DW_TRUST_WORKFLOWS", "1") + run_dir = tmp_path / "demo-workflow" / "20260923-120000-abcdef01" + (run_dir / "final").mkdir(parents=True) + # 16 frames at 4 fps, 8 kHz: two 2 s shots + rng = numpy.random.default_rng(5) + room = torch.from_numpy( + numpy.stack([noise(rng, 32000, -40.0)] * 2).astype(numpy.float32) + ) + path = _write_video( + run_dir / "final" / "cut.mp4", num_frames=16, fps=4, clip=room + ) + recorded = [ + shot_record("shot@a", 0, 8, 0, 16000), + shot_record("shot@b", 8, 8, 16000, 16000), + ] + manifest = { + "steps": [{"step": "join", "files": ["final/cut.mp4"], "shots": recorded}] + } + with open(run_dir / MANIFEST_FILE_NAME, "w") as handle: + json.dump(manifest, handle) + return path + + def test_the_run_manifest_beside_the_file(self, tmp_path, monkeypatch): + path = self._run_with_manifest(tmp_path, monkeypatch) + + result = find_loop_bed(path, max_bin_dbfs=0, max_mean_dbfs=0, max_candidates=20) + + assert result["source"]["shots_source"] == "manifest" + assert [s["name"] for s in result["source"]["shots"]] == ["shot@a", "shot@b"] + assert result["candidates"], result["findings"] + for candidate in result["candidates"]: + assert candidate["shot"] in ("shot@a", "shot@b") + assert result["rejected"]["shot_boundary"] > 0 + + def test_the_argument_beats_the_manifest(self, tmp_path, monkeypatch): + path = self._run_with_manifest(tmp_path, monkeypatch) + + result = find_loop_bed( + path, + shots=[{"name": "whole", "start_frame": 0, "num_frames": 16}], + max_bin_dbfs=0, + max_mean_dbfs=0, + ) + + assert result["source"]["shots_source"] == "argument" + + def test_a_bare_audio_file_has_no_shots(self, tmp_path): + import soundfile + + path = tmp_path / "room.wav" + rng = numpy.random.default_rng(3) + soundfile.write(path, noise(rng, 3 * SR, -72.0).astype(numpy.float32), SR) + + result = find_loop_bed(str(path)) + + assert result["source"]["shots_source"] is None + assert "shot_boundary" not in result["rejected"] From 305305a7eab2bcf69f90bf7591abaa32d1d6309a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 23:24:07 -0500 Subject: [PATCH 030/251] feat(mcp): #546 - wait for the job's estimate, not a fixed 55 s; lem launch sets DW_MCP_MAX_WAIT_SECONDS=1800 Docs and skills ask for plan.estimate plus a margin as timeout_seconds and read timeout_applied_seconds/timeout_capped back; no doc quotes a deployment cap. deploy.sh and dw-serve.service set the cap to 1800 on lem's launch; the code default stays 55. Co-Authored-By: Claude Opus 5.5 --- docs/MCP.md | 13 +-- docs/WORKFLOW_GUIDE.md | 13 ++- dw_mcp/CLAUDE.md | 4 +- plugins/dw/skills/ltx-2.5/SKILL.md | 96 +++++++++---------- plugins/dw/skills/minimax-h3/SKILL.md | 104 +++++++++++---------- plugins/dw/skills/minimax-music3/SKILL.md | 94 ++++++++++--------- plugins/dw/skills/script-to-video/SKILL.md | 4 +- plugins/dw/skills/series-episodes/SKILL.md | 2 +- scripts/deploy.sh | 6 +- scripts/dw-serve.service | 1 + 10 files changed, 177 insertions(+), 160 deletions(-) diff --git a/docs/MCP.md b/docs/MCP.md index f0af5c48..758c44de 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -302,12 +302,12 @@ references written in the same session. | Tool | Arguments | Purpose | | --- | --- | --- | -| `run_workflow(workflow_path=None, inline_workflow=None, arguments=None, acknowledged_cost=False, workspace=None, wait_seconds=0)` | exactly one of `workflow_path` (a catalog name from `list_workflows`, with or without `.json`, or a path to a workflow file on the server) or `inline_workflow`, optional `arguments`, `acknowledged_cost`, `workspace`, `wait_seconds` | Queue a workflow for generation. Returns as soon as the job is queued - unless `wait_seconds` is above 0, in which case the call then waits on the queued job exactly as `wait_for_job(job_id, timeout_seconds=wait_seconds)` would (same 55 s cap per call, clamped not honoured) and the result carries the queued-job fields plus that wait's (`status`, `still_running`, `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds`, `timeout_capped`, the slim `job`); when the cap covers the job's runtime one call is the run and the wait, and a `still_running: true` result is followed with `wait_for_job` as before. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's - `acknowledged_cost` is `true` or the bound `{fingerprint, minutes, downloads}` from the validate plan; a 409 means the plan changed and the message carries the new estimate, and nothing is waited on | +| `run_workflow(workflow_path=None, inline_workflow=None, arguments=None, acknowledged_cost=False, workspace=None, wait_seconds=0)` | exactly one of `workflow_path` (a catalog name from `list_workflows`, with or without `.json`, or a path to a workflow file on the server) or `inline_workflow`, optional `arguments`, `acknowledged_cost`, `workspace`, `wait_seconds` | Queue a workflow for generation. Returns as soon as the job is queued - unless `wait_seconds` is above 0, in which case the call then waits on the queued job exactly as `wait_for_job(job_id, timeout_seconds=wait_seconds)` would (same per-call cap, clamped not honoured) and the result carries the queued-job fields plus that wait's (`status`, `still_running`, `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds`, `timeout_capped`, the slim `job`); when the cap covers the job's runtime one call is the run and the wait, and a `still_running: true` result is followed with `wait_for_job` as before. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's - `acknowledged_cost` is `true` or the bound `{fingerprint, minutes, downloads}` from the validate plan; a 409 means the plan changed and the message carries the new estimate, and nothing is waited on | | `get_job(job_id)` | `job_id` | Get a job's status, warnings, output manifest, error and traceback; each manifest entry's `subfolder` is the in-run subfolder the step declared - by convention `final` for the deliverable, `intermediate` for scratch, `''` for none. A running job also carries `progress` (below) | | `get_job_workflow(job_id)` | `job_id` | The REST equivalent is `GET /api/jobs/{id}/workflow` (see [SERVER.md](SERVER.md#jobs-api)). The workflow the job actually ran. `realized: true` means every mutable input is pinned (arguments, seed, prompts, `output:latest`); `false` means the job predates run tracking and this is the definition as submitted. Pass it to `save_workflow` to keep it under a name | | `export_job(job_id, overwrite=False)` | `job_id`, `overwrite` | Gather one finished job into `/exports//` on the server: the realized workflow, the run's manifest, the job row, a README, and copies of the assets, earlier-run inputs and outputs. Returns the directory, a zip URL, the file list with sizes and the total. The three JSON files are in the zip, not repeated here - get_job_workflow and get_job serve them individually. **The directory is on the machine running the server**, like `download_output`'s destination. `auth_required` says whether opening the zip needs this server's bearer token, which this agent cannot attach to someone else's fetch (#353): when false, fetch `open_url` yourself and unpack it into `exports/` under the session's working directory (a deliverable, not a temp file) - the archive already unpacks into one folder named after the job id, so do not create that folder first; when true, hand `open_url` to the person instead of fetching it | | `get_job_events(job_id, after=-1, limit=200)` | `job_id`, `after`, `limit` | Get a page of a job's progress events | -| `wait_for_job(job_id, timeout_seconds=20)` | `job_id`, `timeout_seconds` | Block until a job reaches a terminal status, or `timeout_seconds` elapses. **One call blocks for at most 55 seconds** — a larger `timeout_seconds` is clamped, not honoured, because no MCP client holds a tool call open for a generation's real runtime, so budget one call per ~55s of the job. Every reply carries `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds` and `timeout_capped`, so a capped return is distinguishable from an elapsed one. Use instead of hand-polling `get_job`/`get_job_events` in a loop; if it returns `still_running: true`, call it again. Returns a slim job - status, warnings, error, `run_id` and `run_version` (the run's `v5`, as the gallery labels it), and the manifest once finished - without the arguments; `get_job` has those. A running job also carries `progress` (below) | +| `wait_for_job(job_id, timeout_seconds=20)` | `job_id`, `timeout_seconds` | Block until a job reaches a terminal status, or `timeout_seconds` elapses. **One call blocks for at most the server's cap** — 55 seconds unless the deployment sets `DW_MCP_MAX_WAIT_SECONDS` higher, and the tool's description states the live value; a larger `timeout_seconds` is clamped, not honoured. Ask for the job's `plan.estimate` plus a margin: under the cap that is one call for the whole job, over it one call per cap. A deployment raises the cap only as far as its clients (and anything between them and the server) hold one silent HTTP request open - Claude Code's limit is `MCP_TOOL_TIMEOUT`. Every reply carries `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds` and `timeout_capped`, so a capped return is distinguishable from an elapsed one. Use instead of hand-polling `get_job`/`get_job_events` in a loop; if it returns `still_running: true`, call it again. Returns a slim job - status, warnings, error, `run_id` and `run_version` (the run's `v5`, as the gallery labels it), and the manifest once finished - without the arguments; `get_job` has those. A running job also carries `progress` (below) | | `cancel_job(job_id)` | `job_id` | Ask a queued or running job to stop | | `clear_memory()` | — | Drop every loaded pipeline and the step cache, freeing VRAM/RAM immediately instead of waiting for the next job to evict one model for another. Also drops the step cache, so a seeded workflow that would otherwise reuse cached results regenerates on its next run. Refused with a 409 while a job is running or queued - the queue is FIFO, so wait for it to finish and retry rather than expecting this call to block until it does (#221) | | `rerun_job(job_id, acknowledged_cost=False, new_seed=False)` | `job_id`, `acknowledged_cost`, `new_seed` | Queue a fresh job from a previous job's stored specification. Costs GPU time, so it passes the same gate as `run_workflow`. `new_seed=true` draws a fresh seed into the workflow's seed variable — without it a seeded workflow's rerun repeats its arguments exactly and the step cache serves the whole run from the earlier one's files (`reused: true`), generating nothing. `get_job_workflow`'s `seed_variable` says whether there is one - `acknowledged_cost` is `true` or the bound `{fingerprint, minutes, downloads}` from the validate plan; a 409 means the plan changed and the message carries the new estimate | @@ -380,10 +380,11 @@ The intended loop: minutes, downloads}`), or `true` when there was no plan — pass a name straight from `list_workflows` as `workflow_path`; queues the job and returns immediately with a `job_id` -3. `wait_for_job(job_id)` to block for a bounded interval instead of - hand-polling — one call covers at most 55 seconds however large - `timeout_seconds` is, so a minutes-long render takes several; call it - again if it comes back `still_running: true` — or +3. `wait_for_job(job_id, timeout_seconds=...)` to block instead of + hand-polling, asking for the plan's estimate plus a margin; one call + covers at most the server's cap (`timeout_applied_seconds` and + `timeout_capped` say what you got), so call it again if it comes back + `still_running: true` — or `get_job_events(job_id)` repeatedly, passing back the previous call's `last_seq` as `after`, for incremental progress instead of just a terminal/not-terminal status. Each event carries `at`, seconds since the diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 54184bde..be4683e7 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -677,10 +677,13 @@ the entry an item needs. loads the same checkpoint, and its per-image figure times the number of images is the number to quote. Say "a few minutes" only when no entry with that pipeline has been measured. -5. `wait_for_job` rather than a polling loop; call it again if it returns - `still_running: true`. One call blocks for at most 55 seconds whatever - `timeout_seconds` says, so a minutes-long render takes several - the - reply's `timeout_capped` and `waited_seconds` say which happened. A +5. `wait_for_job` rather than a polling loop, asking for the job's + `plan.estimate` (in seconds, plus a margin) as `timeout_seconds`, so one + call can cover the whole run. The server caps a single call, and the + tool's description states the cap: the reply's + `timeout_applied_seconds` and `timeout_capped` say what you got, and + `waited_seconds` how long it took. Call it again if it returns + `still_running: true`. Without `timeout_seconds` a call waits 20 seconds. A running job's `progress` carries the step being run; the phase (`loading`, `generating`, `decoding`, `saving`) with the model it names in `phase_detail`; `seconds_in_phase`, time spent in that phase; and @@ -712,7 +715,7 @@ the entry an item needs. arguments={"input_audio": "output:"})` first (free; it takes an audio file or a video's muxed soundtrack directly), then `run_workflow(..., acknowledged_cost={"fingerprint": ..., "minutes": ..., - "downloads": [...]})` bound to that plan with `wait_seconds=55`, then + "downloads": [...]})` bound to that plan with `wait_seconds=60`, then `get_output_text` on the result, and `delete_output(job_id=...)` the scratch run afterward. This workflow's plan comes back `basis: "unknown"` with `minutes: null` - nothing is curated or observed diff --git a/dw_mcp/CLAUDE.md b/dw_mcp/CLAUDE.md index 7e32d580..99523732 100644 --- a/dw_mcp/CLAUDE.md +++ b/dw_mcp/CLAUDE.md @@ -73,7 +73,9 @@ changed since (`_acknowledgement_body` in `diagnose.py`; the 409 is rendered with the new estimate by `DwClient._format_detail`). The three job-queuing tools return as soon as the job is queued, since a generation outlasts any client's tool-call timeout; `run_workflow(wait_seconds=N)` then folds the first `wait_for_job` -into the same call (same `MAX_WAIT_SECONDS` clamp, same budget fields), because +into the same call (same `MAX_WAIT_SECONDS` clamp - 55 unless a deployment +raises it with `DW_MCP_MAX_WAIT_SECONDS`, and interpolated into the tool +descriptions, so no doc or skill quotes a number - same budget fields), because measured over ~1,400 agent-driven cases almost every run was followed by a wait turn of its own. `delete_output(job_id=...)` is the same economy for cleanup: the job record's `run_dir` is the `/` the diff --git a/plugins/dw/skills/ltx-2.5/SKILL.md b/plugins/dw/skills/ltx-2.5/SKILL.md index f582e0c1..d32b79f2 100644 --- a/plugins/dw/skills/ltx-2.5/SKILL.md +++ b/plugins/dw/skills/ltx-2.5/SKILL.md @@ -1,34 +1,33 @@ --- name: ltx-2.5 -description: Use when a dw MCP server is connected and the user wants LTX-2.5 video - a short clip with its own soundtrack, a clip that starts from a picture or runs between two, a sharper full-size render, a 2x upscale, a subject held across a clip from a reference sheet, a blurry or compressed clip restored, or a clip extended or chained longer. Picks the template for the shape, states the schedule and size rules, quotes cost, and carries the trained caption spec the prompt must follow. +description: Use when a dw MCP server is connected and the user wants LTX-2.5 video - a short clip with its own soundtrack, a clip from one picture or between two, a sharper full-size render, a 2x upscale, a subject held across a clip from a reference sheet, a blurry or compressed clip restored, or a clip extended or chained longer. Picks the template, states schedule and size rules, quotes cost, and carries the trained caption spec the prompt must follow. --- # LTX-2.5 on a dw server LTX-2.5 generates video and a soundtrack together, 24 fps, on a distilled schedule that is not a knob. Every template here fits a 24 GB card. This -skill chooses the template and the arguments; the prompt follows the -caption spec below. +skill picks the template and arguments; the prompt follows the caption +spec below. ## Before anything -1. `get_server_info`: the device and the workspace. On `mps` they run - slower than the CUDA `cost` says (only text-to-video has been run +1. `get_server_info`: the device and workspace. On `mps` they run + slower than CUDA `cost` says (only text-to-video has run there); on `cpu` say so and stop. 2. `list_workflows(shape="shot")`: every template in the family carries - that shape. Take their current names, `summary`, `traits` and `cost` from - the listing and trust it over the names quoted below. -3. `get_workflow` on the one chosen, for its variables and their defaults. + that shape. Take current names, `summary`, `traits` and `cost` from + the listing, trusting it over names quoted below. +3. `get_workflow` on the one chosen, for its variables and defaults. 4. Before anything near the card's ceiling - a full-size refine, 481 frames, a 2x upscale - `get_memory` on an idle server and read a `live: true` - reading's `gpu_memory_allocated_mb`. Only those are the worker's own: - `info: null` means nothing is resident, and a `live: false` reading is - cached from another moment. A non-trivial idle figure is an earlier run's - leftover and comes off what this one has. With the server idle, - `clear_memory` clears it (refused while queued or running) - it also - drops the step cache, so the next run, including a seeded rerun, is cold - and regenerates. Re-read `get_memory` to confirm; don't retry into a - failed attempt without clearing first. + reading's `gpu_memory_allocated_mb`; only those are the worker's. + `info: null` means nothing is resident, and `live: false` is cached from + another moment. A non-trivial idle figure is an earlier run's leftover, + subtracted from what this one has. With the server idle, `clear_memory` + clears it (refused while queued or running) and drops the step cache, so + the next run, even a seeded rerun, is cold and regenerates. Re-read + `get_memory` to confirm; don't retry a failed attempt unless cleared. ## Which shape is the request @@ -37,64 +36,63 @@ caption spec below. is 20s in one pass, so extend or chain only past that); `templates/ltx2/keyframes` (first and last frames pinned); `templates/ltx2/enhance-prompt` (a one-line idea plus a picture; the model's - own enhancer writes the caption). + enhancer writes the caption). - **Sharper at full size**: `templates/ltx2/two-stage` - eight sigmas at 768x448, a 2x latent upsample, then renoise and three stage-two sigmas at - 1536x896 carrying the audio latents through. The upsample alone is soft; the - refine pass is where the detail comes from. + 1536x896 carrying audio latents through. The upsample alone is soft; + the refine pass supplies the detail. - `templates/ltx2/diffusion-decode` compares decoders. Do not offer it: without a `shi-labs/natten` build its fallback OOMs on 24GB at any size. - **A generative 2x render**: `templates/ltx2/generative-upscale` draws its own - low-resolution pass and has an IC-LoRA re-render it twice the size, - inventing detail. `base_width`/`base_height` are that first render's size, + low-res pass, then an IC-LoRA re-renders it twice the size, + inventing detail. `base_width`/`base_height` are the first render's size, `width`/`height` the doubled target. - **Keeping a subject across a clip**: `templates/ltx2/reference-sheet`. The - family's only identity route, and the user has to author the reference sheet + family's only identity route; the user must author the reference sheet - one composite image, a clean panel per character, prop and location, no - text, bigger panels for what matters. What is not on the sheet will not + text, bigger panels for what matters. What isn't on the sheet won't appear. `reference_frames` must stay at or above 121. -- **Repairing the user's own footage**: `templates/ltx2/restore-deblur` for +- **Repairing the user's footage**: `templates/ltx2/restore-deblur` for spatial defocus, `templates/ltx2/restore-decompression` for low-bitrate - artefacts. These are the only templates here that read a clip dw did not - make. Each inverts one defect and no other - neither is an upscale, neither - removes motion blur or grain - so say which defect you think it is and let + artefacts. Only these templates here read a clip dw didn't make. Each + inverts one defect and no other - neither is an upscale, neither + removes motion blur or grain - so name the defect and let the user correct you. - **Longer**: `templates/ltx2/extend-clip` continues an opening conditioned on all of it, not one frame; `clip` extends an existing clip (`width`/ `height` matched, shorter than `num_frames`; clip_frames unused) instead - of generating. `templates/ltx2/chained-segments` re-runs per segment on + of generating; `templates/ltx2/chained-segments` re-runs per segment on the previous last frame and stitches. Neither is a Lightricks recipe; a single 481-frame pass reaches 20 seconds before either is needed. -If none fits, compose from `list_tasks` before authoring a new workflow, and +If none fits, compose from `list_tasks` before authoring a new workflow; read the `workflows` guide's authoring section first. ## Hard rules -- `num_frames` is `8k + 1` (121, 241, 481); `width` and `height` are in multiples of 32. - The templates generate at 24 fps. RoPE time is `frame / fps` and the model is +- `num_frames` is `8k + 1` (121, 241, 481); `width` and `height` are multiples of 32, + generated at 24 fps. RoPE time is `frame / fps` and the model is trained around 24, 25, 30 and 60, so for a higher-fps request generate at 24 (or condition at 60 at most - `MAX_CONDITIONING_FPS`, never 120) and let - playback set the rate. 48 and 96 fps are the DFR pipelines', which no - template uses. + playback set the rate. 48 and 96 fps are the DFR pipelines', unused here. - The distilled transformer runs its eight trained sigmas (`DISTILLED_SIGMA_VALUES`) - with `guidance_scale` 1.0 and STG and modality guidance off. No - `num_inference_steps`. Those knobs belong to the dev transformer, which - no 24 GB template ships. + with `guidance_scale` 1.0, STG and modality guidance off. No + `num_inference_steps`. Those knobs are the dev transformer's; no 24 GB + template ships them. - Stage two of the two-stage flow: renoise at 0.909375 (the first `STAGE_2_DISTILLED_SIGMA_VALUES` entry), three sigmas at full size. - An image condition is re-compressed at CRF 18 to match training and needs a PIL image; a multi-frame video condition is not. - Audio is generated in the first pass and nothing refines it, so carry the - audio latents (two-stage) or pair the track back (`pair_audio`) on a step - that works on frames alone. + audio latents (two-stage) or pair the track back (`pair_audio`) on a + frames-only step. ## Prompts A caption, not a tag list: one paragraph of roughly 150 to 220 words in the present progressive, opening on the action, stating for every shot a shot -type, a camera motion (say static when there is none) and a viewpoint, the -soundscape interleaved with the action rather than appended, in plain +type, a camera motion (say static when none) and a viewpoint, soundscape +interleaved with the action rather than appended, in plain observable words. For an image-conditioned clip describe only what changes from the image; restating it invites a scene cut. The `ltx2/` stored prompts (`list_prompts`) follow it. The spec, from @@ -135,17 +133,19 @@ AESTHETIC QUALITY (in addition to the above, without breaking the objective capt ## Run and judge -1. `validate_workflow` first - free, and it catches bad arguments. -2. Quote `plan.estimate` from the validate answer (whole wall clock, loading +1. `validate_workflow` first - free, and catches bad arguments. +2. Quote `plan.estimate` from the validate answer (wall clock, loading included) and name any `downloads_required` - an IC-LoRA template pulls a gated weight the box may not have. Only `text-to-video` and `two-stage` carry a `cost`; for the rest give the shape - a 121-frame clip at 960x544 - is under two minutes cold on a 24 GB card, a minute of it loading; extend + is under two minutes cold on a 24 GB card, a minute loading; extend and chain multiply by their passes. - Get the go-ahead before `run_workflow` with `acknowledged_cost` set to the + Get the go-ahead, then `run_workflow` with `acknowledged_cost` set to the plan's `{fingerprint, minutes, downloads}`. -3. `wait_for_job`, then `get_job` for the manifest. -4. Writing is still not free on a long chain, so only the steps worth writing +3. `wait_for_job` with `timeout_seconds` = the estimate plus a margin + (`timeout_capped` says the server's cap cut it; call again while + `still_running`), then `get_job` for the manifest. +4. Writing still costs on a long chain, so only worthwhile steps should: `"result": {"save": false}` on the rest, as `two-stage` does for `base` and `upscale`. Missing it is silent. What does write carries a `subfolder` - the step the user is shown `final`, every other saving step @@ -156,12 +156,12 @@ AESTHETIC QUALITY (in addition to the above, without breaking the objective capt where the prompt contradicted the image or softness where the refine pass was skipped, and `get_output_audio` for a near-silent soundtrack. Then `get_job` for the manifest and its warnings, `get_gallery_metadata` for - duration, size and whether audio is present, and hand the user the + duration, size and audio presence, and hand the user the gallery `url` (`list_gallery`, or the manifest's file name). 6. After a run worth keeping, `get_job_workflow` and `save_workflow` it, so the next run is by name not pasted JSON; `export_job` bundles it on the server. `auth_required: false` - fetch `open_url` into `exports/` under - the working directory (never a temp dir; unpacks into a job-id folder). + the working dir (never a temp dir; unpacks into a job-id folder). `true` - hand `open_url` to the person instead, keep using `get_output_image`/`_audio`/`_frames` diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index 4d2d006b..fca6bb4d 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -1,22 +1,22 @@ --- name: minimax-h3 -description: Use when a dw MCP server is connected and the user wants MiniMax H3 video - a clip with its own sound, dialogue or a speaking character, a music video, a multi-shot short with cuts, a longer take, or a subject/voice kept consistent from a reference. Picks the template, states the rules that bite, quotes cost, points at MiniMax's own prompt guides. +description: Use when a dw MCP server is connected and the user wants MiniMax H3 video - a clip with its own sound, dialogue or a speaking character, a music video, a multi-shot short with cuts, a longer take, or a consistent subject/voice from a reference. Picks the template, states the rules that bite, quotes cost, points at MiniMax's prompt guides. --- # MiniMax H3 on a dw server H3 generates video and audio together: speech with lip sync, ambient sound, -score. Every template fits a 24 GB card. This skill chooses the template and -arguments; the prompt format is MiniMax's, from their text not here. +score. Every template fits a 24 GB card. This skill picks the template and +arguments; prompt format is MiniMax's, from its text not here. ## Before anything -1. `get_server_info`: the device (H3 is CUDA; quantized configurations do - not run on mps) and which workspace this session is in. +1. `get_server_info`: the device (H3 is CUDA; quantized configs don't + run on mps) and the session's workspace. 2. `list_workflows(shape="shot")`, `list_workflows(shape="sequence")` and `list_workflows(shape="audio")`: the family's templates by current name, - `summary`, `traits` and `cost`. Trust the listing over names quoted below. -3. `get_workflow` on the one chosen, for its variables and their defaults. + `summary`, `traits`, `cost`. Trust the listing over names below. +3. `get_workflow` on the one chosen, for its variables and defaults. ## Which shape is the request @@ -28,23 +28,23 @@ arguments; the prompt format is MiniMax's, from their text not here. first and last `templates/minimax/first-and-last-frame`; last only `templates/minimax/last-frame-only`; a one-line idea plus a picture `templates/minimax/enhance-prompt-with-image`. -- **A subject that must look the same**: `templates/minimax/reference-to-video` +- **A consistent subject**: `templates/minimax/reference-to-video` (an image fixes appearance, an audio clip voice); `templates/minimax/composable-references` adds a video reference for - framing and camera, at ~3.4x the cost; + framing and camera, at ~3.4x cost; `templates/minimax/generated-subject-reference` draws the subject with - Z-Image first and references it in the same workflow; + Z-Image, references it in the same workflow; `templates/minimax/voice-timbre-reference` fixes a voice from a Bark line. -- **Several boards in one generation, one unbroken score**: - `templates/minimax/storyboard` - H3 cuts between the boards inside a single - generation, which no concat can match. One beat only; past that, use the +- **Several boards, one generation, unbroken score**: + `templates/minimax/storyboard` - H3 cuts between boards in one + generation, which no concat matches. One beat only; past that use the cuts pattern below. - **Longer than 14.4 seconds**: chain when one action or line of speech crosses the seam, cut when the scene changes (each cut its own generation). Six distinct scenes are a cuts piece, not a chain. - **As one take (a chain)**: `templates/minimax/chained-segments` (last-frame continuity), `templates/minimax/chain-video-continuity` - (previous segment's tail rides as a video reference - motion, camera and + (previous segment's tail rides as a video reference - motion, camera, voice carry the seam), `templates/minimax/chain-matched-to-audio` (a track sets the length, muxed back seamless), `templates/minimax/chain-matched-and-aligned` (per-segment prompts). Drift compounds per seam: reference the subject @@ -54,13 +54,13 @@ arguments; the prompt format is MiniMax's, from their text not here. `templates/minimax/dialogue-short` (Z-Image draws the cast, one shot per `shots` entry, `concat_videos` splices) and `templates/minimax/music-video` (a song, one slice and one lip-synced shot per entry - `from_file` reuses - an existing cast portrait and skips the drawing). + an existing cast portrait and skips drawing). `shots` is one list: a dialogue entry is `name`, `prompt`, `references` and `num_frames`; a music-video entry is `name`, `prompt`, `start_frame`. A six-shot piece is one more entry, not a new file. The listing's `lists` block says what an entry carries; its `cost` carries `per_entry` when one shot was measured - scale from that, else quote the default list's total. - A cut erases drift, and each shot makes its own audio: score with + A cut erases drift and each shot makes its audio: score with `templates/minimax/music` and `templates/assemble-and-score`'s `pair_audio` mix under the world sound. Keep one voice across shots with a repeated audio reference, not a repeated description. Duck a score burying @@ -78,8 +78,8 @@ arguments; the prompt format is MiniMax's, from their text not here. - **Music alone**: `templates/minimax/music` (Music3); the `minimax-music3` skill - its `audio_duration` is a ceiling, not a length. -If none fits, compose from `list_tasks` before authoring a new workflow, and -read the `workflows` guide's authoring section first. +If none fits, compose from `list_tasks` before writing a new workflow and +read the `workflows` guide's authoring section. ## Hard rules @@ -89,60 +89,60 @@ read the `workflows` guide's authoring section first. The 5-second floor is diffusers'; the model card says 4. - Canvas: 768-pixel short edge, at most 768x1344, in multiples of 32, aspect 1:4 to 4:1. Output audio is 32 kHz stereo. -- A checkpoint comes with a canvas, a sigma shift and an alpha, and they move +- A checkpoint comes with a canvas, a sigma shift and an alpha and they move together - change one, change all. `video_shift`, `audio_shift` and `lora_alpha` are variables everywhere, so a swap is arguments, not a file. Three combinations are tested, nothing else: 544p FL2VA turbo, 960x544, - shift 12/3, alpha unset - the default; 768p FL2VA turbo, 1344x768, shift + shift 12/3, alpha unset - default; 768p FL2VA turbo, 1344x768, shift **6**/3, alpha unset - `video-with-audio-768p`; 768p Ref2VA turbo, shift 12/3, alpha unset - every `ref2va` template. The two 768p LoRAs differ in shift; do not generalise. Never put an FL2VA LoRA on a reference template: `ref2va` holds - `transformer_ref` alone, so whatever is handed there only degrades the - output. `validate_workflow` refuses it and warns on a `weight_name` - naming neither path. + `transformer_ref` alone, so anything handed there degrades output. + `validate_workflow` refuses it and warns on a `weight_name` naming + neither path. - Fit a crowd's action to what it holds: a candle-holding crowd asked to clap rendered three-handed people. State an action the prop allows. - A directed crowd move needs its landmark in frame - "the town marches up - to the house" put the house behind camera. Track from the side with the - landmark ahead, in the direction of travel. + to the house" put the house behind camera. Track from the side with the landmark + ahead, in the direction of travel. - Two people in frame can lip-sync the wrong one, seemingly by whichever voice enters first: credit a shout to the crowd, spell the sung line in - `` tags, and frame the singer face-on at medium close-up. + `` tags, frame the singer face-on at medium close-up. - State the crowd's age range - an unstated crowd skews young. -- A three-reference shot can open on the reference portraits' own backdrop - for ~1.5s (about 1 in 25): check a shot's first frames, and say in +- A three-reference shot can open on the portraits' backdrop + for ~1.5s (about 1 in 25): check a shot's first frames and say in `subject_definitions` that the portrait's backdrop, pose and framing are not reused. - Ref2VA reaches 1344x768 too, not only the FL2VA 768p template - see the `recipes` guide's MiniMax-H3 section for the trade-off. At that canvas the 24 GB ceiling on the `17n+5` grid drops with each reference: 243 frames at one, 209 at two, 175 at three, 141 at four. `validate_workflow` projects - every `for_each` shot on its own frames and references and refuses the - largest one over budget, at that shot's path. + every `for_each` shot on its frames and references, refusing the + largest one over budget at that shot's path. - Nine steps for an eight-step LoRA: the scheduler counts sigma grid points, terminal zero included, so `denoise_total_steps` reports 8. A null `lora_model_name` drops the LoRA; raise steps and shifts too. - Nothing carries between generations except a passed reference: no latent memory, no extension mode. Identity rides on a picture, voice on an audio clip, motion/camera on a video tail (a chain's), score across cuts under - the concat. + concat. - H3 is guidance-distilled: no `guidance_scale`, no negative prompt. - Keep `release_pipeline` where the template puts it: frees Z-Image before H3 - loads, and H3 itself on `shot` before a concat runs, or a warm worker can - SIGKILL near the end where a fresh one would not. + loads, and H3 on `shot` before a concat runs, or a warm worker may + SIGKILL near the end, unlike a fresh one. - Ref2VA limits: at most 9 images, 3 videos, 3 audio clips, 12 files; audio can never be the only reference. References are labelled in order. - Write the prompt for the length generated: timestamps span the duration. ## Prompts -H3 wants Context-IR, MiniMax's own format. `get_prompt` shows the shape; -the rules come from MiniMax, not from paraphrasing one: +H3 wants Context-IR, MiniMax's format. `get_prompt` shows the shape; +the rules are MiniMax's, not a paraphrase: 1. If the `h3-prompt-writing` skill is installed, use it. If not, say once that `npx skills add MiniMax-AI/MiniMax-H3 --skill h3-prompt-writing` - installs it, and go on without it. + installs it and go on without it. 2. Else read the guides on the model card: https://huggingface.co/MiniMaxAI/MiniMax-H3/raw/main/docs/VIDEO_PROMPT_WRITING_GUIDE_base_en.md for text and frame conditioning, and @@ -152,31 +152,33 @@ the rules come from MiniMax, not from paraphrasing one: built-in enhancer writes the format from those guides. Its `idea` is framed as `Task: T2VA. Duration: 5.17 seconds. Idea: ...`. -Whichever route: write the whole script before the first shot, then place the -lines. Repeat a speaker's voice description verbatim across shots, and when a -reference picture should fix identity but not framing, say so in the prompt -itself - or every shot inherits the portrait's composition. +Either way: write the whole script before the first shot, then place +lines. Repeat a speaker's voice description verbatim across shots and when a +reference picture should fix identity but not framing, say so in the prompt - +or every shot inherits the portrait's composition. ## Run and judge -1. `validate_workflow` first - free, and it catches arguments the pipeline +1. `validate_workflow` first - free, catches arguments the pipeline rejects. 2. Quote `plan.estimate` from the validate answer (warm minutes; a first - load or a `downloads_required` is longer). When `basis` is `unknown`, - say so and give the shape instead: a 124-frame turbo clip is a few minutes + load or `downloads_required` is longer). When `basis` is `unknown`, + say so and give the shape: a 124-frame turbo clip is a few minutes on a 24 GB card, 345 frames three times that, an image reference twice a turbo clip, a video reference 3.4x again, a chain times its segments. Get the go-ahead, then `run_workflow` with `acknowledged_cost` = the plan's `{fingerprint, minutes, downloads}`. -3. `wait_for_job`, then `get_job` for the manifest. A cancelled H3 job runs - on to its next step boundary. Silence is no hang: `denoise_step` is null +3. `wait_for_job` with `timeout_seconds` = the estimate plus a margin + (`timeout_capped` says the server's cap cut it; call again while + `still_running`), then `get_job` for the manifest. A cancelled H3 job runs + to its next step boundary. Silence is no hang: `denoise_step` is null through the reference encode (~90 s; 629 s for one 5 s 960x544 video - reference on a 3090), and the block cache makes later steps uneven - + reference on a 3090) and the block cache makes later steps uneven - two-minute gaps are healthy. `phase_stall` in `get_job_events` narrates it, not a fault; judge by `denoise_step`. Each entry carries `subfolder`: `final` is the deliverable (`episode`, `music_video`, `voyage`), `intermediate` the scratch; keep that split in what you compose. -4. Judge it yourself: `get_output_frames(count=12)` for a clip's shape, +4. Judge: `get_output_frames(count=12)` for a clip's shape, `seams=true` (each later shot's start frame) for a cut's joins - a character that changes between shots (reference the same portraits everywhere), a portrait imposing its framing on every shot - `at` late in @@ -186,16 +188,16 @@ itself - or every shot inherits the portrait's composition. `get_output_text`). Dialogue gaps (`shot_dead_air`) need room tone: `find_loop_bed` the `output:` cut; `slice_audio`->`loop_audio`->`mix_audio` its pick at its `gain`. - Then `get_gallery_metadata` for duration/audio presence, and hand the + Then `get_gallery_metadata` for duration/audio presence and hand the user the gallery `url` (`list_gallery`). `get_output_image` works only on - image steps (the Z-Image portraits and boards). + image steps (Z-Image portraits and boards). 5. After a run worth keeping, `get_job_workflow` and `save_workflow` it; `export_job` bundles it on the server. `auth_required: false` - fetch `open_url` into `exports/` (never a temp dir). `true` - hand `open_url` - to the person instead and keep using `get_output_image`/`_audio`/`_frames`. + to the person and keep using `get_output_image`/`_audio`/`_frames`. ## Sources -MiniMax-H3 model card and prompt guides, `h3-prompt-writing`, the diffusers +MiniMax-H3 model card and prompt guides, `h3-prompt-writing`, diffusers' MiniMax-H3 modular pipeline, `lightx2v/Minimax-h3-Turbo` LoRA notes; read 2026-09-07; audit `docs/proposals/audits/2026-09-07-minimax-h3-audit.md`. diff --git a/plugins/dw/skills/minimax-music3/SKILL.md b/plugins/dw/skills/minimax-music3/SKILL.md index ca0ea1b8..381dfb89 100644 --- a/plugins/dw/skills/minimax-music3/SKILL.md +++ b/plugins/dw/skills/minimax-music3/SKILL.md @@ -1,6 +1,6 @@ --- name: minimax-music3 -description: Use when a dw MCP server is connected and the user wants music from MiniMax Music 3 - a song with sung lyrics, an instrumental, a score to lay under a film or a cuts piece, or the soundtrack a music video is cut to. Picks the template for the shape, states the ceiling and tag rules that bite, quotes cost, and points at MiniMax's own caption skill for the prompt. +description: Use when a dw MCP server is connected and the user wants music from MiniMax Music 3 - a song with sung lyrics, an instrumental, a score to lay under a film or a cuts piece, or the soundtrack a music video is cut to. Picks the template for the shape, states the ceiling and tag rules that bite, quotes cost, and points at MiniMax's caption skill for the prompt. --- # MiniMax Music 3 on a dw server @@ -13,25 +13,25 @@ shapes; do not author a new workflow until the shape decision below fails. ## Before anything 1. `get_server_info`: the device. On `mps` the `music` template runs, slower - than the CUDA `cost` says; `music-video` (it also runs H3) has not been + than the CUDA `cost` says; `music-video` (it also runs H3) hasn't been run there. On `cpu` stop and say so. 2. `list_workflows(shape="audio")`, and `shape="sequence"` for the music video. - Trust the names quoted below only after the listing confirms them. -3. `get_workflow` on the one chosen, for its variables and their defaults. -4. Before a long track or the music video - anything that will sit near the + Trust the names below only after the listing confirms them. +3. `get_workflow` on the one chosen, for its variables and defaults. +4. Before a long track or the music video - anything near the card's ceiling - `get_memory` with the server idle and read a `live: true` reading's `gpu_memory_allocated_mb`. Only a live reading is the worker's own - and only live readings compare with each other: `info: null` means nothing + and only live readings compare: `info: null` means nothing is resident, go ahead, while a `live: false` reading with a populated `info` is cached from another moment - it reads low while a job is loading a model, - so ask again once the server is idle rather than trusting it. A non-trivial - idle figure is what an earlier run left behind, and it comes off the ~22 GB + so ask again once idle rather than trusting it. A non-trivial + idle figure is what an earlier run left behind, coming off the ~22 GB these templates need. With the server idle, `clear_memory` clears it (refused while a job is queued or running) - it also drops the step cache, so the next run, including a seeded rerun, is cold and regenerates. Re-read `get_memory` afterwards to confirm. Don't retry into a failed attempt - without clearing first - a failed attempt is itself what leaves weight - resident, so an immediate retry starts from less than the attempt that just + without clearing first - a failed attempt is what leaves weight + resident, so an immediate retry starts from less than the attempt that failed had. ## Which shape is the request @@ -41,58 +41,58 @@ shapes; do not author a new workflow until the shape decision below fails. - **An instrumental**: the same template with a tag-only lyrics body (`[intro]`, `[instrumental]`, `[solo]`, `[outro]`, one per line) and a caption that says instrumental and names the instrument carrying the lead. The - diffusers pipeline has no instrumental flag and rejects empty lyrics, so the - tags are how it is asked for. + diffusers pipeline has no instrumental flag and rejects empty lyrics, so + tags are how it's requested. - **A score under a film or a cuts piece**: an instrumental generated to a ceiling comfortably longer than the cut, trimmed and faded with - `templates/audio-trim-fade`, then mixed under the picture the way + `templates/audio-trim-fade`, then mixed under the picture as `templates/assemble-and-score` does with `pair_audio`. Each H3 shot should - have written `non_diegetic_music: N/A` so the two scores do not fight. A + have written `non_diegetic_music: N/A` so the two scores don't fight. A score that buries a shot's voice-over is not a `world_gain` fix - see the `minimax-h3` skill's ducking recipe: `gain_audio` regions on the score - itself, one per voice-over shot, applied before it is passed as `score`. + itself, one per voice-over shot, applied before it's passed as `score`. - **A music video**: `templates/minimax/music-video`. The song is written first, `slice_audio` deals frame-exact pieces to lip-synced H3 shots, and `pair_audio` lays the unbroken track back over the edit. The ceiling must - exceed the total sliced length with real margin, not by a fraction of a + exceed the total sliced length by a real margin, not a fraction of a second, since the model may stop early. If none fits, compose from `list_tasks` (`slice_audio`, `fade_audio`, -`pair_audio`, `mix_audio`, `concat_videos`) before authoring, and read the +`pair_audio`, `mix_audio`, `concat_videos`) before authoring; read the `workflows` guide's authoring section first. ## Hard rules - `audio_duration` is a ceiling, not a target: the language model stops when - the song ends, and generation is cut at the ceiling if it has not. Default + the song ends, and generation is cut at the ceiling if it hasn't. Default 60 seconds. Ask for more than the piece needs and trim; the run's time - follows the length actually generated, so the margin is free. + follows the length generated, so the margin is free. The consequence to size by: the arc the caption describes stretches to fill - the budget it is given, so a ceiling set at the intended length is not a - piece that ends early with room to spare - it is the same piece pulled out - to the ceiling and then cut off at it. A caption written for about 20 + its given budget, so a ceiling set at the intended length isn't a + piece that ends early with room to spare - it's the same piece stretched + to the ceiling and cut off there. A caption written for about 20 seconds under a 22-second ceiling comes back stretched and truncated. Set the ceiling to at least 1.5x the length wanted and trim with - `templates/audio-trim-fade`. This is also why a track whose - `duration_seconds` lands within 0.2 s of its ceiling should be read as cut + `templates/audio-trim-fade`. This is why a track whose + `duration_seconds` lands within 0.2 s of its ceiling reads as cut off rather than finished. - The engine caps a track at 9000 frames at 25 frames per second, 360 seconds. MiniMax supports five minutes; stay at or under 300. - The caption is capped at 5,000 tokens and a longer one is an error, not a truncation. A 250-450 word caption is nowhere near it; a pasted pile of example captions is. -- Lyrics: a section tag stands alone on its line and is lower-cased on the way - in; any words on the same line as a leading tag are dropped. The vocabulary +- Lyrics: a section tag stands alone on its line and is lower-cased on input; + any words on the same line as a leading tag are dropped. The vocabulary on the model card is `[Intro]`, `[Verse]`, `[Pre-Chorus]`, `[Chorus]`, `[Post-Chorus]`, `[Bridge]`, `[Instrumental]`, `[Solo]`, `[Outro]`. A tag can - carry a local direction for its section; the caption's arrangement text is - where that direction is spelled out. + carry a local direction for its section, spelled out in the caption's + arrangement text. - Name the vocal gender and timbre in the caption, or the model may drift instrumental. An instrumental says so in the same place. - Output is 44.1 kHz stereo, the vocoder's native rate. The model card's 32 kHz is what MiniMax's reference server resamples to; the templates' result - declares `sample_rate: 44100` because the modular output carries no rate of - its own. + declares `sample_rate: 44100` because the modular output carries no rate + itself. - The flow stage denoises in 200-frame windows at 30 steps each, with classifier-free guidance fixed at 1.7 on the pipeline's guider. Neither is a call argument; leave them unless a template exposes `num_inference_steps`. @@ -103,13 +103,13 @@ If none fits, compose from `list_tasks` (`slice_audio`, `fade_audio`, vendor says tempo, key, structure and lyrics may not match every detail. Iterate at 30-60 seconds before asking for a long track. - Music 3 ignores per-section singer directions. Before staging lip-sync on - a song with two or more singers, run `attribute_voices` with a reference + a multi-singer song, run `attribute_voices` with a reference span per singer and cast each shot from its answer. Never infer the singer from pitch: a tenor and a mezzo share the range. ## Prompts -The caption format is MiniMax's own. Do not invent it and do not paraphrase it +The caption format is MiniMax's own. Don't invent or paraphrase it from the templates' examples: 1. If the `music-caption-rewriter` skill is installed (MiniMax ships it in @@ -117,24 +117,24 @@ from the templates' examples: router, 18 family indexes and 1,000 example captions), use it. If it is not, tell the user once that `npx skills add MiniMax-AI/MiniMax-Music3 --skill music-caption-rewriter` - installs it; this is the family where the local skill pays off, since the - templates are what a fetch of `SKILL.md` alone does not reach. + installs it; this is the family where the local skill pays off, since a + fetch of `SKILL.md` alone doesn't reach the templates. 2. Else read its `SKILL.md` and `references/genre-router.md` at that path. The contract is three headings in order - Global Metadata, Vocal Details, Arrangement - in 250-450 English words, with no title, no reasoning, and no lyric line copied into the caption. The pipeline strips markdown - headings and emphasis on the way in, so the vendor's caption pastes + headings and emphasis on input, so the vendor's caption pastes straight into `prompt`. This is the recommended route: a 24-shot field report's three-heading caption produced a correct, complete song on its first seed (#484). 3. The concise one-paragraph form the templates' stored prompts use (genre, BPM, key, emotional progression, listening scenario, production profile, - vocals, arrangement) is the model card's own example and works; the + vocals, arrangement) is the model card's example and works; the three-heading form is for precise control. Both are on the shelf: `list_prompts(intended_model="minimax-music3")` names them and - `get_prompt` reads one. Read the exemplar before writing a caption - it - is what the format looks like when it is right, which is not the same as - a source to paraphrase the rules from. + `get_prompt` reads one. Read the exemplar before writing a caption - it's + what the format looks like when right, not a source to paraphrase the + rules from. Lyrics carry the structure: tags for the sections, the words to sing under them, and nothing else. The tag list is the model card's "Fine-Grained Music @@ -148,10 +148,12 @@ Control" section. was measured on; a first load is longer). When `basis` is `unknown`, say so and give the shape of the spend: the autoregressive stage runs at 25 frames per second of audio and dominates, so time scales with the length the model - actually sings, not the ceiling. Get the user's go-ahead before + sings, not the ceiling. Get the user's go-ahead before `run_workflow` with `acknowledged_cost` set to the plan's `{fingerprint, minutes, downloads}`. -3. `wait_for_job`, then `get_job` for the manifest. Each manifest entry +3. `wait_for_job` with `timeout_seconds` = the estimate plus a margin + (`timeout_capped` says the server's cap cut it; call again while + `still_running`), then `get_job` for the manifest. Each manifest entry carries `subfolder`: `templates/minimax/music-video` puts the cut in `final` and the song, the singer's portrait and each shot in `intermediate`, and `list_gallery(subfolder="final")` lists only @@ -161,7 +163,7 @@ Control" section. 4. Judge it yourself. `get_gallery_metadata` for duration and sample rate: `media.duration_seconds` within 0.2 s of `audio_duration` means the ceiling cut the track (raise it and rerun); well short of it means the - song finished on its own. Its `peak_dbfs` is a single sample and does not + song finished on its own. Its `peak_dbfs` is a single sample and doesn't say how loud the song reads end to end - `integrated_lufs` (BS.1770, whole-track) is the field for that, and what `normalize_audio`'s `target_lufs` targets when a mix must match another track by ear, not @@ -169,19 +171,19 @@ Control" section. `limit: true`; `limiter_heavy` means lower the target. Then listen with `get_output_audio` (a long track in `start`/`duration` excerpts) for the family's failure modes: a - song that went instrumental (name the vocals in the caption), an ending - cut mid-note (raise the ceiling, then trim), a structure that ignored the + song gone instrumental (name the vocals in the caption), an ending + cut mid-note (raise the ceiling, then trim), a structure ignoring the tags (fewer sections, plainer directions). Hand the user the gallery `url` (`list_gallery`, or the manifest's file name). 5. To use the track in a later workflow, `keep_output` makes it an `asset:`; to trim it in the same run, chain `templates/audio-trim-fade` on the output. 6. After an inline run worth keeping, `get_job_workflow` and `save_workflow` it, - so the next run is by name rather than by pasting JSON; `export_job` bundles + so the next run is by name rather than pasting JSON; `export_job` bundles the run — workflow, manifest, job row and media — for git, on the server. If `auth_required` is false, fetch `open_url` and unpack it into `exports/` under the session's working directory, never a temp directory (the archive already unpacks into a job-id folder, don't make one first). - If true, this agent can't attach the token itself - hand `open_url` to + If true, this agent can't attach the token - hand `open_url` to the person, and keep working via `get_output_image`/`get_output_audio`/`get_output_frames`. diff --git a/plugins/dw/skills/script-to-video/SKILL.md b/plugins/dw/skills/script-to-video/SKILL.md index 0ae59fa8..a72fdc52 100644 --- a/plugins/dw/skills/script-to-video/SKILL.md +++ b/plugins/dw/skills/script-to-video/SKILL.md @@ -66,7 +66,9 @@ must not do even when running unattended. ## Run and judge -After the job completes, read `get_job_events` / `get_gallery_metadata` for +Wait with `wait_for_job`, asking for the plan's estimate plus a margin as +`timeout_seconds`; `timeout_capped` says the server's cap cut it short, and +a `still_running` reply means call again. After the job completes, read `get_job_events` / `get_gallery_metadata` for the warnings the engine already emits (`audio_no_headroom`, `audio_clipped`, an elision diagnostic) and `rerun_job(new_seed=True)` a shot that reads wrong - not by inventing a new heuristic, by reading what is already diff --git a/plugins/dw/skills/series-episodes/SKILL.md b/plugins/dw/skills/series-episodes/SKILL.md index 403b909a..0b709585 100644 --- a/plugins/dw/skills/series-episodes/SKILL.md +++ b/plugins/dw/skills/series-episodes/SKILL.md @@ -166,7 +166,7 @@ returns sound, not text: `validate_workflow(name="templates/transcribe-audio", arguments={"input_audio": "output:"})` first (free; it takes the episode's muxed soundtrack directly), then `run_workflow(..., acknowledged_cost=, -wait_seconds=55)`, then `get_output_text` on the result, and +wait_seconds=60)`, then `get_output_text` on the result, and `delete_output(job_id=...)` the scratch run afterward. This workflow's plan is `basis: "unknown"` with `minutes: null` - quote seconds, not minutes; it runs in a few seconds. diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 0e6b952c..7387af89 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -31,6 +31,7 @@ # Environment overrides, all optional: # DW_DIR (checkout, default ~/diffusers-workflow), DW_TOKEN (default xyz), # DW_PORT (8765), DW_WORKSPACE (~/diffusers-workspace), DW_HOST (0.0.0.0), +# DW_MCP_MAX_WAIT_SECONDS (1800, the per-call wait_for_job cap), # DW_NODE_BIN (the directory holding npm, when it is not on a # non-interactive PATH and not in one of the places find_npm looks). set -euo pipefail @@ -40,6 +41,9 @@ DW_TOKEN="${DW_TOKEN:-xyz}" DW_PORT="${DW_PORT:-8765}" DW_HOST="${DW_HOST:-0.0.0.0}" DW_WORKSPACE="${DW_WORKSPACE:-$HOME/diffusers-workspace}" +# One MCP wait_for_job call may block this long (dw_mcp/diagnose.py; the code +# default is 55). #377: one call covers a long render. +DW_MCP_MAX_WAIT_SECONDS="${DW_MCP_MAX_WAIT_SECONDS:-1800}" LOG="$HOME/dw-serve.log" SCREEN_SESSION="dw-serve" HEALTH="http://localhost:$DW_PORT/api/health" @@ -172,7 +176,7 @@ else fi # start the new one in its own screen window -cmd="cd $DW_DIR && source venv/bin/activate && exec python -m dw.serve --host $DW_HOST --port $DW_PORT --mcp --token $DW_TOKEN --workspace $DW_WORKSPACE --examples-dir $DW_DIR/workflows >> $LOG 2>&1" +cmd="cd $DW_DIR && source venv/bin/activate && DW_MCP_MAX_WAIT_SECONDS=$DW_MCP_MAX_WAIT_SECONDS exec python -m dw.serve --host $DW_HOST --port $DW_PORT --mcp --token $DW_TOKEN --workspace $DW_WORKSPACE --examples-dir $DW_DIR/workflows >> $LOG 2>&1" if screen -ls | grep -q "\.${SCREEN_SESSION}[[:space:]]"; then screen -S "$SCREEN_SESSION" -X screen -t serve bash -c "$cmd" else diff --git a/scripts/dw-serve.service b/scripts/dw-serve.service index 4501ce61..aac43af6 100644 --- a/scripts/dw-serve.service +++ b/scripts/dw-serve.service @@ -17,6 +17,7 @@ After=network.target WorkingDirectory=%h/diffusers-workflow Environment=VIRTUAL_ENV=%h/diffusers-workflow/venv Environment=PATH=%h/diffusers-workflow/venv/bin:/usr/local/bin:/usr/bin:/bin +Environment=DW_MCP_MAX_WAIT_SECONDS=1800 ExecStart=%h/diffusers-workflow/venv/bin/python -m dw.serve --host 0.0.0.0 --port 8765 --mcp --token xyz --workspace %h/diffusers-workspace --examples-dir %h/diffusers-workflow/workflows KillSignal=SIGTERM TimeoutStopSec=30 From b8499c32f07093789a0c0dce0be68a69e75f4d25 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 23:26:21 -0500 Subject: [PATCH 031/251] docs: #218 - find_loop_bed design record and todo entry Co-Authored-By: Claude Opus 5.5 --- .../complete/find-loop-bed-complete.md | 122 ++++++++++++++++++ docs/proposals/todo.md | 4 + 2 files changed, 126 insertions(+) create mode 100644 docs/proposals/complete/find-loop-bed-complete.md diff --git a/docs/proposals/complete/find-loop-bed-complete.md b/docs/proposals/complete/find-loop-bed-complete.md new file mode 100644 index 00000000..4bebcb5d --- /dev/null +++ b/docs/proposals/complete/find-loop-bed-complete.md @@ -0,0 +1,122 @@ +# `find_loop_bed`: ranked room-tone loop windows (#218) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out, +2026-09-27). Plan v1 approved by Don with every default (Q1-Q5) on +2026-09-27; stages #544 (A) and #545 (B), both verified. + +## The report + +Known Issues episodes 7 and 8 (2026-09-25) needed a room-tone bed under H3 +dialogue cuts, and the session found one by downloading the WAVs and +running numpy locally, because the metadata envelope has only 1 s bins +(#464, closed as a duplicate). #465 shows that H3 dialogue shots hold +0.5-2 s at -70 to -84 dBFS *inside* the shot, so every dialogue cut needs a +bed. Picking the window has three traps a level check misses: + +1. **Near-programme material**: speech about 30 dB down reads as quiet but + is audible once it repeats. +2. **Lap-rate modulation**: the loop's repeat rate beats against the + source's own level movement, which one pass through the source can't show. +3. **Ticks**: a 1 ms transient is invisible to 50 ms RMS and recurs once per + lap. Episode 8's first pick held one 19.7 dB over the median 1 ms peak. + +The field thresholds (every 50 ms bin at or below -55 dBFS, mean at or below +-60, 1 ms spike margin at or below 12 dB, ranked on the *looped* result's +50 ms ripple) became the task's defaults. + +## Decisions (plan v1, approved as written) + +- **Q1: not an assessment probe.** It is a search, not a check of a cut. + `register_command` already had the `assessment` flag (from #485, default + False), so no registry change was needed. The three-probe test is + unchanged. +- **Q2: rank by looped `ripple_db` alone.** `envelope_peak_db` is a reading, + with a `lap_modulation` warning above -15 dB. No combined score. +- **Q3: shots in stage B.** +- **Q4: task only.** No `assess_output` probe and no sync route, so the + one-step workflow queues behind a running GPU job. +- **Q5: no in-workflow wiring into `slice_audio`.** Candidates carry + `start_seconds`/`duration_seconds` (`slice_audio`'s names) and `gain` + (`mix_audio`'s linear multiplier), so the agent copies the numbers. +- **No separate tremolo estimator.** The envelope spectrum of the looped + result captures the beat whatever causes it. + +## What was built + +**Stage A, #544** (server): `dw/tasks/loop_bed.py`, registered with +`returns="json"`; domain entries in `dw/task_domains.py`; an FFT +`_harmonicity` (numerically equal to the old `numpy.correlate`, `bleed_join` +unchanged); `decode_soundtrack` in `dw/media_audio.py`; a `docs/TASKS.md` +section, with the room-tone recipe now starting from the task. + +- Arguments: `audio` (path, `AudioTrack` or `AudioVideo`), + `start_seconds`/`end_seconds`, `min_seconds` 0.5, `max_seconds` 2.0, + `max_bin_dbfs` -55, `max_mean_dbfs` -60, `max_spike_db` 12, `crossfade_ms` + 250 (`loop_audio`'s default), `loop_seconds` 10, `target_bed_dbfs` -60, + `max_candidates` 5. +- Output: `source`, `criteria`, ranked `candidates` (each with its level, + spike, flatness/harmonicity and `looped` readings, `gain_db`/`gain`, + `warnings`), `rejected` counts, and one `no_loop_bed` finding (with + `rejected_by`) when nothing survives. No candidates is an answer, not a + failed job. +- Deviations from the plan, as built: + - A video's soundtrack is read as float32 from the audio stream alone + (`decode_soundtrack`), not through `extract_audio`, whose s16 output + requantizes -70 to -85 dBFS room tone near 16-bit's floor. + - `rejected` is a true tally: every (start, length) window on the grid is + counted once, under the first rule it fails (too_loud, silent, spike, + tonal). + - The tonal test (#198's flatness < 0.3 or harmonicity >= 0.45, thresholds + unchanged) runs over every 0.1 s and 0.2 s block inside a window, not + the whole window: pauses between syllables dilute intermittent speech + below the threshold. A candidate reports its worst block's readings. + - Flatness is measured over the band the source occupies + (`_occupied_rate`), the #198 mechanism: an upsampled source's empty top + band otherwise reads as tonal whatever the material. + - The spike check also reads 5 ms just outside each end of a window, so a + window ending on a click is rejected. + - The looped measurement runs on a fixed pool of 200 non-overlapping + survivors (`LOOPED_POOL`), so `max_candidates` only truncates the final + ranking and a smaller value returns a prefix of a larger one. + - `criteria` also echoes `min_seconds`, `max_seconds` and + `target_bed_dbfs`. + +**Stage B, #545** (server + plugin): `shots` (the probes' +`{name, start_frame, num_frames}` shape, optionally with samples) and `fps`. +They resolve as the argument (`"argument"`), then shots a `previous_result:` +video carries (`"artifact"`), then the run manifest beside the file +(`"manifest"`), else `null`. With shots, no candidate crosses a boundary, +each names its `shot`, `source.shots` places each shot in seconds, and +`rejected.shot_boundary` counts the windows a boundary cut through (present +only when shots resolved). Without shots the output is stage A's. #465's +`shot_dead_air` message now names `find_loop_bed`, and so do the +`minimax-h3` and `series-episodes` skills (no new numbers). + +## Bounces per stage + +| Stage | Bounces | Cause | +|---|---|---| +| A, #544 | 2 | 1: `max_candidates` cut the pool before the looped ranking (C-F163); a window ending on the click; `rejected` not a tally and whole-window tonality missing a voice under the bed (C-F161). 2: C-F161 still failed; the upsampled fixture's empty band read as tonal, fixed with occupied-band flatness and 0.1 s blocks. | +| B, #545 | 0 | | + +No stage comment names `usage:` figures, so per-stage cost is not recorded. +The plan estimated about $7-11 total. + +Open against the suite, not the code: harnest#37 (relax C-F164's +`rejected` equality across a lossy re-encode) and the tester's C-F168 +fixture amendment (assemble-and-score normalizes the film to about -24 dBFS, +so the case's bed is no longer quiet). Both cases keep their `pending:` lines +until those are ruled on. + +## Deferred + +- **A sync route or `assess_output` probe** (Q4). Back if the queue wait + behind a GPU job hurts in practice. +- **Wiring a candidate into `slice_audio`** inside one workflow (Q5), which + would be a syntax change. +- **A combined ranking score** (Q2). The ranking is fitted to two field + cases; every reading is returned so an agent can re-rank. +- **Thresholds for louder rooms.** The defaults come from one programme (H3 + dialogue). An ambience-heavy LTX shot may reject everything at -55/-60; + the arguments and the `no_loop_bed` finding's `rejected_by` are the + mitigation until a second programme gives numbers. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 79fe3f49..91404046 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -64,6 +64,10 @@ remaining deferred fix is recorded in #513-#514), shipped 2026-09-27. Record, including what was deferred (a template, register-external-output, the `concat_videos` silent-input desync filed as #553): `complete/join-into-song-complete.md`. +- **`find_loop_bed`, ranked room-tone loop windows** (#218, stages + #544-#545), shipped 2026-09-27. Record, including what was deferred (a + sync route, in-workflow wiring into `slice_audio`, thresholds for louder + rooms): `complete/find-loop-bed-complete.md`. ## Declined From 4a826ae8bac0cd4ac8250561c31087064b97e27e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 23:40:19 -0500 Subject: [PATCH 032/251] feat(catalog): #548 - templates/ltx2/upscale-clip, generative 2x upscale of the caller's clip Pins the Pixel-Spatial-Upscaler card (factor 2, strength 1.0) for both upscale-clip and generative-upscale; pair_audio reads an mp4's track. Co-Authored-By: Claude Opus 5.5 --- tests/test_catalog_structure.py | 5 +- tests/test_ltx2_ic_loras.py | 77 +++++++- tests/test_pair_audio.py | 38 ++++ workflows/templates/ltx2/upscale-clip.json | 197 +++++++++++++++++++++ 4 files changed, 309 insertions(+), 8 deletions(-) create mode 100644 workflows/templates/ltx2/upscale-clip.json diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index 37c02693..e08c23b0 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -408,7 +408,10 @@ def test_no_stale_entry_in_the_allowlist(): # at 8_762 - the cost of a real diffusers refusal (not a floor like # `num_frames`) being checked before the pipeline loads rather than 80s # after it. -COMPACT_BUDGET = 8_850 +# Then to 9_000 for `templates/ltx2/upscale-clip` (#548), measured at 8_910: +# about 125 tokens, the same as each of its restore siblings - the only +# catalog route to a generative upscale of a clip the caller brings. +COMPACT_BUDGET = 9_000 FILTERED_BUDGET = 1_500 diff --git a/tests/test_ltx2_ic_loras.py b/tests/test_ltx2_ic_loras.py index f1503574..01176c39 100644 --- a/tests/test_ltx2_ic_loras.py +++ b/tests/test_ltx2_ic_loras.py @@ -8,7 +8,12 @@ here rather than trusted to a reading. Sources, read from `lem` with its own Hugging Face token on 2026-09-14: -Lightricks/LTX-2.5-22b-IC-LoRA-{Ingredients,Deblur,Decompression}. +Lightricks/LTX-2.5-22b-IC-LoRA-{Ingredients,Deblur,Decompression}; and +Lightricks/LTX-2.5-22b-IC-LoRA-Pixel-Spatial-Upscaler on 2026-09-27 (#548). +The upscaler's card states the weight, factor 2 and strength 1.0 but no +trained bucket and no caption convention: its entry below pins the +family's 960x544 / 121 defaults, a 480x272 source doubled, which is the +card's "~280p" base run through the 2x. """ import json @@ -50,6 +55,15 @@ 121, 1.0, ), + "upscale-clip": ( + "Lightricks/LTX-2.5-22b-IC-LoRA-Pixel-Spatial-Upscaler", + "ltx-2.5-22b-ic-lora-pixel-spatial-upscaler-x2-1.0.safetensors", + 2, + 960, + 544, + 121, + 1.0, + ), } @@ -81,8 +95,8 @@ def test_the_strength_is_the_cards_default(self, name): assert variables[lora["scale"].removeprefix("variable:")] == strength def test_the_reference_is_encoded_at_the_output_resolution(self, name): - """Every one of these three states downscale factor 1 - unlike the - spatial upscaler, whose reference is half size.""" + """The conditioning LoRAs state downscale factor 1; the spatial + upscaler states 2, its reference half the output's size.""" _repo, _weight, factor, *_ = CARDS[name] arguments = conditioned_step(name)["pipeline"]["arguments"] @@ -131,11 +145,35 @@ def test_the_default_sheet_length_satisfies_it(self): assert variables["reference_frames"] >= 121 -class TestTheRestorationTemplatesReadFootageTheyDidNotGenerate: - """The point of both: a reference the workflow did not make. Every other - IC-LoRA use in this catalog conditions on an earlier step's output.""" +class TestTheGenerativeUpscaleMatchesTheSameCard: + """`generative-upscale` runs the same upscaler on a clip it generated + first; its numbers are the card's as much as `upscale-clip`'s are.""" + + def test_it_loads_the_upscaler_at_factor_two(self): + repo, weight, factor, *_ = CARDS["upscale-clip"] + pipeline = conditioned_step("generative-upscale")["pipeline"] + (lora,) = pipeline["loras"] + + assert lora["model_name"] == repo + assert lora["weight_name"] == weight + assert lora["scale"] == 1.0 + assert pipeline["arguments"]["reference_downscale_factor"] == factor + + def test_its_output_is_twice_its_base(self): + variables = definition("generative-upscale")["variables"] + + assert variables["width"] == 2 * variables["base_width"] + assert variables["height"] == 2 * variables["base_height"] - @pytest.mark.parametrize("name", ["restore-deblur", "restore-decompression"]) + +class TestTheTemplatesThatReadFootageTheyDidNotGenerate: + """A reference the workflow did not make: the caller's own clip, named + by `source_video`. `reference-sheet` and `generative-upscale` condition + on an earlier step's output instead.""" + + @pytest.mark.parametrize( + "name", ["restore-deblur", "restore-decompression", "upscale-clip"] + ) def test_the_reference_is_the_callers_own_clip(self, name): (reference,) = conditioned_step(name)["pipeline"]["arguments"][ "reference_conditions" @@ -164,3 +202,28 @@ def test_the_default_prompt_is_in_the_trained_form(self, name, marker): assert "Reference shows" in text assert marker in text + + +class TestTheUpscaledClipKeepsItsSourcesSoundtrack: + """#548: the deliverable carries the caller's own track, cut to the + output's length, not the one the IC-LoRA pass generates beside it.""" + + def test_the_deliverable_pairs_the_upscale_with_the_source(self): + upscaled, paired = definition("upscale-clip")["steps"] + + assert upscaled["result"]["subfolder"] == "intermediate" + assert paired["task"]["command"] == "pair_audio" + assert paired["task"]["arguments"] == { + "video": "previous_result:upscaled", + "audio": "variable:source_video", + "fit": "video", + } + assert paired["result"]["subfolder"] == "final" + + def test_the_default_prompt_is_a_literal_the_caller_replaces(self): + """The card states no caption convention, so there is no trained + form to store; the description says to describe the source.""" + workflow = definition("upscale-clip") + + assert not workflow["variables"]["prompt"].startswith("prompt:") + assert "Replace the default prompt" in workflow["description"] diff --git a/tests/test_pair_audio.py b/tests/test_pair_audio.py index f9375859..fde74d8a 100644 --- a/tests/test_pair_audio.py +++ b/tests/test_pair_audio.py @@ -35,6 +35,44 @@ def test_pairs_frames_with_a_soundtrack_read_from_a_file(tmp_path): assert paired.sample_rate == 16000 +def test_takes_the_soundtrack_of_an_mp4_file(tmp_path): + """#548: templates/ltx2/upscale-clip names the caller's own clip as the + 'audio', so the deliverable carries the original track rather than one + the IC-LoRA pass invented. The track is read out of the mp4 itself.""" + from diffusers.utils.export_utils import encode_video + + path = tmp_path / "clip.mp4" + sample_rate = 16000 + encode_video( + [Image.new("RGB", (16, 16)) for _ in range(8)], + fps=4, + output_path=str(path), + audio=torch.full((2, 2 * sample_rate), 0.25), + audio_sample_rate=sample_rate, + ) + + paired = pair_audio( + [Image.new("RGB", (32, 32)) for _ in range(4)], str(path), fps=4, fit="video" + ) + + assert paired.sample_rate == sample_rate + assert paired.audio.shape == (2, sample_rate) + assert numpy.abs(paired.audio).max() > 0.1 + + +def test_an_mp4_without_a_soundtrack_is_named_as_the_fault(tmp_path): + """A silent source fails the pairing step with an error that says why.""" + from diffusers.utils.export_utils import encode_video + + path = tmp_path / "silent.mp4" + encode_video( + [Image.new("RGB", (16, 16)) for _ in range(4)], fps=4, output_path=str(path) + ) + + with pytest.raises(ValueError, match="carries no audio track"): + pair_audio(["frame"], str(path), fps=4, fit="video") + + def test_pairs_frames_with_a_bare_waveform(): paired = pair_audio(["frame1", "frame2"], _waveform(), sample_rate=24000) diff --git a/workflows/templates/ltx2/upscale-clip.json b/workflows/templates/ltx2/upscale-clip.json new file mode 100644 index 00000000..988d4002 --- /dev/null +++ b/workflows/templates/ltx2/upscale-clip.json @@ -0,0 +1,197 @@ +{ + "id": "LTX2UpscaleClip", + "description": "Re-renders a clip from the asset library at twice its size with the spatial-upscaler in-context LoRA, the clip attached as the reference for the whole denoise - so composition, motion and subject carry over while fine detail (texture, edges, micro-contrast) is invented, not interpolated. It is a generative re-render, not a faithful resize: the vendor scopes it as a creative step, not for live-action footage where factual fidelity matters, and a heavily compressed source is 'restore-decompression''s job first. The source is scaled and center-cropped to width/2 x height/2 with no error, so set 'width'/'height' to twice the source's size, keeping its aspect ratio - a different aspect ratio is silently cropped and a larger source silently downscaled. 'num_frames' must not exceed the source's length: the reference is trimmed to it, and frames past the end of a shorter source are generated with no reference. Read the source's size, frame rate and length first with get_gallery_metadata('asset:...'). Defaults fit a 480x272, 121-frame, 24 fps source. Replace the default prompt with a description of the source's subject, setting, motion and lighting - the vendor card states no caption convention, and the prompt is what anchors the re-render to the scene. 'lora_scale' is 1.0 because the weights ship pre-scaled. The upscale is saved to intermediate/; the deliverable in final/ carries the source's own soundtrack cut to the output length, so a source with no audio fails at that step, after the upscale is saved.", + "summary": "2x generative re-render of your own clip, soundtrack kept. width/height = 2x the source's; num_frames <= its length.", + "cost_drivers": [ + "num_frames", + "width", + "height" + ], + "variables": { + "transformer_weights_dtype": "{uint4}", + "text_encoder_weights_dtype": "{int8}", + "source_video": "asset:clip.mp4", + "prompt": "A high-resolution rendering of the scene in the reference video, with the same subject, framing and motion, and sharp, fine detail.", + "negative_prompt": "constant:diffusers.pipelines.ltx2.utils.DEFAULT_NEGATIVE_PROMPT", + "width": 960, + "height": 544, + "num_frames": 121, + "frame_rate": 24.0, + "lora_scale": 1.0, + "seed": 42 + }, + "variable_constraints": { + "width": { + "modulus": 32, + "remainder": 0, + "reason": "LTX-2.5's pipelines refuse a width not divisible by 32 outright (ValueError, not a silent floor), so it is checked here before the pipeline load rather than after it" + }, + "height": { + "modulus": 32, + "remainder": 0, + "reason": "LTX-2.5's pipelines refuse a height not divisible by 32 outright (ValueError, not a silent floor), so it is checked here before the pipeline load rather than after it" + }, + "num_frames": { + "modulus": 8, + "remainder": 1, + "min_frames": 9, + "reason": "LTX-2.5's video VAE encodes 8 * n + 1 frames, so an off-grid count is not refused by the pipeline but floored to the grid below - a shorter clip than the one asked for. Refused here instead: no 'snap', because rounding a length the other way from the pipeline's own would be a second silent change" + } + }, + "seed": "variable:seed", + "steps": [ + { + "name": "upscaled", + "pipeline": { + "configuration": { + "component_type": "LTX2InContextPipeline", + "pre_load_modules": [ + "sdnq" + ], + "components": { + "transformer": { + "device": "cuda" + }, + "text_encoder": { + "group_offload": { + "offload_type": "leaf_level", + "use_stream": true + } + }, + "connectors": { + "group_offload": { + "offload_type": "leaf_level", + "use_stream": true + } + }, + "vae": { + "device": "cuda" + }, + "audio_vae": { + "device": "cuda" + }, + "vocoder": { + "device": "cuda" + } + }, + "vae": { + "enable_tiling": true + } + }, + "from_pretrained_arguments": { + "model_name": "Lightricks/LTX-2.5-Diffusers", + "torch_dtype": "torch.bfloat16" + }, + "transformer": { + "configuration": { + "component_type": "LTX2VideoTransformer3DModel", + "preserve_device_placement": true + }, + "quantization_config": { + "configuration": { + "config_type": "sdnq.SDNQConfig" + }, + "arguments": { + "weights_dtype": "variable:transformer_weights_dtype", + "quantization_device": "cuda", + "return_device": "cuda", + "use_quantized_matmul": true, + "dequantize_fp32": false + } + }, + "from_pretrained_arguments": { + "model_name": "Lightricks/LTX-2.5-Diffusers", + "subfolder": "transformer", + "torch_dtype": "torch.bfloat16" + } + }, + "text_encoder": { + "configuration": { + "component_type": "transformers.Gemma4UnifiedForConditionalGeneration", + "preserve_device_placement": true + }, + "quantization_config": { + "configuration": { + "config_type": "sdnq.SDNQConfig" + }, + "arguments": { + "weights_dtype": "variable:text_encoder_weights_dtype", + "quantization_device": "cuda", + "return_device": "cpu", + "dequantize_fp32": false, + "modules_to_not_convert": [ + "embed_vision", + "embed_audio", + "lm_head" + ] + } + }, + "from_pretrained_arguments": { + "model_name": "Lightricks/LTX-2.5-Diffusers", + "subfolder": "text_encoder", + "torch_dtype": "torch.bfloat16" + } + }, + "arguments": { + "prompt": "variable:prompt", + "negative_prompt": "variable:negative_prompt", + "width": "variable:width", + "height": "variable:height", + "num_frames": "variable:num_frames", + "frame_rate": "variable:frame_rate", + "sigmas": "constant:diffusers.pipelines.ltx2.utils.DISTILLED_SIGMA_VALUES", + "guidance_scale": 1.0, + "audio_guidance_scale": 1.0, + "stg_scale": 0.0, + "audio_stg_scale": 0.0, + "modality_scale": 1.0, + "audio_modality_scale": 1.0, + "output_type": "{np}", + "reference_conditions": [ + { + "reference_type": "diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora.LTX2ReferenceCondition", + "from_arguments": { + "frames": { + "media_type": "video", + "location": "variable:source_video" + }, + "strength": 1.0 + } + } + ], + "reference_downscale_factor": 2, + "conditioning_attention_strength": 1.0 + }, + "loras": [ + { + "model_name": "Lightricks/LTX-2.5-22b-IC-LoRA-Pixel-Spatial-Upscaler", + "weight_name": "ltx-2.5-22b-ic-lora-pixel-spatial-upscaler-x2-1.0.safetensors", + "adapter_name": "ic_lora", + "scale": "variable:lora_scale" + } + ] + }, + "result": { + "content_type": "video/mp4", + "fps": "variable:frame_rate", + "subfolder": "intermediate" + } + }, + { + "name": "with_source_audio", + "task": { + "command": "pair_audio", + "arguments": { + "video": "previous_result:upscaled", + "audio": "variable:source_video", + "fit": "video" + } + }, + "result": { + "content_type": "video/mp4", + "fps": "variable:frame_rate", + "subfolder": "final" + } + } + ] +} From 53baa0b131916064fe1c0c881a4bf9d1f2b53823 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Sun, 27 Sep 2026 23:42:25 -0500 Subject: [PATCH 033/251] feat(docs): #548 - upscale-clip in the ltx-2.5 skill, the ltx2 README and CLAUDE.md Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 12 ++++++++---- plugins/dw/skills/ltx-2.5/SKILL.md | 8 +++++--- workflows/templates/ltx2/README.md | 3 ++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4861cd5d..5afa2816 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -205,12 +205,16 @@ change both when one changes. ### LTX-2.5 IC-LoRAs -The catalog's IC-LoRA templates are `templates/ltx2/generative-upscale` and three +The catalog's IC-LoRA templates are two upscalers, +`templates/ltx2/generative-upscale` and `upscale-clip`, and three conditioning templates, all through `LTX2InContextPipeline` + `LTX2ReferenceCondition`; the three run at `reference_downscale_factor: 1` (the -upscaler's is 2). `reference-sheet` -drives Ingredients — the family's only identity route, and the first two -templates here whose reference is a file the workflow did not make; the sheet +upscalers' is 2). `upscale-clip` runs the upscaler over the caller's own +`source_video` and then `pair_audio`s that file's soundtrack back onto the +upscale, so the `final/` deliverable carries the original track (a silent +source fails there, after the upscale is saved in `intermediate/`). +`reference-sheet` drives Ingredients — the family's only identity route, and +one of the templates here whose reference is a file the workflow did not make; the sheet is a still, so a `loop_frames` step (`dw/tasks/video_utils.py`, the video analogue of `loop_audio`) laps it into the static video the LoRA reads through its 121-frame bucket. `restore-deblur` and `restore-decompression` diff --git a/plugins/dw/skills/ltx-2.5/SKILL.md b/plugins/dw/skills/ltx-2.5/SKILL.md index d32b79f2..689e5334 100644 --- a/plugins/dw/skills/ltx-2.5/SKILL.md +++ b/plugins/dw/skills/ltx-2.5/SKILL.md @@ -1,6 +1,6 @@ --- name: ltx-2.5 -description: Use when a dw MCP server is connected and the user wants LTX-2.5 video - a short clip with its own soundtrack, a clip from one picture or between two, a sharper full-size render, a 2x upscale, a subject held across a clip from a reference sheet, a blurry or compressed clip restored, or a clip extended or chained longer. Picks the template, states schedule and size rules, quotes cost, and carries the trained caption spec the prompt must follow. +description: Use when a dw MCP server is connected and the user wants LTX-2.5 video - a short clip with its own soundtrack, a clip from one picture or between two, a sharper full-size render, a 2x upscale of its own or the user's clip, a subject held across a clip from a reference sheet, a blurry or compressed clip restored, or a clip extended or chained longer. Picks the template, states schedule and size rules, quotes cost, and carries the trained caption spec the prompt must follow. --- # LTX-2.5 on a dw server @@ -46,7 +46,9 @@ spec below. - **A generative 2x render**: `templates/ltx2/generative-upscale` draws its own low-res pass, then an IC-LoRA re-renders it twice the size, inventing detail. `base_width`/`base_height` are the first render's size, - `width`/`height` the doubled target. + `width`/`height` the doubled target. For the user's own clip, + `templates/ltx2/upscale-clip`: `width`/`height` 2x the source's, + `num_frames` at most its length; its soundtrack is kept. - **Keeping a subject across a clip**: `templates/ltx2/reference-sheet`. The family's only identity route; the user must author the reference sheet - one composite image, a clean panel per character, prop and location, no @@ -54,7 +56,7 @@ spec below. appear. `reference_frames` must stay at or above 121. - **Repairing the user's footage**: `templates/ltx2/restore-deblur` for spatial defocus, `templates/ltx2/restore-decompression` for low-bitrate - artefacts. Only these templates here read a clip dw didn't make. Each + artefacts. Each inverts one defect and no other - neither is an upscale, neither removes motion blur or grain - so name the defect and let the user correct you. diff --git a/workflows/templates/ltx2/README.md b/workflows/templates/ltx2/README.md index 31d483ee..ccd1fd56 100644 --- a/workflows/templates/ltx2/README.md +++ b/workflows/templates/ltx2/README.md @@ -42,6 +42,7 @@ Read them in this order and each introduces one new idea on top of the last. | ------- | ------------------ | | [two-stage.json](two-stage.json) | The distilled two-stage flow in its three moves: render at half size, double the latents, renoise and refine at full size. Lightricks' newer DFR pipeline is the follow-up | | [generative-upscale.json](generative-upscale.json) | A generative 2x upscale: an in-context LoRA re-renders a clip at twice the size, inventing detail | +| [upscale-clip.json](upscale-clip.json) | The same 2x re-render of a clip the caller brings (`source_video`), with its soundtrack paired back on by `pair_audio`. Set `width`/`height` to twice the source's at its aspect ratio - any other ratio is center-cropped - and `num_frames` to no more than its length. Generative, not a faithful resize; restore a compressed source first (#548) | | [diffusion-decode.json](diffusion-decode.json) | LTX-2.5's other video decoder: a small diffusion model in place of the convolutional VAE, held against `text-to-video` frame for frame. An experiment, not a recommendation - it is silent (audio comes back as latents), and it needs a `shi-labs/natten` build for the installed torch: the FlexAttention fallback needs ~25.5GiB for the smallest canvas its kernel accepts, so on 24GB it decodes nothing at all (#153) | ## Keeping a subject @@ -52,7 +53,7 @@ Read them in this order and each introduces one new idea on top of the last. ## Restoring footage you did not generate -Both of these read a clip the workflow did not make, with the source attached +Both of these (and `upscale-clip` above) read a clip the workflow did not make, with the source attached as an in-context reference for the whole denoise - so identity, framing and background geometry are the source's and only the defect changes. A different trade from `two-stage`, which invents a sharper version of a scene it From d8d49708be7be67b2125e20c2d46b611943d0e57 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 00:00:17 -0500 Subject: [PATCH 034/251] feat(docs): #548 - upscale-clip in the ltx-2.5 skill's Repairing section Co-Authored-By: Claude Opus 5.5 --- plugins/dw/skills/ltx-2.5/SKILL.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/plugins/dw/skills/ltx-2.5/SKILL.md b/plugins/dw/skills/ltx-2.5/SKILL.md index 689e5334..1844e0ff 100644 --- a/plugins/dw/skills/ltx-2.5/SKILL.md +++ b/plugins/dw/skills/ltx-2.5/SKILL.md @@ -46,9 +46,7 @@ spec below. - **A generative 2x render**: `templates/ltx2/generative-upscale` draws its own low-res pass, then an IC-LoRA re-renders it twice the size, inventing detail. `base_width`/`base_height` are the first render's size, - `width`/`height` the doubled target. For the user's own clip, - `templates/ltx2/upscale-clip`: `width`/`height` 2x the source's, - `num_frames` at most its length; its soundtrack is kept. + `width`/`height` the doubled target. The user's clip: `templates/ltx2/upscale-clip`. - **Keeping a subject across a clip**: `templates/ltx2/reference-sheet`. The family's only identity route; the user must author the reference sheet - one composite image, a clean panel per character, prop and location, no @@ -56,10 +54,11 @@ spec below. appear. `reference_frames` must stay at or above 121. - **Repairing the user's footage**: `templates/ltx2/restore-deblur` for spatial defocus, `templates/ltx2/restore-decompression` for low-bitrate - artefacts. Each - inverts one defect and no other - neither is an upscale, neither - removes motion blur or grain - so name the defect and let - the user correct you. + artefacts. Each inverts one defect and no other - neither upscales or + removes motion blur or grain - so name the defect and let the user + correct you. To upscale or sharpen: `templates/ltx2/upscale-clip`, + `width`/`height` 2x the source's, `num_frames` at most its length, + soundtrack kept. - **Longer**: `templates/ltx2/extend-clip` continues an opening conditioned on all of it, not one frame; `clip` extends an existing clip (`width`/ `height` matched, shorter than `num_frames`; clip_frames unused) instead From cf45718ad9eccb10baab64e96b3801be40bf7285 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 00:13:48 -0500 Subject: [PATCH 035/251] feat(catalog): #549 - templates/ltx2/refine-clip, 2x latent refine of the caller's clip Co-Authored-By: Claude Opus 5.5 --- tests/test_catalog_structure.py | 61 +++++- workflows/templates/ltx2/refine-clip.json | 239 ++++++++++++++++++++++ 2 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 workflows/templates/ltx2/refine-clip.json diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index e08c23b0..29a36c8c 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -411,7 +411,11 @@ def test_no_stale_entry_in_the_allowlist(): # Then to 9_000 for `templates/ltx2/upscale-clip` (#548), measured at 8_910: # about 125 tokens, the same as each of its restore siblings - the only # catalog route to a generative upscale of a clip the caller brings. -COMPACT_BUDGET = 9_000 +# Then to 9_150 for `templates/ltx2/refine-clip` (#543, 2026-09-28), measured +# at 9_052 before its curated `cost` (about 15 tokens more): the latent-refine +# route to 2x for a clip the caller brings, kept beside upscale-clip because +# the two trade differently (source latents vs. an IC-LoRA re-render). +COMPACT_BUDGET = 9_150 FILTERED_BUDGET = 1_500 @@ -559,6 +563,61 @@ def test_the_base_and_refine_passes_share_one_pipeline(self): assert not _step(definition, "base").get("release_pipeline", False) +class TestLtxRefineClip: + """two-stage's refine pass fed a clip dw did not make (#543): the latent + upsampler encodes the source itself (its `video` argument), so the refine + starts from the source's own latents, and the soundtrack is the source's, + read first so a silent source fails before any pipeline loads.""" + + def _definition(self): + path = os.path.join( + REPO_ROOT, "workflows", "templates", "ltx2", "refine-clip.json" + ) + return json.load(open(path, encoding="utf-8")) + + def test_the_renoise_scale_is_the_first_stage_two_sigma(self): + from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES + + refine = _step(self._definition(), "refine") + + assert ( + refine["pipeline"]["arguments"]["noise_scale"] + == STAGE_2_DISTILLED_SIGMA_VALUES[0] + ) + assert ( + refine["pipeline"]["arguments"]["sigmas"] + == "constant:diffusers.pipelines.ltx2.utils.STAGE_2_DISTILLED_SIGMA_VALUES" + ) + + def test_the_upsampler_encodes_the_source_and_refine_reads_its_latents(self): + definition = self._definition() + upscale = _step(definition, "upscale")["pipeline"] + refine = _step(definition, "refine")["pipeline"] + + assert upscale["arguments"]["video"] == "variable:source_video" + assert upscale["arguments"]["output_type"] == "{latent}" + assert refine["arguments"]["latents"] == "previous_result:upscale.frames" + # No audio latents: the source's track is paired back instead + assert "audio_latents" not in refine["arguments"] + assert upscale["configuration"]["shared_components"] == ["vae"] + assert refine["configuration"]["reused_components"] == ["vae"] + + def test_the_source_track_is_read_first_and_paired_at_minus_three(self): + definition = self._definition() + first = definition["steps"][0] + final = _step(definition, "with_source_audio") + + assert first["task"]["command"] == "normalize_audio" + assert first["task"]["arguments"]["audio"] == "variable:source_video" + assert first["task"]["arguments"]["peak_dbfs"] == -3.0 + assert final["task"]["command"] == "pair_audio" + assert final["task"]["arguments"]["video"] == "previous_result:refine" + assert final["task"]["arguments"]["audio"] == ( + f"previous_result:{first['name']}" + ) + assert final["task"]["arguments"]["fit"] == "video" + + LINK_PATTERN = re.compile(r"\]\(([^)]+)\)") READMES = sorted( os.path.relpath(path, REPO_ROOT) diff --git a/workflows/templates/ltx2/refine-clip.json b/workflows/templates/ltx2/refine-clip.json new file mode 100644 index 00000000..3c3880a0 --- /dev/null +++ b/workflows/templates/ltx2/refine-clip.json @@ -0,0 +1,239 @@ +{ + "id": "LTX2RefineClip", + "description": "Refines a clip from the asset library at exactly twice its size through LTX-2.5's stage-two path, starting from the clip's own latents rather than a reference: the latent upsampler VAE-encodes the source (resized, not cropped, to 'width' x 'height'), doubles the latents, and 'refine' renoises them at 0.909375 and runs the three stage-two distilled sigmas - two-stage's refine pass, fed a clip dw did not make. Composition and motion are the source's; fine texture is regenerated at that noise level, so it is not a faithful resize. Set 'width'/'height' to the source's own size, each divisible by 32: the output is exactly 2x them, and a source of a different aspect ratio is stretched to fit, not cropped or refused. 'num_frames' (8 * n + 1) must not exceed the source's length - the upsampler keeps only that many frames. Read the source's size, frame rate and length first with get_gallery_metadata('asset:...'). Defaults fit a 512x288, 121-frame, 24 fps source. Replace the default prompt with a description of the source's subject, setting, motion and lighting. The soundtrack is the source's own, not generated: its track is read and normalized to -3 dBFS as the first step, so a source with no soundtrack is refused there, before any pipeline loads, and the refined picture (saved to intermediate/) is paired with it, cut to the output length, in final/. Versus upscale-clip: that template re-renders with the source as an IC-LoRA reference (invented detail, center-crops a different aspect ratio, gated LoRA download); this one keeps the source's own latents, stretches rather than crops, needs no LoRA and denoises three sigmas rather than eight.", + "summary": "2x latent refine of your own clip, soundtrack kept. width/height = the source's size; num_frames <= its length.", + "cost_drivers": [ + "num_frames", + "width", + "height" + ], + "variables": { + "transformer_weights_dtype": "{uint4}", + "text_encoder_weights_dtype": "{int8}", + "source_video": "asset:clip.mp4", + "prompt": "A high-resolution rendering of the source footage, with the same subject, framing, motion and lighting, and sharp, fine detail.", + "negative_prompt": "constant:diffusers.pipelines.ltx2.utils.DEFAULT_NEGATIVE_PROMPT", + "width": 512, + "height": 288, + "num_frames": 121, + "frame_rate": 24.0, + "seed": 42 + }, + "variable_constraints": { + "width": { + "modulus": 32, + "remainder": 0, + "reason": "LTX-2.5's pipelines refuse a width not divisible by 32 outright (ValueError, not a silent floor), so it is checked here before the pipeline loads rather than after it" + }, + "height": { + "modulus": 32, + "remainder": 0, + "reason": "LTX-2.5's pipelines refuse a height not divisible by 32 outright (ValueError, not a silent floor), so it is checked here before the pipeline loads rather than after it" + }, + "num_frames": { + "modulus": 8, + "remainder": 1, + "min_frames": 9, + "reason": "LTX-2.5's video VAE encodes 8 * n + 1 frames, so an off-grid count is not refused by the pipeline but floored to the grid below - a shorter clip than the one asked for. Refused here instead: no 'snap', because rounding a length the other way from the pipeline's own would be a second silent change" + } + }, + "seed": "variable:seed", + "steps": [ + { + "name": "source_audio", + "task": { + "command": "normalize_audio", + "arguments": { + "audio": "variable:source_video", + "peak_dbfs": -3.0 + } + } + }, + { + "name": "upscale", + "pipeline": { + "configuration": { + "component_type": "LTX2LatentUpsamplePipeline", + "shared_components": [ + "vae" + ], + "vae": { + "enable_tiling": true + } + }, + "from_pretrained_arguments": {}, + "vae": { + "configuration": { + "component_type": "AutoencoderKLLTX2Video" + }, + "from_pretrained_arguments": { + "model_name": "Lightricks/LTX-2.5-Diffusers", + "subfolder": "vae", + "torch_dtype": "torch.bfloat16" + } + }, + "latent_upsampler": { + "configuration": { + "component_type": "diffusers.pipelines.ltx2.latent_upsampler.LTX2LatentUpsamplerModel" + }, + "from_pretrained_arguments": { + "model_name": "Lightricks/LTX-2.5-Diffusers", + "subfolder": "latent_upsampler", + "torch_dtype": "torch.bfloat16" + } + }, + "arguments": { + "video": "variable:source_video", + "width": "variable:width", + "height": "variable:height", + "num_frames": "variable:num_frames", + "adain_factor": 0.0, + "tone_map_compression_ratio": 0.0, + "output_type": "{latent}" + } + }, + "result": { + "content_type": "video/mp4", + "save": false, + "fps": "variable:frame_rate" + } + }, + { + "name": "refine", + "release_pipeline": true, + "pipeline": { + "configuration": { + "component_type": "LTX2Pipeline", + "pre_load_modules": [ + "sdnq" + ], + "components": { + "transformer": { + "device": "cuda" + }, + "text_encoder": { + "group_offload": { + "offload_type": "leaf_level", + "use_stream": true + } + }, + "connectors": { + "group_offload": { + "offload_type": "leaf_level", + "use_stream": true + } + }, + "audio_vae": { + "device": "cuda" + }, + "vocoder": { + "device": "cuda" + }, + "duration_head": { + "device": "cuda" + } + }, + "vae": { + "enable_tiling": true + }, + "reused_components": [ + "vae" + ] + }, + "from_pretrained_arguments": { + "model_name": "Lightricks/LTX-2.5-Diffusers", + "torch_dtype": "torch.bfloat16" + }, + "transformer": { + "configuration": { + "component_type": "LTX2VideoTransformer3DModel", + "preserve_device_placement": true + }, + "quantization_config": { + "configuration": { + "config_type": "sdnq.SDNQConfig" + }, + "arguments": { + "weights_dtype": "variable:transformer_weights_dtype", + "quantization_device": "cuda", + "return_device": "cuda", + "use_quantized_matmul": true, + "dequantize_fp32": false + } + }, + "from_pretrained_arguments": { + "model_name": "Lightricks/LTX-2.5-Diffusers", + "subfolder": "transformer", + "torch_dtype": "torch.bfloat16" + } + }, + "text_encoder": { + "configuration": { + "component_type": "transformers.Gemma4UnifiedForConditionalGeneration", + "preserve_device_placement": true + }, + "quantization_config": { + "configuration": { + "config_type": "sdnq.SDNQConfig" + }, + "arguments": { + "weights_dtype": "variable:text_encoder_weights_dtype", + "quantization_device": "cuda", + "return_device": "cpu", + "dequantize_fp32": false, + "modules_to_not_convert": [ + "embed_vision", + "embed_audio", + "lm_head" + ] + } + }, + "from_pretrained_arguments": { + "model_name": "Lightricks/LTX-2.5-Diffusers", + "subfolder": "text_encoder", + "torch_dtype": "torch.bfloat16" + } + }, + "arguments": { + "prompt": "variable:prompt", + "negative_prompt": "variable:negative_prompt", + "latents": "previous_result:upscale.frames", + "noise_scale": 0.909375, + "sigmas": "constant:diffusers.pipelines.ltx2.utils.STAGE_2_DISTILLED_SIGMA_VALUES", + "width": "variable:width", + "height": "variable:height", + "num_frames": "variable:num_frames", + "frame_rate": "variable:frame_rate", + "guidance_scale": 1.0, + "audio_guidance_scale": 1.0, + "stg_scale": 0.0, + "audio_stg_scale": 0.0, + "modality_scale": 1.0, + "audio_modality_scale": 1.0, + "output_type": "{np}" + } + }, + "result": { + "content_type": "video/mp4", + "fps": "variable:frame_rate", + "subfolder": "intermediate" + } + }, + { + "name": "with_source_audio", + "task": { + "command": "pair_audio", + "arguments": { + "video": "previous_result:refine", + "audio": "previous_result:source_audio", + "fit": "video" + } + }, + "result": { + "content_type": "video/mp4", + "fps": "variable:frame_rate", + "subfolder": "final" + } + } + ] +} From cd7fcd302217b7ce0ba27423329d3da26163cb45 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 00:18:22 -0500 Subject: [PATCH 036/251] feat(docs): #549 - refine-clip in the ltx-2.5 skill, the ltx2 README and RECIPES_24GB Co-Authored-By: Claude Opus 5.5 --- docs/RECIPES_24GB.md | 3 +++ plugins/dw/skills/ltx-2.5/SKILL.md | 37 ++++++++++++++---------------- workflows/templates/ltx2/README.md | 1 + 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/docs/RECIPES_24GB.md b/docs/RECIPES_24GB.md index 5a6995d0..abbb9fc9 100644 --- a/docs/RECIPES_24GB.md +++ b/docs/RECIPES_24GB.md @@ -183,6 +183,9 @@ through their DFR pipeline instead, which diffusers ships and nothing here uses Measured on an RTX 3090 the refined clip is sharper than the 2x upsample alone at the same seed: fur, branches and snow texture resolve where the upsample-only frame is a soft blur. About eight warm minutes, three and a half of them writing the full-size clip. +The same refine on a clip dw did not make is +[refine-clip.json](../workflows/templates/ltx2/refine-clip.json): the upsampler encodes +the source itself, and the source's soundtrack is paired back on. **Examples:** [text-to-video.json](../workflows/templates/ltx2/text-to-video.json) (t2v), [two-stage.json](../workflows/templates/ltx2/two-stage.json) (base -> latent upsample -> refine), diff --git a/plugins/dw/skills/ltx-2.5/SKILL.md b/plugins/dw/skills/ltx-2.5/SKILL.md index 1844e0ff..3104b13f 100644 --- a/plugins/dw/skills/ltx-2.5/SKILL.md +++ b/plugins/dw/skills/ltx-2.5/SKILL.md @@ -6,9 +6,7 @@ description: Use when a dw MCP server is connected and the user wants LTX-2.5 vi # LTX-2.5 on a dw server LTX-2.5 generates video and a soundtrack together, 24 fps, on a distilled -schedule that is not a knob. Every template here fits a 24 GB card. This -skill picks the template and arguments; the prompt follows the caption -spec below. +schedule that is not a knob. Every template here fits a 24 GB card. ## Before anything @@ -24,10 +22,9 @@ spec below. reading's `gpu_memory_allocated_mb`; only those are the worker's. `info: null` means nothing is resident, and `live: false` is cached from another moment. A non-trivial idle figure is an earlier run's leftover, - subtracted from what this one has. With the server idle, `clear_memory` - clears it (refused while queued or running) and drops the step cache, so - the next run, even a seeded rerun, is cold and regenerates. Re-read - `get_memory` to confirm; don't retry a failed attempt unless cleared. + subtracted from what this one has. `clear_memory` (idle server only) + clears it and drops the step cache, so even a seeded rerun is cold. Re-read + `get_memory` to confirm; retry a failed attempt only once cleared. ## Which shape is the request @@ -40,7 +37,7 @@ spec below. - **Sharper at full size**: `templates/ltx2/two-stage` - eight sigmas at 768x448, a 2x latent upsample, then renoise and three stage-two sigmas at 1536x896 carrying audio latents through. The upsample alone is soft; - the refine pass supplies the detail. + the refine pass supplies the detail. On the user's clip: `refine-clip`. - `templates/ltx2/diffusion-decode` compares decoders. Do not offer it: without a `shi-labs/natten` build its fallback OOMs on 24GB at any size. - **A generative 2x render**: `templates/ltx2/generative-upscale` draws its own @@ -56,9 +53,11 @@ spec below. spatial defocus, `templates/ltx2/restore-decompression` for low-bitrate artefacts. Each inverts one defect and no other - neither upscales or removes motion blur or grain - so name the defect and let the user - correct you. To upscale or sharpen: `templates/ltx2/upscale-clip`, - `width`/`height` 2x the source's, `num_frames` at most its length, - soundtrack kept. + correct you. To upscale or sharpen: `templates/ltx2/upscale-clip` + (IC-LoRA re-render), `width`/`height` 2x the source's, or `refine-clip` + (its own latents, no LoRA), `width`/`height` the source's and output 2x, + other ratios stretched. Both: `num_frames` at most its length, soundtrack + kept, silent source refused. - **Longer**: `templates/ltx2/extend-clip` continues an opening conditioned on all of it, not one frame; `clip` extends an existing clip (`width`/ `height` matched, shorter than `num_frames`; clip_frames unused) instead @@ -85,8 +84,8 @@ read the `workflows` guide's authoring section first. - An image condition is re-compressed at CRF 18 to match training and needs a PIL image; a multi-frame video condition is not. - Audio is generated in the first pass and nothing refines it, so carry the - audio latents (two-stage) or pair the track back (`pair_audio`) on a - frames-only step. + audio latents (two-stage) or pair the track back (`pair_audio`, as the + `-clip` upscales do) on a frames-only step. ## Prompts @@ -137,15 +136,14 @@ AESTHETIC QUALITY (in addition to the above, without breaking the objective capt 1. `validate_workflow` first - free, and catches bad arguments. 2. Quote `plan.estimate` from the validate answer (wall clock, loading included) and name any `downloads_required` - an IC-LoRA template pulls a - gated weight the box may not have. Only `text-to-video` and `two-stage` - carry a `cost`; for the rest give the shape - a 121-frame clip at 960x544 + gated weight the box may not have. Only `text-to-video`, `two-stage` + and `refine-clip` carry a `cost`; for the rest give the shape - a 121-frame clip at 960x544 is under two minutes cold on a 24 GB card, a minute loading; extend and chain multiply by their passes. Get the go-ahead, then `run_workflow` with `acknowledged_cost` set to the plan's `{fingerprint, minutes, downloads}`. 3. `wait_for_job` with `timeout_seconds` = the estimate plus a margin - (`timeout_capped` says the server's cap cut it; call again while - `still_running`), then `get_job` for the manifest. + (call again while `still_running`), then `get_job` for the manifest. 4. Writing still costs on a long chain, so only worthwhile steps should: `"result": {"save": false}` on the rest, as `two-stage` does for `base` and `upscale`. Missing it is silent. What does write carries a @@ -159,9 +157,8 @@ AESTHETIC QUALITY (in addition to the above, without breaking the objective capt `get_job` for the manifest and its warnings, `get_gallery_metadata` for duration, size and audio presence, and hand the user the gallery `url` (`list_gallery`, or the manifest's file name). -6. After a run worth keeping, `get_job_workflow` and `save_workflow` it, so - the next run is by name not pasted JSON; `export_job` bundles it on the - server. `auth_required: false` - fetch `open_url` into `exports/` under +6. Save a run worth keeping (`get_job_workflow`, `save_workflow`) to rerun + it by name; `export_job` bundles it on the server. `auth_required: false` - fetch `open_url` into `exports/` under the working dir (never a temp dir; unpacks into a job-id folder). `true` - hand `open_url` to the person instead, keep using `get_output_image`/`_audio`/`_frames` diff --git a/workflows/templates/ltx2/README.md b/workflows/templates/ltx2/README.md index ccd1fd56..481e3e33 100644 --- a/workflows/templates/ltx2/README.md +++ b/workflows/templates/ltx2/README.md @@ -43,6 +43,7 @@ Read them in this order and each introduces one new idea on top of the last. | [two-stage.json](two-stage.json) | The distilled two-stage flow in its three moves: render at half size, double the latents, renoise and refine at full size. Lightricks' newer DFR pipeline is the follow-up | | [generative-upscale.json](generative-upscale.json) | A generative 2x upscale: an in-context LoRA re-renders a clip at twice the size, inventing detail | | [upscale-clip.json](upscale-clip.json) | The same 2x re-render of a clip the caller brings (`source_video`), with its soundtrack paired back on by `pair_audio`. Set `width`/`height` to twice the source's at its aspect ratio - any other ratio is center-cropped - and `num_frames` to no more than its length. Generative, not a faithful resize; restore a compressed source first (#548) | +| [refine-clip.json](refine-clip.json) | `two-stage`'s refine pass on a clip the caller brings (`source_video`): the latent upsampler encodes the source and doubles its latents, then the three stage-two sigmas refine them - the source's own latents, not a reference re-render, and no LoRA. Set `width`/`height` to the source's own size (the output is exactly 2x; another aspect ratio is stretched) and `num_frames` to no more than its length. The source's soundtrack is paired back on; a silent source is refused before any pipeline loads (#543) | | [diffusion-decode.json](diffusion-decode.json) | LTX-2.5's other video decoder: a small diffusion model in place of the convolutional VAE, held against `text-to-video` frame for frame. An experiment, not a recommendation - it is silent (audio comes back as latents), and it needs a `shi-labs/natten` build for the installed torch: the FlexAttention fallback needs ~25.5GiB for the smallest canvas its kernel accepts, so on 24GB it decodes nothing at all (#153) | ## Keeping a subject From 8b72d5be8d118485ec0f59ba4ab45354c016be73 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 00:23:40 -0500 Subject: [PATCH 037/251] feat(catalog): #549 - measured cost for refine-clip, 2.9 min cold on the RTX 3090 Co-Authored-By: Claude Opus 5.5 --- workflows/templates/ltx2/refine-clip.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/workflows/templates/ltx2/refine-clip.json b/workflows/templates/ltx2/refine-clip.json index 3c3880a0..e6d8d21c 100644 --- a/workflows/templates/ltx2/refine-clip.json +++ b/workflows/templates/ltx2/refine-clip.json @@ -2,6 +2,14 @@ "id": "LTX2RefineClip", "description": "Refines a clip from the asset library at exactly twice its size through LTX-2.5's stage-two path, starting from the clip's own latents rather than a reference: the latent upsampler VAE-encodes the source (resized, not cropped, to 'width' x 'height'), doubles the latents, and 'refine' renoises them at 0.909375 and runs the three stage-two distilled sigmas - two-stage's refine pass, fed a clip dw did not make. Composition and motion are the source's; fine texture is regenerated at that noise level, so it is not a faithful resize. Set 'width'/'height' to the source's own size, each divisible by 32: the output is exactly 2x them, and a source of a different aspect ratio is stretched to fit, not cropped or refused. 'num_frames' (8 * n + 1) must not exceed the source's length - the upsampler keeps only that many frames. Read the source's size, frame rate and length first with get_gallery_metadata('asset:...'). Defaults fit a 512x288, 121-frame, 24 fps source. Replace the default prompt with a description of the source's subject, setting, motion and lighting. The soundtrack is the source's own, not generated: its track is read and normalized to -3 dBFS as the first step, so a source with no soundtrack is refused there, before any pipeline loads, and the refined picture (saved to intermediate/) is paired with it, cut to the output length, in final/. Versus upscale-clip: that template re-renders with the source as an IC-LoRA reference (invented detail, center-crops a different aspect ratio, gated LoRA download); this one keeps the source's own latents, stretches rather than crops, needs no LoRA and denoises three sigmas rather than eight.", "summary": "2x latent refine of your own clip, soundtrack kept. width/height = the source's size; num_frames <= its length.", + "cost": [ + { + "device": "cuda", + "name": "RTX 3090", + "vram_gb": 24, + "minutes": 2.9 + } + ], "cost_drivers": [ "num_frames", "width", From b500dc0589c0ff69b7dcf838cea35fd5f9a7f5f1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 00:54:32 -0500 Subject: [PATCH 038/251] feat(catalog): #549 - trim the source to num_frames before the upsampler LTX2LatentUpsamplePipeline overwrites num_frames with len(video), so a source longer than num_frames was encoded whole: a 97-frame run came out 121 frames with a 4 s generated track (the tester's M-F065 short arm). A loop_frames step now hands the upsampler the source's opening num_frames frames. Co-Authored-By: Claude Opus 5.5 --- tests/test_catalog_structure.py | 29 ++++++++++++++++++++++- workflows/templates/ltx2/refine-clip.json | 18 ++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index 29a36c8c..e79cc388 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -594,7 +594,16 @@ def test_the_upsampler_encodes_the_source_and_refine_reads_its_latents(self): upscale = _step(definition, "upscale")["pipeline"] refine = _step(definition, "refine")["pipeline"] - assert upscale["arguments"]["video"] == "variable:source_video" + trim = _step(definition, "source_frames")["task"] + + # The upsampler encodes every frame it is handed (num_frames is + # overwritten by len(video)), so the source is trimmed first (#549) + assert trim["command"] == "loop_frames" + assert trim["arguments"] == { + "video": "variable:source_video", + "num_frames": "variable:num_frames", + } + assert upscale["arguments"]["video"] == "previous_result:source_frames" assert upscale["arguments"]["output_type"] == "{latent}" assert refine["arguments"]["latents"] == "previous_result:upscale.frames" # No audio latents: the source's track is paired back instead @@ -602,6 +611,24 @@ def test_the_upsampler_encodes_the_source_and_refine_reads_its_latents(self): assert upscale["configuration"]["shared_components"] == ["vae"] assert refine["configuration"]["reused_components"] == ["vae"] + def test_the_trim_keeps_the_sources_opening_frames_in_order(self): + # The short arm of #549's bounce: a 130-frame source asked for 97 + # must reach the upsampler as its first 97 frames, not all 130 + import numpy + + from dw.tasks.video_utils import loop_frames + + source = numpy.arange(130, dtype=numpy.uint8)[:, None, None, None] + source = numpy.broadcast_to(source, (130, 4, 4, 3)).copy() + + trimmed = loop_frames(source, 97) + + assert trimmed.shape == (97, 4, 4, 3) + assert numpy.array_equal( + (trimmed[:, 0, 0, 0] * 255).round().astype(numpy.uint8), + numpy.arange(97, dtype=numpy.uint8), + ) + def test_the_source_track_is_read_first_and_paired_at_minus_three(self): definition = self._definition() first = definition["steps"][0] diff --git a/workflows/templates/ltx2/refine-clip.json b/workflows/templates/ltx2/refine-clip.json index e6d8d21c..b2481b07 100644 --- a/workflows/templates/ltx2/refine-clip.json +++ b/workflows/templates/ltx2/refine-clip.json @@ -1,6 +1,6 @@ { "id": "LTX2RefineClip", - "description": "Refines a clip from the asset library at exactly twice its size through LTX-2.5's stage-two path, starting from the clip's own latents rather than a reference: the latent upsampler VAE-encodes the source (resized, not cropped, to 'width' x 'height'), doubles the latents, and 'refine' renoises them at 0.909375 and runs the three stage-two distilled sigmas - two-stage's refine pass, fed a clip dw did not make. Composition and motion are the source's; fine texture is regenerated at that noise level, so it is not a faithful resize. Set 'width'/'height' to the source's own size, each divisible by 32: the output is exactly 2x them, and a source of a different aspect ratio is stretched to fit, not cropped or refused. 'num_frames' (8 * n + 1) must not exceed the source's length - the upsampler keeps only that many frames. Read the source's size, frame rate and length first with get_gallery_metadata('asset:...'). Defaults fit a 512x288, 121-frame, 24 fps source. Replace the default prompt with a description of the source's subject, setting, motion and lighting. The soundtrack is the source's own, not generated: its track is read and normalized to -3 dBFS as the first step, so a source with no soundtrack is refused there, before any pipeline loads, and the refined picture (saved to intermediate/) is paired with it, cut to the output length, in final/. Versus upscale-clip: that template re-renders with the source as an IC-LoRA reference (invented detail, center-crops a different aspect ratio, gated LoRA download); this one keeps the source's own latents, stretches rather than crops, needs no LoRA and denoises three sigmas rather than eight.", + "description": "Refines a clip from the asset library at exactly twice its size through LTX-2.5's stage-two path, starting from the clip's own latents rather than a reference: the latent upsampler VAE-encodes the source (resized, not cropped, to 'width' x 'height'), doubles the latents, and 'refine' renoises them at 0.909375 and runs the three stage-two distilled sigmas - two-stage's refine pass, fed a clip dw did not make. Composition and motion are the source's; fine texture is regenerated at that noise level, so it is not a faithful resize. Set 'width'/'height' to the source's own size, each divisible by 32: the output is exactly 2x them, and a source of a different aspect ratio is stretched to fit, not cropped or refused. 'num_frames' (8 * n + 1) must not exceed the source's length: a first step trims the source to its opening 'num_frames' frames (the upsampler itself would encode every frame it is handed), and a shorter source is lapped from its first frame rather than refused. Read the source's size, frame rate and length first with get_gallery_metadata('asset:...'). Defaults fit a 512x288, 121-frame, 24 fps source. Replace the default prompt with a description of the source's subject, setting, motion and lighting. The soundtrack is the source's own, not generated: its track is read and normalized to -3 dBFS as the first step, so a source with no soundtrack is refused there, before any pipeline loads, and the refined picture (saved to intermediate/) is paired with it, cut to the output length, in final/. Versus upscale-clip: that template re-renders with the source as an IC-LoRA reference (invented detail, center-crops a different aspect ratio, gated LoRA download); this one keeps the source's own latents, stretches rather than crops, needs no LoRA and denoises three sigmas rather than eight.", "summary": "2x latent refine of your own clip, soundtrack kept. width/height = the source's size; num_frames <= its length.", "cost": [ { @@ -57,6 +57,20 @@ } } }, + { + "name": "source_frames", + "task": { + "command": "loop_frames", + "arguments": { + "video": "variable:source_video", + "num_frames": "variable:num_frames" + } + }, + "result": { + "content_type": "video/mp4", + "save": false + } + }, { "name": "upscale", "pipeline": { @@ -91,7 +105,7 @@ } }, "arguments": { - "video": "variable:source_video", + "video": "previous_result:source_frames", "width": "variable:width", "height": "variable:height", "num_frames": "variable:num_frames", From 23adfe5b88b789523c72b4a01a8d98e48e97f2f5 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 01:12:20 -0500 Subject: [PATCH 039/251] docs(proposals): #543 - close-out record for ltx2/refine-clip Co-Authored-By: Claude Opus 5.5 --- .../complete/ltx2-refine-clip-complete.md | 136 ++++++++++++++++++ docs/proposals/todo.md | 5 + 2 files changed, 141 insertions(+) create mode 100644 docs/proposals/complete/ltx2-refine-clip-complete.md diff --git a/docs/proposals/complete/ltx2-refine-clip-complete.md b/docs/proposals/complete/ltx2-refine-clip-complete.md new file mode 100644 index 00000000..d2991374 --- /dev/null +++ b/docs/proposals/complete/ltx2-refine-clip-complete.md @@ -0,0 +1,136 @@ +# `ltx2/refine-clip`: LTX-2.5 two-stage refine of an existing mp4 (#543) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out, +2026-09-28). Plan v2 approved by Don (Q1-Q3 answered as the defaults) on +2026-09-27; one stage, #549, verified 2026-09-28. + +## The report + +`templates/ltx2/two-stage`'s `refine` step (renoise at 0.909375, the three +`STAGE_2_DISTILLED_SIGMA_VALUES`) could only start from latents its own +`base` pass made. #543 asked for the same refine on a clip dw did not make, +and assumed that meant a new engine task to VAE-encode an mp4 into LTX-2.5 +latents, plus answers on audio latents, the encode symbol, size/frame +constraints and VRAM. It is the companion to #542 (`upscale-clip`, the +IC-LoRA route). + +## The finding that shrank it + +**The 2x case needs no encode task.** `LTX2LatentUpsamplePipeline.__call__` +takes `video=` as an alternative to `latents`: it VAE-encodes the frames, +resizes them to `height`/`width` and returns un-normalized 5D latents at 2x, +the same form `two-stage`'s `upscale` hands to `refine`. `LTX2Pipeline` +normalizes 5D input itself, so there is no double-normalization risk. The +whole chain is existing pieces in one template. + +Other corrections the plan made to the issue: +- **`audio_latents=None` is accepted** (refine samples fresh audio noise), + but three sigmas of noise is not a deliverable, so the template drops the + generated track and pairs the source's own back with `pair_audio`. +- **No encode symbol to pin.** What is pinned is the renoise scale + (`STAGE_2_DISTILLED_SIGMA_VALUES[0]`) and the sigmas by `constant:`. +- **The upsampler encodes with `sample_mode="sample"`**, not argmax. Noted + as a risk; deterministic under the seeded generator, and the renoise at + 0.91 swamps it. + +## Decisions (plan v2) + +- **Q1: keep both routes.** #542 re-renders with the source as a reference + (invents detail, center-crops, needs a gated LoRA); this one renoises the + source's own latents (stays closer to its structure, stretches, no LoRA). +- **Q2: refuse a silent source early**, before any pipeline loads. Don may + revisit (e.g. keeping refine's generated track) in a new plan version. +- **Q3: the 1x refine and the audio encode stay cut** (see Deferred). + +## What was built + +**Stage A, #549** (server catalog + plugin skill; no engine code): +`workflows/templates/ltx2/refine-clip.json` (id `LTX2RefineClip`, shape +`shot`, traits `needs-input-media`, `has-audio`). Steps: + +1. `source_audio`: `normalize_audio` of `source_video` to -3 dBFS, not + saved. It is also the silent-source refusal: `load_audio` raises on a + video with no audio stream, so the job fails at step 0 in about 1 s. +2. `source_frames`: `loop_frames(video: variable:source_video, num_frames: + variable:num_frames)`, not saved. Added after the bounce (below). +3. `upscale`: `LTX2LatentUpsamplePipeline` on `previous_result:source_frames`, + latents out, not saved; its VAE is an explicit component with tiling, + published as `shared_components: ["vae"]`. +4. `refine`: `LTX2Pipeline`, `latents: previous_result:upscale.frames`, no + `audio_latents`, `noise_scale` 0.909375, the stage-two sigmas, guidance 1, + STG/modality off, `reused_components: ["vae"]`; saved to `intermediate/`. +5. `with_source_audio`: `pair_audio(fit: "video")` of the refined picture + and `previous_result:source_audio`; saved to `final/`. + +- Variables: `source_video` (placeholder `asset:clip.mp4`, which does not + exist, like `upscale-clip`'s), `prompt`, `negative_prompt`, `width` 512, + `height` 288, `num_frames` 121, `frame_rate` 24.0, the SDNQ dtypes, `seed`. +- `variable_constraints`: `num_frames` on `8n+1`, `width`/`height` multiples + of 32, no `snap` (copied from `two-stage`). +- `cost_drivers`: `num_frames`, `width`, `height`. `cost`: **2.9 min**, + RTX 3090 24 GB, measured cold on lem at the defaults (job `dad16ac7253d`). + No `vram_estimate`, like the siblings; the measured run fit 24 GB. +- Tests: `TestLtxRefineClip` in `tests/test_catalog_structure.py` (the + renoise scale, the latents and trim wiring, the -3 dBFS pair, and a real + `loop_frames` run cutting 130 frames to the first 97). `COMPACT_BUDGET` + raised to 9_150. +- Docs: the `ltx-2.5` skill (two-stage entry, the Repairing section's + contrast with `upscale-clip`, the audio rule, the cost line), the + `workflows/templates/ltx2/README.md` row, and a `docs/RECIPES_24GB.md` + pointer. + +Deviations from the plan, as built: +- **`refine`'s `width`/`height` are the source size, not 2x.** `LTX2Pipeline` + reads its shape off the latents, as `two-stage`'s refine does; the output + is 2x the stated size. +- **The silent-source check is the first step**, not a separate probe. The + tester accepted a first-step refusal (M-F066 arm b). +- **The upsampler reads `previous_result:source_frames`**, not + `variable:source_video`. The plan's premise that "the upsampler truncates" + to `num_frames` was wrong (next section). +- **A source shorter than `num_frames` is lapped from its first frame** + (`loop_frames`' behavior), not refused; the description says not to do + it. Probing the source at validate stayed cut. +- **An off-ratio source is stretched**, as planned, and documented. + +## The plan's gap: the upsampler does not truncate + +`LTX2LatentUpsamplePipeline.__call__` overwrites `num_frames` with +`len(video)` when handed `video` (`pipeline_ltx2_latent_upsample.py:354`); +its only trim is a floor to `8k+1`. So a 121-frame source asked for 97 came +out at 121 frames of picture while refine's (discarded) audio followed +`num_frames`, and `pair_audio` paired the full track. The `source_frames` +trim step fixes it; its float [0,1] frames match the old PIL path through +`VideoProcessor.preprocess_video` exactly at the same size and within 1/255 +after a resize. `cost` was not re-measured for a CPU frame copy. + +## Bounces per stage + +| Stage | Bounces | Cause | +|---|---|---| +| A, #549 | 1 | M-F065 short arm: `num_frames: 97` gave a 121-frame picture (the upsampler ignores `num_frames` for `video=`). Fixed with the `source_frames` trim. | + +No stage comment names `usage:` figures, so per-stage cost is not recorded. +The plan estimated $4-6. + +Open against the suite, not the code: the tester filed an amendment on +dkackman/harnest to bring M-F062's text in line with the as-built wiring +(`source_frames`, source-size refine, the first-step audio check). The +silent fixture M-F066 uses is `asset:refine/src-512x288-silent.mp4`, made +on-box and kept. + +## Deferred + +- **A 1x refine** (re-detail at the source's own size). The one case that + really needs a VAE-encode task: a latents `returns` kind or a + `save: false` artifact task, plus a second LTX VAE through `cached_model`. + Back on a field report of a clip to clean up without doubling it. +- **Encoding the source soundtrack into `audio_latents`.** No diffusers LTX2 + pipeline calls `audio_vae.encode`, and nothing provides the log-mel front + end to pin. Back when diffusers ships one, or on a report that the paired + original track is wrong for the refined picture. +- **Keeping refine's generated track for a silent source** (Q2's revisit). +- **Probing the source at validate** (its length or aspect ratio), cut the + same way as #542. +- **Quality on foreign footage** is evidenced by the verify runs only: the + stage-two sigmas were tuned for the same model's base-pass latents. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 91404046..25855ef8 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -68,6 +68,11 @@ remaining deferred fix is recorded in #544-#545), shipped 2026-09-27. Record, including what was deferred (a sync route, in-workflow wiring into `slice_audio`, thresholds for louder rooms): `complete/find-loop-bed-complete.md`. +- **`ltx2/refine-clip`, LTX-2.5's two-stage refine on an existing mp4** + (#543, stage #549), shipped 2026-09-28 as one template with no engine + task. Record, including what was deferred (a 1x refine and the + VAE-encode task it needs, encoding the source soundtrack into audio + latents): `complete/ltx2-refine-clip-complete.md`. ## Declined From 9d8089a6c26cadb938d48dbadf8a2d1c374cf226 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 01:37:18 -0500 Subject: [PATCH 040/251] fix(mcp): #554 - refuse names/boundaries on get_output_frames without seams Passing `names` or `boundaries` alongside `at` or `count` silently dropped them with no warning. Refuse the combination up front, naming the offending argument(s), and say in the tool description and docs that they are modifiers of `seams` only. Co-Authored-By: Claude Sonnet 5 --- docs/MCP.md | 2 +- dw_mcp/media.py | 20 +++++++++++++++----- tests/test_mcp_media.py | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/MCP.md b/docs/MCP.md index 758c44de..4507116e 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -235,7 +235,7 @@ when no single workflow covers it. | --- | --- | --- | | `get_output_image(name, max_dimension=768, workspace=None, crop=None)` | `name`, `max_dimension`, `workspace`, `crop` | Look at a generated image, downscaled to `max_dimension` on its longest side. Returns the image plus a text part reporting `original_size`, `returned_size` and `bytes`, so a downscale is never silent. `crop` is `[x, y, width, height]` in the original's pixels, cut before the downscale and reported back clamped - the way to see a region of a 2K still at 100%, where the whole would be shrunk past what a small element or a decode-tiling seam can be judged at. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | | `get_output_audio(name, start=None, duration=None, workspace=None)` | `name`, `start`, `duration`, `workspace` | Listen to a generated soundtrack as base64 - an audio output, or the track muxed into a video (#193) - in its own encoding when served whole, WAV when extracted or excerpted. No downscale exists for audio, so a whole clip over the 4MB budget is refused rather than cut (#204); ask for the part instead with `start` and `duration` in seconds, and the text part names what was cut (`excerpt: 2.0s from 10.0s of 240.0s`) so a slice is never mistaken for the whole. `get_gallery_metadata`'s envelope says where in a track to look. `workspace` names the workspace for this one call without switching the session to it | -| `get_output_frames(name, at=None, seams=None, count=None, boundaries=None, names=None, max_dimension=512, hear=None, workspace=None)` | `name`, `at`, `seams`, `count`, `boundaries`, `names`, `max_dimension`, `hear`, `workspace` | See a generated video as frames, since there is no video content type over MCP (#193). One selector per call: `count` for an evenly spaced contact sheet, `at` for moments (seconds or `"frame:N"`), `seams` (true, or seam numbers from 1) for the last frame before and first frame after each join side by side (each pair carries `difference`, the mean pixel change across the join, 0-255 - rank seams by it and look at the worst). On an output joined from shots (`concat_videos`, `dissolve_videos`, a chained step) `seams` alone is enough: the seams and their names are the `media.shots` its run recorded. For any other file pass `boundaries` - each later shot's first frame, the running sum of the shots' `frame_count` from `get_gallery_metadata` on their own `intermediate/` files - and `names` to name them; either one given overrides the recorded value. Tiles are fitted to `max_dimension` and, when the set would exceed the 4MB budget, shrunk together rather than dropped; the text part lists each tile and says so. `hear=N` also returns N seconds of soundtrack centred on each `at` moment, after its image - the way to check a hit point or lip-sync without reconciling two clocks; a mute clip keeps its frames and says `no soundtrack` | +| `get_output_frames(name, at=None, seams=None, count=None, boundaries=None, names=None, max_dimension=512, hear=None, workspace=None)` | `name`, `at`, `seams`, `count`, `boundaries`, `names`, `max_dimension`, `hear`, `workspace` | See a generated video as frames, since there is no video content type over MCP (#193). One selector per call: `count` for an evenly spaced contact sheet, `at` for moments (seconds or `"frame:N"`), `seams` (true, or seam numbers from 1) for the last frame before and first frame after each join side by side (each pair carries `difference`, the mean pixel change across the join, 0-255 - rank seams by it and look at the worst). `boundaries` and `names` are modifiers of `seams` only - passing either alongside `at` or `count` is refused, naming the offending argument(s). On an output joined from shots (`concat_videos`, `dissolve_videos`, a chained step) `seams` alone is enough: the seams and their names are the `media.shots` its run recorded. For any other file pass `boundaries` - each later shot's first frame, the running sum of the shots' `frame_count` from `get_gallery_metadata` on their own `intermediate/` files - and `names` to name them; either one given overrides the recorded value. Tiles are fitted to `max_dimension` and, when the set would exceed the 4MB budget, shrunk together rather than dropped; the text part lists each tile and says so. `hear=N` also returns N seconds of soundtrack centred on each `at` moment, after its image - the way to check a hit point or lip-sync without reconciling two clocks; a mute clip keeps its frames and says `no soundtrack` | | `get_output_text(name, max_characters=20000, workspace=None)` | `name`, `max_characters`, `workspace` | Read a text output — a prompt enhancement, or any step whose result is `text/plain` or JSON. Reports the file's real length and whether it was truncated. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | | `assess_output(name, probe=None, detail=False, workspace=None)` | `name` (a gallery name or `asset:`), `probe`, `detail`, `workspace` | Measure a finished cut on the server, without queueing: the assessment probes (`analyze_shots`, `analyze_seams`, `analyze_sync_drift`) run in the server process over one decode of the file, beside whatever job holds the GPU. Shot boundaries are the ones its run recorded (the manifest for an output, the `keep_output` sidecar for an asset). The answer merges every applicable probe's `findings` (`{rule, severity, at, value, threshold, says}`) with `rules_applied`, `rules_skipped` and `not_applicable` (`{probe: why}` - a still, a mute file, a file with no recorded shots); `detail=true` adds each probe's full body under `probes`, and `probe=` returns that one probe's full body. `probe` is checked against the three names before anything else is read, and an unknown one is refused naming them. Findings are places to look, not verdicts: drill in with `get_output_frames(seams=[n])` / `get_output_audio(start, duration)`. See *Assessing a run's output* in docs/WORKFLOW_GUIDE.md | | `download_output(name, destination=None, overwrite=False, workspace=None)` | `name`, `destination`, `overwrite`, `workspace` | Save one output file to local disk, of any content type. `destination` may be a full path or a directory; `~` expands and missing parent directories are created. `overwrite=True` is required to replace a file already at the resolved path. Over the stdio `dw-mcp`, omitting `destination` saves under the output's own name in the current working directory. Over a `dw.serve --mcp` endpoint the file lands on the server confined to that workspace (a relative path is joined onto it), and `destination` is required there - an omitted one is refused rather than dropped loose in the workspace root, where nothing can find or delete it later (#353); use the `url` `list_gallery` reports, `get_output_image`/`get_output_audio`/`get_output_frames` for inline content, or `keep_output` to make it a named asset instead. Returns nothing to the conversation but where the file landed — unlike the other media tools, the point is a file on disk, not a payload in context. Writes on the machine running the MCP server - over `dw.serve --mcp` that is the GPU box. A write that fails there (a path that exists only on the client, for instance) comes back as an error naming the server-side write and the client-side alternatives, not as an anonymous tool failure. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | diff --git a/dw_mcp/media.py b/dw_mcp/media.py index c731bd48..f54cb6c4 100644 --- a/dw_mcp/media.py +++ b/dw_mcp/media.py @@ -242,11 +242,14 @@ def get_output_frames( selector per call: `at` (moments: seconds, or "frame:N"), `count` (an evenly spaced contact sheet) or `seams` (True, or seam numbers from 1: the last frame before and the first frame after each boundary, side by - side). `boundaries` is the list of frame indexes each shot after the - first starts at - the running sum of the shots' `frame_count` from - `get_gallery_metadata` on their own files - `names` the shots' names. - Without `boundaries`, an output joined from shots uses the boundaries - its run recorded (`get_gallery_metadata`'s `media.shots`). + side). `boundaries` and `names` are modifiers of `seams` only, for a + file with no `media.shots` of its own - they do nothing alongside `at` + or `count`, and passing either without `seams` is refused. `boundaries` + is the list of frame indexes each shot after the first starts at - the + running sum of the shots' `frame_count` from `get_gallery_metadata` on + their own files - `names` the shots' names. Without `boundaries`, an + output joined from shots uses the boundaries its run recorded + (`get_gallery_metadata`'s `media.shots`). `crop` is `[x, y, width, height]` in the video's own source pixels - the same convention `get_output_image` uses - resolved once against @@ -276,6 +279,13 @@ def get_output_frames( "Pass exactly one of `at`, `count` or `seams`" + (f" - got {', '.join(chosen)}" if chosen else "") ) + if (boundaries or names) and not seams: + modifiers = [n for n, v in (("boundaries", boundaries), ("names", names)) if v] + raise DwApiError( + f"`{'` and `'.join(modifiers)}` only appl{'y' if len(modifiers) > 1 else 'ies'} " + "alongside `seams` - pass `seams=true` (or seam numbers) to use " + f"{'them' if len(modifiers) > 1 else 'it'}" + ) if hear is not None: if not at: raise DwApiError( diff --git a/tests/test_mcp_media.py b/tests/test_mcp_media.py index 62ea7333..d9ce59b4 100644 --- a/tests/test_mcp_media.py +++ b/tests/test_mcp_media.py @@ -1046,6 +1046,20 @@ def test_two_selectors_are_refused_before_any_request(): get_output_frames(client, "x.mp4", at=[0.0], count=4) +def test_names_without_seams_is_refused_before_any_request(): + seen = [] + client = frames_server([], seen) + + with pytest.raises(DwApiError, match="only appl.* alongside `seams`"): + get_output_frames(client, "x.mp4", names=["a", "b"], count=3) + with pytest.raises(DwApiError, match="only appl.* alongside `seams`"): + get_output_frames(client, "x.mp4", boundaries=[8], at=[0.0]) + with pytest.raises(DwApiError, match="only appl.* alongside `seams`"): + get_output_frames(client, "x.mp4", boundaries=[8], names=["a", "b"], count=3) + + assert seen == [] # refused before any request reached the server + + def test_tiles_over_budget_are_shrunk_together_and_say_so(): # three noisy 2048x1024 tiles: well over 4MB base64 between them tiles = [] From f6bc57755e9e2b6107f8c111405551501d3d5ffd Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 01:57:29 -0500 Subject: [PATCH 041/251] fix(mcp): #555 - mix_audio's dB-typo heuristic ignores non-integer gains find_loop_bed hands back a computed multiplier (e.g. 10.374) as mix_audio's documented gains value, but the heuristic flagged any loud gain regardless of shape, warning on the server's own recommendation. A hand-typed dB figure is a round number, so only an integer above the threshold is a likely unit typo now. Co-Authored-By: Claude Sonnet 5 --- dw/tasks/audio_utils.py | 10 ++++++++-- tests/test_mix_audio.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/dw/tasks/audio_utils.py b/dw/tasks/audio_utils.py index 6bca5ba0..82b6d507 100644 --- a/dw/tasks/audio_utils.py +++ b/dw/tasks/audio_utils.py @@ -1050,11 +1050,17 @@ def mix_audio(audios, gains=None, sample_rate=None): # only a gain loud enough that a caller almost certainly meant it as # dB (12, 6, 20, ...) is worth flagging. GAIN_LOOKS_LIKE_DB_ABOVE sits # above any observed catalog default and below the smallest figure a - # dB-as-multiplier typo would produce (a "6 dB" or "12 dB" boost) + # dB-as-multiplier typo would produce (a "6 dB" or "12 dB" boost). + # #555: a hand-typed dB figure is a round number; a computed + # multiplier (find_loop_bed's "gain", meant for this argument) almost + # never lands on an exact integer, so only an integer value above the + # threshold is flagged. loud = [ g for g in gains - if as_number(g) is not None and as_number(g) > GAIN_LOOKS_LIKE_DB_ABOVE + if as_number(g) is not None + and as_number(g) > GAIN_LOOKS_LIKE_DB_ABOVE + and float(as_number(g)).is_integer() ] if loud: emit_warning( diff --git a/tests/test_mix_audio.py b/tests/test_mix_audio.py index fd90f2dd..4cdd5817 100644 --- a/tests/test_mix_audio.py +++ b/tests/test_mix_audio.py @@ -119,6 +119,24 @@ def test_a_modest_boost_does_not_warn(self): ] assert warnings == [] + def test_a_computed_multiplier_does_not_warn(self): + # #555: find_loop_bed hands back a non-integer gain (e.g. 10.374) as + # mix_audio's documented multiplier - a hand-typed dB figure is a + # round number, so only an integer above the threshold is a likely typo. + warnings = [ + e + for e in self.events_of( + lambda: mix_audio( + [_tone(10, 0.1), _tone(10, 0.1)], + gains=[1.0, 10.374], + sample_rate=44100, + ), + "warning", + ) + if e.get("kind") == "mix_audio_gain_not_db" + ] + assert warnings == [] + def test_the_applied_gains_are_logged(self): logs = self.events_of( lambda: mix_audio( From 51bdb9d9976de3c00f4a844c56f7926b8d93ec9d Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 02:48:45 -0500 Subject: [PATCH 042/251] fix(mcp): #556 - keep_output carries its source job's provenance get_gallery_metadata on a kept asset reported job: null even when the file's job was still in history, because keep_output copied only bytes. It now records {job, run_id, version, workspace, source} in the same manifest.json sidecar #393 uses for shots, and gallery_metadata reads it back for an asset the same way it already read shots. Co-Authored-By: Claude Sonnet 5 --- dw/runs.py | 61 ++++++++++++++++++++----- dw/server/app.py | 79 ++++++++++++++++++++++++--------- dw_mcp/catalog.py | 14 +++--- tests/test_server_workspaces.py | 58 ++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 36 deletions(-) diff --git a/dw/runs.py b/dw/runs.py index 1d15e931..bce3b3c8 100644 --- a/dw/runs.py +++ b/dw/runs.py @@ -696,20 +696,25 @@ def recorded_shots(output_root, relative_path): return None -def record_kept_shots(directory, file_name, shots): +def record_kept_shots(directory, file_name, shots, provenance=None): """Write or update the manifest sidecar beside a kept asset so `shots_beside` can read the shot boundaries the source run recorded for - it (#393). + it (#393), and `kept_provenance` can read which job and run it was kept + from (#556). Keeping a file copies its bytes but not the run directory it lived in, so a join's shot records - `pair_audio`'s picture is unchanged, but nothing carried them past `keep_output` - were unreachable from the - asset and every probe saw `shots_source: "none"`. One manifest per - directory, keyed by file name, in the same shape a run's own - `manifest.json` uses, so the existing manifest-reading path (used by - both outputs and assets) picks it up with no change of its own. A - re-keep replaces the entry for that name rather than leaving a stale - one from a differently-shot source; `shots` of None or [] removes it. + asset and every probe saw `shots_source: "none"`. Likewise the job that + wrote the file is the one thing `keep_output` knows and a plain asset + never can, since nothing on the server otherwise remembers which job + produced an asset. One manifest per directory, keyed by file name, in + the same shape a run's own `manifest.json` uses, so the existing + manifest-reading path (used by both outputs and assets) picks it up + with no change of its own. A re-keep replaces the entry for that name + rather than leaving a stale one from a differently-shot or + differently-sourced kept file; `shots` and `provenance` of None (or + `shots` of []) remove the entry entirely. """ manifest_path = os.path.join(directory, MANIFEST_FILE_NAME) manifest = _read_manifest(directory) or {} @@ -718,8 +723,13 @@ def record_kept_shots(directory, file_name, shots): for entry in manifest.get("steps") or [] if not (isinstance(entry, dict) and entry.get("files") == [file_name]) ] - if shots: - steps.append({"step": "keep_output", "files": [file_name], "shots": shots}) + if shots or provenance: + entry = {"step": "keep_output", "files": [file_name]} + if shots: + entry["shots"] = shots + if provenance: + entry["provenance"] = provenance + steps.append(entry) if not steps: try: os.remove(manifest_path) @@ -773,3 +783,34 @@ def shots_beside(path): if shots: return shots return None + + +def kept_provenance(path): + """The job/run `keep_output` recorded for a kept asset, read back from + the same sidecar manifest `shots_beside` reads (#556). None when the + file was never kept through `keep_output` (no sidecar, or an entry with + no `provenance` - kept before this fix, or the source had no job + history of its own).""" + path = os.path.abspath(path) + run_dir = os.path.dirname(path) + for _ in range(MANIFEST_SEARCH_DEPTH): + if os.path.isfile(os.path.join(run_dir, MANIFEST_FILE_NAME)): + break + parent = os.path.dirname(run_dir) + if parent == run_dir: + return None + run_dir = parent + else: + return None + manifest = _read_manifest(run_dir) + if manifest is None: + return None + own = os.path.relpath(path, run_dir).replace(os.sep, "/") + for entry in manifest.get("steps") or []: + if not isinstance(entry, dict) or entry.get("reused"): + continue + if own in (entry.get("files") or []): + provenance = entry.get("provenance") + if provenance: + return provenance + return None diff --git a/dw/server/app.py b/dw/server/app.py index 839561e9..4e3e783c 100644 --- a/dw/server/app.py +++ b/dw/server/app.py @@ -104,6 +104,7 @@ REALIZED_FILE_NAME, is_output_reference, is_run_id, + kept_provenance, record_kept_shots, record_run_versions, resolve_output_reference, @@ -2784,6 +2785,31 @@ def _asset_file(reference, ws): _resolution_roots(ws), ) + def _job_provenance(name, ws): + """The job that wrote an output file, and which run and version - + `gallery_metadata`'s output branch and `keep_output_as_asset` (#556) + both need this, the latter so it can carry it into the kept asset's + sidecar before the run directory it came from is pruned.""" + try: + # Scoped to this workspace: two workspaces can each write a + # file with the same relative name, and an unscoped lookup + # could attribute this one to the wrong workspace's job + job = manager.history.job_for_file(name, workspace=ws.name) + except Exception: + job = None + run_id, version = "", None + folder, run_id, _subfolder = split_run_path(name) + if run_id: + try: + identity_dir = validate_path( + os.path.join(ws.outputs, folder), ws.outputs + ) + except SecurityError: + identity_dir = None + if identity_dir: + version = run_versions(identity_dir).get(run_id) + return job, run_id, version + def _static_files_for(root): """The StaticFiles instance bound to one root, built on first use and cached on app.state - see the comment where the cache is created.""" @@ -3156,37 +3182,35 @@ def gallery_metadata( what decide whether a call will work at all, and for a file the caller is about to *consume* they were previously unobtainable: the only way to read a wav's length was to run a job that copied it - into the output directory. `job` is null for an asset (nothing here - produced it) and `source` says which of the two roots answered.""" + into the output directory. `job` is null for an asset with no + recorded provenance, and `source` says which of the two roots + answered. + + A kept asset (`keep_output`) is not always provenance-blind: when + the file it was kept from still had a job in history, `keep_output` + recorded `{job, run_id, version}` beside it, and this route reads it + back the same way it reads a kept file's shots (#556). `run_id`/ + `version` are that source run's own - `job.id` is still the one to + pass `get_job_workflow`, since a pruned run leaves no `workflow.json` + of its own to reopen.""" run_id, version = "", None name = _strip_output_prefix(name) if is_asset_reference(name): path = _asset_file(name, ws) source, job = "asset", None + kept = kept_provenance(path) + if kept: + job = kept.get("job") + run_id = kept.get("run_id") or "" + version = kept.get("version") else: path = _output_file(name, ws.outputs) source = "output" - try: - # Scoped to this workspace: two workspaces can each write a - # file with the same relative name, and an unscoped lookup - # could attribute this one to the wrong workspace's job - job = manager.history.job_for_file(name, workspace=ws.name) - except Exception: - job = None # Which run wrote it, and that run's ordinal - the same 'v4' the # listing reports. After "look at version 3" this is the next # call, so it confirms the right file was reached rather than # sending the caller back to the listing - folder, run_id, _subfolder = split_run_path(name) - if run_id: - try: - identity_dir = validate_path( - os.path.join(ws.outputs, folder), ws.outputs - ) - except SecurityError: - identity_dir = None - if identity_dir: - version = run_versions(identity_dir).get(run_id) + job, run_id, version = _job_provenance(name, ws) metadata = read_embedded_metadata(path) extension = os.path.splitext(path)[1].lower() media = ( @@ -4143,11 +4167,26 @@ def keep_output_as_asset( # The source run's shot boundaries - carrying bytes without them left # a kept multi-shot cut looking like one shot to every probe, with no - # sign anything was missing (#393) + # sign anything was missing (#393). Alongside them, the job and run + # that wrote the file: keep_output is the one place the server still + # knows which job produced it, since a plain asset has none (#556) + job, source_run_id, source_version = _job_provenance(kept_name, ws) + provenance = ( + { + "job": job, + "run_id": source_run_id, + "version": source_version, + "workspace": ws.name, + "source": kept_name, + } + if job + else None + ) record_kept_shots( os.path.dirname(destination), os.path.basename(destination), recorded_shots(ws.outputs, kept_name), + provenance, ) logger.info(f"Kept output {request.name} as asset:{asset_name}") diff --git a/dw_mcp/catalog.py b/dw_mcp/catalog.py index aa6c7888..2f597119 100644 --- a/dw_mcp/catalog.py +++ b/dw_mcp/catalog.py @@ -281,9 +281,11 @@ def get_gallery_metadata(client, name, envelope=False, workspace=None): always null for audio and video, since neither format has a slot this writer uses. `job` is the fallback recipe when one is known - `next` then names `get_job_workflow(job_id)`, which reads the run's realized - workflow instead. A kept asset (`source: "asset"`) has no job at all, - so nothing on the server remembers which run made it; `next` says so - rather than pretending a lookup exists. + workflow instead. A kept asset (`source: "asset"`) carries `job` too + when `keep_output` recorded one at keep time - `run_id`/`version` are + that source run's own, not the asset's; a kept asset with no recorded + job (kept before this, or from a file with no job history) still + answers `job: null`, and `next` says so. `name` is a gallery name - the `name` field `list_gallery` reports, not its `label` (a display-only basename that is not a valid reference) - @@ -312,9 +314,9 @@ def get_gallery_metadata(client, name, envelope=False, workspace=None): ) elif body.get("source") == "asset": hints.append( - "metadata is null and this is a kept asset, which carries " - "no provenance - nothing on the server remembers which job, " - "if any, produced the file it was kept from." + "metadata is null and this is a kept asset with no " + "recorded job - it was kept before provenance tracking, " + "or from a file with no job history of its own." ) if media and body.get("source") == "asset": hints.append( diff --git a/tests/test_server_workspaces.py b/tests/test_server_workspaces.py index 7ee80763..4d2062b6 100644 --- a/tests/test_server_workspaces.py +++ b/tests/test_server_workspaces.py @@ -572,6 +572,64 @@ def test_a_kept_outputs_shots_survive_and_report_through_the_gallery( sidecar = os.path.join(workspace_root.assets, "qa-cast", "manifest.json") assert os.path.isfile(sidecar) + def test_a_kept_outputs_job_survives_and_reports_through_the_gallery( + self, server, workspace_root + ): + """#556: keeping a file copied only its bytes, so a kept asset's + `get_gallery_metadata` reported `job: null` even when the source file's + job was still in history - keep_output now carries the source job, run + id and version into the same sidecar #393 uses for shots.""" + + class FinishedJob: + id = "9bd7d90745c3" + workflow_name = "w" + catalog_name = None + status = "complete" + created_at = 1.0 + started_at = 1.0 + finished_at = 2.0 + manifest = [{"step": "generate", "files": ["Gyre/20260905-101500-bbbbbbbb/still.png"]}] + warnings = [] + error = None + run_id = "20260905-101500-bbbbbbbb" + run_dir = "Gyre/20260905-101500-bbbbbbbb" + acknowledged = "none" + spec = {"arguments": {}, "workflow_path": "w.json", "workspace": None} + events = [] + + run_dir = os.path.join(workspace_root.outputs, "Gyre/20260905-101500-bbbbbbbb") + os.makedirs(run_dir, exist_ok=True) + with open(os.path.join(run_dir, "still.png"), "wb") as handle: + handle.write(b"png-bytes") + + with server() as client: + manager = client.app.state.job_manager + manager.history.record(FinishedJob()) + + kept = client.post( + "/api/assets/keep", + json={ + "name": "Gyre/20260905-101500-bbbbbbbb/still.png", + "asset_name": "qa-cast/hero.png", + }, + ) + assert kept.status_code == 201 + + metadata = client.get("/api/gallery/asset:qa-cast/hero.png/metadata").json() + + assert metadata["source"] == "asset" + assert metadata["job"] == {"id": "9bd7d90745c3", "status": "complete"} + assert metadata["run_id"] == "20260905-101500-bbbbbbbb" + assert metadata["version"] == 1 + + sidecar = os.path.join(workspace_root.assets, "qa-cast", "manifest.json") + with open(sidecar) as handle: + saved = json.load(handle) + provenance = saved["steps"][0]["provenance"] + assert provenance["job"] == {"id": "9bd7d90745c3", "status": "complete"} + assert provenance["run_id"] == "20260905-101500-bbbbbbbb" + assert provenance["source"] == "Gyre/20260905-101500-bbbbbbbb/still.png" + def test_keeping_stays_inside_the_workspace(self, server, workspace_root): """The source is read from the named workspace's outputs and the copy lands in its assets - neither reaches the default workspace.""" From a2d4ba28295a52db3c90efed40d7b81bf8f49f00 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 02:51:37 -0500 Subject: [PATCH 043/251] style: #556 - ruff-format the new keep_output provenance test Co-Authored-By: Claude Sonnet 5 --- tests/test_server_workspaces.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_server_workspaces.py b/tests/test_server_workspaces.py index 4d2062b6..59625bd2 100644 --- a/tests/test_server_workspaces.py +++ b/tests/test_server_workspaces.py @@ -588,7 +588,12 @@ class FinishedJob: created_at = 1.0 started_at = 1.0 finished_at = 2.0 - manifest = [{"step": "generate", "files": ["Gyre/20260905-101500-bbbbbbbb/still.png"]}] + manifest = [ + { + "step": "generate", + "files": ["Gyre/20260905-101500-bbbbbbbb/still.png"], + } + ] warnings = [] error = None run_id = "20260905-101500-bbbbbbbb" From 5370ac9bb5263ff70240bcb3ede024274cb91f35 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 02:57:51 -0500 Subject: [PATCH 044/251] fix(mcp): #556 - update stale kept-asset-provenance test test_gallery_metadata_says_a_kept_asset_has_no_provenance asserted the pre-fix docstring/hint wording ("no provenance"), which no longer holds now that keep_output carries provenance when it can. Renamed and updated to match the current hint ("no recorded job"). Co-Authored-By: Claude Sonnet 5 --- tests/test_mcp_catalog.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_mcp_catalog.py b/tests/test_mcp_catalog.py index e7b786ea..db22c1e8 100644 --- a/tests/test_mcp_catalog.py +++ b/tests/test_mcp_catalog.py @@ -322,10 +322,11 @@ def test_gallery_metadata_points_at_get_job_workflow_when_embedded_is_null(): assert "c68bc29607ec" in result["next"] -def test_gallery_metadata_says_a_kept_asset_has_no_provenance(): - """#384: a kept asset has 'job: null' by construction - nothing traces - it back to the run that made it, and the hint has to say that rather - than staying silent about the null 'metadata'.""" +def test_gallery_metadata_says_a_kept_asset_has_no_recorded_job(): + """#556: a kept asset with no recorded job (kept before provenance + tracking, or from a file with no job history) still answers 'job: null', + and the hint has to say that rather than staying silent about the null + 'metadata'.""" body = { "name": "asset:qa-cast/ep37-shot2-alibi.mp4", "source": "asset", @@ -344,7 +345,7 @@ def test_gallery_metadata_says_a_kept_asset_has_no_provenance(): result = catalog.get_gallery_metadata(client, "asset:qa-cast/ep37-shot2-alibi.mp4") - assert "no provenance" in result["next"] + assert "no recorded job" in result["next"] assert "get_job_workflow" not in result["next"] From 989af547e9613fa580938ebf8e5bcf7ade829a29 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 03:13:58 -0500 Subject: [PATCH 045/251] fix(mcp): #557 - round a frame-addressed slice's end once, not in two halves slice_audio computed start and length from separately rounded halves (round(start_frame) + round(num_frames) samples), so a slice meant to reach the source's exact end could land a sample short and fire a 'dropping its tail' warning whose own figures read 0.00 s. The end sample is now round(end_frame * sr / fps), matching how the frame grid is meant to align. Also floors the tail-trim warning at 10 ms, the same rounding-noise threshold the pad warning already uses, so a remainder too short to act on says nothing regardless of how it arose. Co-Authored-By: Claude Sonnet 5 --- dw/tasks/audio_utils.py | 22 ++++++++++---- tests/test_audio_utils.py | 60 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/dw/tasks/audio_utils.py b/dw/tasks/audio_utils.py index 82b6d507..18dd5f0d 100644 --- a/dw/tasks/audio_utils.py +++ b/dw/tasks/audio_utils.py @@ -41,6 +41,12 @@ SLICE_TRIM_WARN_SECONDS = 10.0 SLICE_TRIM_WARN_FRACTION = 0.05 +# A remainder shorter than this is the rounding frame-aligned slicing +# produces (or a sub-millisecond seconds-addressed remainder), not a +# noticeable dropped tail - the same floor SLICE_PAD_WARN_MS applies on the +# other side of a slice +SLICE_TRIM_WARN_MIN_MS = 10.0 + def as_channels_samples(audio): """Normalize a waveform to a (channels, samples) float32 numpy array. @@ -568,11 +574,15 @@ def slice_audio( if fps is None: raise ValueError("slice_audio needs 'fps' to address a slice in frames") start = frames_to_samples(start_frame or 0, fps, sample_rate) - length = ( - max(total - start, 0) - if num_frames is None - else frames_to_samples(num_frames, fps, sample_rate) - ) + if num_frames is None: + length = max(total - start, 0) + else: + # Round the end frame directly rather than adding two separately + # rounded halves - start's and num_frames' - which can each round + # down half a sample and together land one sample short of a + # slice meant to reach the source's exact end (#557) + end = frames_to_samples((start_frame or 0) + num_frames, fps, sample_rate) + length = end - start else: raise ValueError( "slice_audio needs either 'start_seconds'/'duration_seconds' or " @@ -779,6 +789,8 @@ def _warn_on_slice_trims_tail(waveform, total, start, length, sample_rate): if remainder <= 0: return remainder_seconds = remainder / float(sample_rate) + if remainder_seconds * 1000.0 < SLICE_TRIM_WARN_MIN_MS: + return if remainder_seconds >= SLICE_TRIM_WARN_SECONDS: return if remainder_seconds / (length / float(sample_rate)) >= SLICE_TRIM_WARN_FRACTION: diff --git a/tests/test_audio_utils.py b/tests/test_audio_utils.py index b3e0b93b..dd71ccac 100644 --- a/tests/test_audio_utils.py +++ b/tests/test_audio_utils.py @@ -1377,6 +1377,66 @@ def test_a_start_beyond_the_end_is_reported_as_all_padding(self): assert warnings[0]["padded_seconds"] == pytest.approx(2.0) +class TestSliceEndingExactlyAtTheSourceReachesIt: + """#557: a frame-addressed slice used to round its start and its length + separately (round(start_frame) samples + round(num_frames) samples), so + a slice meant to reach the source's exact end could land a sample short + and fire a 'dropping its tail' warning whose own figures read 0.00 s. + The end is now rounded once - round(end_frame * sr / fps) - so a slice + whose start_frame + num_frames is the source's own frame count reaches + its last sample exactly and warns about nothing. + """ + + def events_from(self, call): + from dw.events import RunContext, activate_context, deactivate_context + + events = [] + token = activate_context(RunContext(on_event=events.append)) + try: + result = call() + finally: + deactivate_context(token) + return result, [e for e in events if e.get("kind") == "slice_trimmed_tail"] + + def tone(self, count, rate, channels=1): + return numpy.full((channels, count), 0.5, dtype=numpy.float32) + + def test_the_557_repro_lands_exactly_and_warns_nothing(self): + from dw.tasks.audio_utils import slice_audio + + # 294 frames @ 24 fps, 44.1 kHz -> 540225 samples total; the second + # half (147..294) is exactly the tail the #557 repro sliced + sliced, warnings = self.events_from( + lambda: slice_audio( + self.tone(540225, 44100), + start_frame=147, + num_frames=147, + fps=24, + sample_rate=44100, + ) + ) + + assert samples(sliced).shape == (270113, 1) + assert warnings == [] + + def test_a_sub_ten_millisecond_remainder_says_nothing(self): + """Even outside the frame-addressed case, a remainder too short to + act on is rounding noise, not a dropped tail worth flagging.""" + from dw.tasks.audio_utils import slice_audio + + sliced, warnings = self.events_from( + lambda: slice_audio( + self.tone(10000, 1000), + start_seconds=0, + duration_seconds=9.9955, + sample_rate=1000, + ) + ) + + assert samples(sliced).shape == (9996, 1) + assert warnings == [] + + class TestSliceAudioCarriesSourceLevel: """#309: a slice out of already-quiet source material (room tone) is not a defect the slice introduced - slice_audio measures the source's own From 8ecc028d12c28428e94a0bb22f2a7c574f73ac57 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 04:21:52 -0500 Subject: [PATCH 046/251] fix(mcp): #559 - request timestamps internally for every Whisper transcribe_audio call Without return_timestamps, Whisper's decoder can emit an early end-of-text after a pause between lines, so plain-text mode truncated a multi-line clip well under the 30s long-form threshold while segment/word mode (which already forced return_timestamps) returned the full text. Force it on for every Whisper call regardless of length; only the caller's requested mode still surfaces chunks. Co-Authored-By: Claude Sonnet 5 --- dw/tasks/audio_transcription.py | 14 +++++++---- tests/test_transcribe_audio.py | 43 ++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/dw/tasks/audio_transcription.py b/dw/tasks/audio_transcription.py index 7aa455de..1dd75630 100644 --- a/dw/tasks/audio_transcription.py +++ b/dw/tasks/audio_transcription.py @@ -26,7 +26,6 @@ _DEFAULT_ASR_MODEL = "openai/whisper-base" _ASR_SAMPLE_RATE = 16000 -_WHISPER_WINDOW_SECONDS = 30 # Public: dw/scalar_result_validation.py checks a literal `timestamps` # argument against this same tuple to catch a `result.content_type` that # does not match the {text, chunks} dict shape timestamps switches the @@ -99,9 +98,14 @@ def load_pipe(): # Whisper takes 30 s per window and refuses a longer clip ("more than 3000 # mel input features") unless it predicts timestamps, which is how its - # long-form mode stitches windows. Asked for only of Whisper and only past - # 30 s: transformers raises for a CTC model unless the value is "char" or - # "word", and for any other seq2seq model at all + # long-form mode stitches windows. Without timestamps, Whisper's decoder + # can also emit an early end-of-text after a pause between lines - a + # multi-line clip with gaps between shots then transcribes only the first + # line even well under 30 s (#559). Forcing return_timestamps on for every + # Whisper call, plain-text included, avoids both: only "word"/"segment" + # callers see the chunks, plain mode still returns just the full text. + # Asked for only of Whisper: transformers raises for a CTC model unless + # the value is "char" or "word", and for any other seq2seq model at all is_whisper = getattr(pipe, "type", None) == "seq2seq_whisper" options = {} if timestamps == "word": @@ -109,7 +113,7 @@ def load_pipe(): options["return_timestamps"] = "word" elif timestamps == "segment": options["return_timestamps"] = True - elif is_whisper and len(mono) > _WHISPER_WINDOW_SECONDS * _ASR_SAMPLE_RATE: + elif is_whisper: options["return_timestamps"] = True result = pipe( {"raw": mono.astype(numpy.float32), "sampling_rate": _ASR_SAMPLE_RATE}, diff --git a/tests/test_transcribe_audio.py b/tests/test_transcribe_audio.py index 0d39caeb..3e8b3da3 100644 --- a/tests/test_transcribe_audio.py +++ b/tests/test_transcribe_audio.py @@ -102,14 +102,55 @@ def test_a_long_clip_on_a_ctc_model_does_not_ask_for_timestamps( self.assertNotIn("return_timestamps", pipe.call_args.kwargs) @patch("dw.tasks.audio_transcription.hf_pipeline") - def test_a_short_clip_does_not_ask_for_timestamps(self, mock_pipeline): + def test_a_short_clip_on_whisper_still_asks_for_timestamps(self, mock_pipeline): + # Without return_timestamps, Whisper's decoder can emit an early + # end-of-text after a pause between lines and truncate a well-under- + # 30s multi-line clip (#559) - so plain mode asks for timestamps + # internally too, on every Whisper call regardless of length pipe = self._mock_pipe(mock_pipeline) waveform, rate = self._waveform(seconds=5.0) transcribe_audio(waveform, device="cpu", sample_rate=rate) + self.assertIs(pipe.call_args.kwargs.get("return_timestamps"), True) + + @patch("dw.tasks.audio_transcription.hf_pipeline") + def test_a_short_clip_on_a_ctc_model_does_not_ask_for_timestamps( + self, mock_pipeline + ): + pipe = self._mock_pipe(mock_pipeline, kind="ctc") + waveform, rate = self._waveform(seconds=5.0) + + transcribe_audio(waveform, device="cpu", sample_rate=rate) + self.assertNotIn("return_timestamps", pipe.call_args.kwargs) + @patch("dw.tasks.audio_transcription.hf_pipeline") + def test_plain_mode_returns_full_text_across_a_pause(self, mock_pipeline): + # The real-world repro (#559): a joined clip with silence between + # three spoken lines. The timestamped decode path already returns + # every line in `text`; plain mode must return the same text, not + # just the first line before the pause. + self._mock_pipe( + mock_pipeline, + "I never touched the pistachio. Spoon was in your sink, Hal. " + "Pistachio on the handle.", + chunks=[ + {"text": "I never touched the pistachio.", "timestamp": (0.0, 2.0)}, + {"text": "Spoon was in your sink, Hal.", "timestamp": (2.0, 4.0)}, + {"text": "Pistachio on the handle.", "timestamp": (4.0, 7.0)}, + ], + ) + waveform, rate = self._waveform(seconds=6.8) + + result = transcribe_audio(waveform, device="cpu", sample_rate=rate) + + self.assertEqual( + result, + "I never touched the pistachio. Spoon was in your sink, Hal. " + "Pistachio on the handle.", + ) + @patch("dw.tasks.audio_transcription.hf_pipeline") def test_resamples_to_16khz(self, mock_pipeline): pipe = self._mock_pipe(mock_pipeline) From edb90fca97a1cfe2bd797a090ff0f65ddba03550 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 04:33:13 -0500 Subject: [PATCH 047/251] fix(mcp): #560 - tag composed sub-workflow manifest entries with parent_step A for_each step whose members each compose the same catalog template, with no result of its own, rolled the child's manifest entries into the parent's manifest unattributed - two members produced two entries under the identical, repeated child step name with nothing tying an entry back to the for_each member that produced it. Tag each rolled-up entry with the nearest composing step's name (setdefault, so a deeper nested rollup keeps its own tag rather than being overwritten by an ancestor). Co-Authored-By: Claude Sonnet 5 --- dw/workflow.py | 10 +++++- tests/test_workflow.py | 78 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/dw/workflow.py b/dw/workflow.py index bb5e9e51..efdec2ea 100644 --- a/dw/workflow.py +++ b/dw/workflow.py @@ -1671,7 +1671,15 @@ def run( if not parent_saves_this: self.manifest.append(manifest_entry) # roll the child's saves up so job history and the gallery see - # every file + # every file. Each entry is tagged with the composing step + # that produced it - a for_each member's files otherwise sit + # under the child template's own (repeated) step name with + # nothing tying an entry back to its member (#560). A deeper + # rollup (a child composing a grandchild) already carries its + # own tag, which stays: the nearest composing step is the one + # that matters for grouping + for sub_entry in sub_manifest: + sub_entry.setdefault("parent_step", step.name) self.manifest.extend(sub_manifest) step_end_data = {"files": saved_files, "subfolder": subfolder} if reused: diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 6ce79c28..e8a13693 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -1121,6 +1121,84 @@ def test_a_composed_file_carries_the_parent_step_name(self, tmp_path): "sub.child-write" ) + def test_the_childs_own_entry_is_tagged_with_the_composing_step(self, tmp_path): + """A parent that declares no result leaves the child's own manifest + entry in place under the child's step name ('write') - it now also + carries 'parent_step' naming the composing step ('sub'), so a + consumer can tie the entry back to what produced it without parsing + filenames (#560).""" + workflow = self._compose(tmp_path, parent_result=False) + + by_step = {entry["step"]: entry for entry in workflow.manifest} + assert "parent_step" not in by_step["sub"] + assert by_step["write"]["parent_step"] == "sub" + + +class TestForEachOverComposedTemplate: + """A for_each step whose members each compose a catalog template, with + no result of its own, used to list every member's files under the same + repeated child step name with nothing tying an entry to its member + (#560).""" + + def _run(self, tmp_path): + import json + + workflows = tmp_path / "workflows" + workflows.mkdir() + child = { + "id": "child", + "variables": {"text": "x"}, + "steps": [ + { + "name": "write", + "task": { + "command": "compose_text", + "arguments": {"parts": ["variable:text"]}, + }, + "result": {"content_type": "text/plain"}, + } + ], + } + (workflows / "child.json").write_text(json.dumps(child)) + parent = { + "id": "parent", + "variables": {"shots": [{"name": "answer"}, {"name": "insist"}]}, + "steps": [ + { + "name": "shot", + "for_each": "variable:shots", + "workflow": { + "path": "child.json", + "arguments": {"text": "item:name"}, + }, + } + ], + } + parent_path = workflows / "parent.json" + parent_path.write_text(json.dumps(parent)) + + from dw.workflow import workflow_from_file + + workflow = workflow_from_file( + str(parent_path), str(tmp_path / "outputs"), str(workflows) + ) + workflow.run({}, {}) + return workflow + + def test_each_members_files_are_tagged_with_its_own_member_name(self, tmp_path): + workflow = self._run(tmp_path) + + write_entries = [e for e in workflow.manifest if e["step"] == "write"] + assert len(write_entries) == 2 + assert {e["parent_step"] for e in write_entries} == { + "shot@answer", + "shot@insist", + } + member_entries = { + e["parent_step"]: e["files"] for e in write_entries if e["files"] + } + assert len(member_entries) == 2 + class TestSubWorkflowPreviousResultArgument: """A 'previous_result:' argument folded into a sub-workflow step is From 090705db996484d30b867d26ab4d9b2fd7a3d773 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 05:21:59 -0500 Subject: [PATCH 048/251] fix(mcp): #561 - conform a composed child's own artifact when its parent owns saving A composed for_each member's own last step never wrote a file when its parent step declared a result (parent_saves_this, #92): the fps stamp-back and audio-to-frames fit that save_audio_video applied only ever ran as a side effect of that write. With the write skipped, the artifact handed up to the parent kept fps=None, and the parent's own result_definition - the composing step's own result block, not the child's - declares no fps either, so video_fps() fell back to DEFAULT_VIDEO_FPS (8) with no warning. LTX-2.5 image-to-video members were written at 8fps instead of the child template's declared 24. Factor the stamp-back and fit out of save_audio_video into Result.conform_artifact/conform_artifacts, and call conform_artifacts from workflow.py's parent_saves_this branch in place of the skipped save(), so a child's declared fps and audio fit always land on the artifact whether or not that child ends up writing its own file. Co-Authored-By: Claude Sonnet 5 --- dw/result.py | 55 ++++++++++++++++++++++++++++++++++++++------------ dw/workflow.py | 16 ++++++++++----- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/dw/result.py b/dw/result.py index cb7eaf45..1a598303 100644 --- a/dw/result.py +++ b/dw/result.py @@ -1189,17 +1189,17 @@ def video_fps(self, artifact): ) return declared - def save_audio_video(self, artifact, output_path, content_type): - """Write a video and the audio generated with it into a single file. - - encode_video muxes the two into an h264/mp4 file with PyAV. When PyAV is missing, - the container is not mp4, or nothing told us the sample rate, the video is written - on its own and the audio is dropped. - - Args: - artifact: AudioVideo holding the frames and their waveform - output_path: Path of the file to write - content_type: MIME type of the video being written + def conform_artifact(self, artifact): + """Stamp this result's declared fps onto artifact and fit its audio + to its own frame count. Returns the resolved fps. + + save_audio_video applies this right before writing. A composed + child's own last step never reaches that write when its parent owns + saving (#92, parent_saves_this in workflow.py) - so without this + method a child's declared fps, and its own audio-to-frames fit, + would never reach the artifact a later previous_result: consumer + (concat_videos, dissolve_videos, or the parent's own save) reads + (#561). Called from both places so the two cannot drift. """ fps = self.video_fps(artifact) # A declared result.fps is the rate this video now plays at, so it is @@ -1251,8 +1251,37 @@ def save_audio_video(self, artifact, output_path, content_type): from .shots import measured_num_samples measured_num_samples(artifact.shots, fitted.shape[axis]) - audio = fitted - artifact.audio = audio + artifact.audio = fitted + return fps + + def conform_artifacts(self): + """conform_artifact for every AudioVideo this result holds. + + Called in place of a skipped save (parent_saves_this, workflow.py) + so a composed child that never writes its own file still leaves its + declared fps and frame-fitted audio on the artifacts it hands up. + """ + for artifact in self.get_artifacts(): + if isinstance(artifact, AudioVideo): + self.conform_artifact(artifact) + + def save_audio_video(self, artifact, output_path, content_type): + """Write a video and the audio generated with it into a single file. + + encode_video muxes the two into an h264/mp4 file with PyAV. When PyAV is missing, + the container is not mp4, or nothing told us the sample rate, the video is written + on its own and the audio is dropped. + + Args: + artifact: AudioVideo holding the frames and their waveform + output_path: Path of the file to write + content_type: MIME type of the video being written + """ + fps = self.conform_artifact(artifact) + sample_rate = self.result_definition.get( + "audio_sample_rate", artifact.sample_rate + ) + audio = artifact.audio # Segment-backed frames (a chained step with save_segments) replay from # disk one segment at a time, so the final video is streamed instead of diff --git a/dw/workflow.py b/dw/workflow.py index efdec2ea..059c7535 100644 --- a/dw/workflow.py +++ b/dw/workflow.py @@ -1605,14 +1605,20 @@ def run( ) if not reused: - saved_files = ( - [] - if parent_saves_this - else result.save( + if parent_saves_this: + # No file is written here - the parent owns that + # (#92) - but this step's own declared fps and its + # audio-to-frames fit must still land on the artifact + # before it is handed up, or the parent (and any + # previous_result: consumer) sees an unstamped one + # and falls back to DEFAULT_VIDEO_FPS (#561) + result.conform_artifacts() + saved_files = [] + else: + saved_files = result.save( self.step_output_dir(step_data), self.step_save_name(workflow_id, step.name, i), ) - ) if is_cacheable: step_cache.put( workflow_id, From 465c6b8f3e52c0f37d9fce0fd4f091609e1c2ab8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 05:22:04 -0500 Subject: [PATCH 049/251] test(mcp): #561 - cover conform_artifacts, the parent-owns-saving code path Result.conform_artifact/conform_artifacts (previous commit) is exactly what workflow.py now calls when a composed child's own save() is skipped because its parent owns saving - exercise it directly rather than only through save_audio_video, so the declared-fps stamp-back and audio-to-frames fit are covered on the path that never reaches a write. Co-Authored-By: Claude Sonnet 5 --- tests/test_result.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_result.py b/tests/test_result.py index dd960f6c..10b0c08f 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -964,6 +964,53 @@ def test_missing_pyav_falls_back_to_export(self): export.assert_called_once() +class TestConformArtifacts: + """#561: a composed child's own last step, when its parent owns saving + (parent_saves_this in workflow.py), never calls save() - so without a + way to reach the fps stamp-back and audio-to-frames fit outside of + save_audio_video, a child's declared fps never lands on the artifact it + hands up, and the parent (or a previous_result: consumer) falls back to + DEFAULT_VIDEO_FPS. conform_artifacts is workflow.py's replacement for + the skipped save() call.""" + + def test_declared_fps_is_stamped_without_a_save_call(self): + artifact = AudioVideo("frames", torch.zeros((2, 100)), 48000) + result = Result({"content_type": "video/mp4", "fps": 24}) + result.add_result(artifact) + + result.conform_artifacts() + + assert artifact.fps == 24 + + def test_audio_is_fit_to_the_frame_count_without_a_save_call(self): + frames = ["frame"] * 48 + audio = torch.zeros((2, 1900)) # 48 frames @ 24fps @ 1000Hz -> 2000 + artifact = AudioVideo(frames, audio, 1000) + result = Result({"content_type": "video/mp4", "fps": 24}) + result.add_result(artifact) + + result.conform_artifacts() + + assert artifact.audio.shape == (2, 2000) + + def test_every_audio_video_artifact_in_the_result_is_conformed(self): + first = AudioVideo("frames", torch.zeros((2, 100)), 48000) + second = AudioVideo("frames", torch.zeros((2, 100)), 48000) + result = Result({"content_type": "video/mp4", "fps": 30}) + result.add_result([first, second]) + + result.conform_artifacts() + + assert first.fps == 30 + assert second.fps == 30 + + def test_a_non_audio_video_artifact_is_left_alone(self): + result = Result({"content_type": "image/png", "fps": 24}) + result.add_result("not-an-audio-video") + + result.conform_artifacts() # must not raise + + class TestNormalizeAudio: """Test conversion of pipeline audio output into writable waveforms""" From f616fd542d94a4f5b5db79cc562fbc65669cdc6e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 05:22:11 -0500 Subject: [PATCH 050/251] fix(mcp): #562 - conform each concat_videos input's audio to its own frame count concat_videos only ever fit the aggregate joined track to the frame grid, once, at the very end (#553). A middle input whose own audio ran short or long against its own clip's frame count shifted every seam after it before that final catch-up - small enough per seam to land under analyze_sync_drift's single-seam threshold, but compounding across however many further joins took that output as an input. Fit each real-audio input to its own frames_to_samples() length before it is concatenated, reusing fit_audio_to_frames (which already pads a short input and warns past a frame of padding). fit_audio_to_frames deliberately leaves an over-length input alone (#378 - a shot keeps the samples it actually took), so add a new audio_frame_drift warning for that direction, since that is exactly the drift this issue is about and the existing pad warning doesn't cover it. Co-Authored-By: Claude Sonnet 5 --- dw/tasks/concat_videos.py | 28 +++++++++++++ tests/test_concat_videos.py | 82 +++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/dw/tasks/concat_videos.py b/dw/tasks/concat_videos.py index 47a570b2..fecfa2c5 100644 --- a/dw/tasks/concat_videos.py +++ b/dw/tasks/concat_videos.py @@ -292,6 +292,34 @@ def concat_videos( ) else: waveform = waveforms[index] + input_fps = fps or getattr(video, "fps", None) + if input_fps: + # A per-input shortfall against its own frame grid propagates + # into the join and compounds across every further join that + # takes this one's own output as an input (#435/#553's + # remedy for the aggregate track, extended here per input - + # #562). fit_audio_to_frames pads a short track and warns + # once the gap is a frame or more; it deliberately leaves an + # *over*-length track alone (#378 - a shot keeps the samples + # it actually took), so that direction is warned here + # instead, since it is exactly the drift analyze_sync_drift's + # single-seam threshold misses + wanted_samples = frames_to_samples(len(clip), input_fps, sample_rate) + overrun_samples = waveform.shape[1] - wanted_samples + if overrun_samples > sample_rate / input_fps: + emit_warning( + f"concat_videos: {names[index]}'s audio is " + f"{overrun_samples} sample(s) longer than its own " + f"{len(clip)}-frame length before the join - drift " + f"like this compounds at the seam", + command="concat_videos", + kind="audio_frame_drift", + video=names[index], + drift_samples=int(overrun_samples), + ) + waveform = fit_audio_to_frames( + waveform, sample_rate, len(clip), input_fps, "concat_videos" + ) if audio is None: audio = waveform diff --git a/tests/test_concat_videos.py b/tests/test_concat_videos.py index 9293dfa8..48969800 100644 --- a/tests/test_concat_videos.py +++ b/tests/test_concat_videos.py @@ -811,6 +811,88 @@ def test_no_fps_leaves_the_track_alone(self): assert result.audio.shape[1] == 90 + 100 +class TestPerInputAudioConforming: + """#562: only the aggregate joined track was reconciled against the + frame grid, at the very end (see TestJoinedAudioFitsTheFrameGrid) - so a + middle input's own shortfall against its own frame grid shifted every + seam after it before that final catch-up, small enough per seam to miss + analyze_sync_drift's single-seam threshold but compounding across + however many joins came after it.""" + + def test_a_middle_inputs_own_shortfall_lands_the_next_seam_on_grid(self): + from dw.tasks.audio_utils import frames_to_samples + + short = AudioVideo( + frames(4), numpy.full((2, 70), 0.5, dtype=numpy.float32), 100 + ) + + result = concat_videos( + [audio_video(4, 0.5), short, audio_video(4, 0.5)], fps=4 + ) + + third_shot = result.shots[-1] + assert third_shot["start_sample"] == frames_to_samples(8, 4, 100) + + def test_a_middle_inputs_shortfall_is_still_warned(self): + from dw.events import RunContext, activate_context, deactivate_context + + short = AudioVideo( + frames(4), numpy.full((2, 70), 0.5, dtype=numpy.float32), 100 + ) + + events = [] + token = activate_context(RunContext(on_event=events.append)) + try: + concat_videos([audio_video(4, 0.5), short, audio_video(4, 0.5)], fps=4) + finally: + deactivate_context(token) + + warnings = [ + e for e in events if e.get("kind") == "joined_audio_padded_to_frames" + ] + assert len(warnings) == 1 + assert warnings[0]["pad_samples"] == 30 + + def test_an_over_length_input_warns_and_is_left_unaligned(self): + from dw.events import RunContext, activate_context, deactivate_context + from dw.tasks.audio_utils import frames_to_samples + + long = AudioVideo( + frames(4), numpy.full((2, 130), 0.5, dtype=numpy.float32), 100 + ) + + events = [] + token = activate_context(RunContext(on_event=events.append)) + try: + result = concat_videos([audio_video(4, 0.5), long], fps=4) + finally: + deactivate_context(token) + + warnings = [e for e in events if e.get("kind") == "audio_frame_drift"] + assert len(warnings) == 1 + assert warnings[0]["video"] == "video 2" + assert warnings[0]["drift_samples"] == 30 + # #378: an overrun is measured, never trimmed + assert result.audio.shape[1] == 100 + 130 + assert result.shots[-1]["num_samples"] == frames_to_samples(4, 4, 100) + 30 + + def test_a_sub_frame_overrun_draws_no_warning(self): + from dw.events import RunContext, activate_context, deactivate_context + + barely_long = AudioVideo( + frames(4), numpy.full((2, 105), 0.5, dtype=numpy.float32), 100 + ) + + events = [] + token = activate_context(RunContext(on_event=events.append)) + try: + concat_videos([audio_video(4, 0.5), barely_long], fps=4) + finally: + deactivate_context(token) + + assert [e for e in events if e.get("kind") == "audio_frame_drift"] == [] + + class TestFrameRateTravelsWithTheJoin: """result.fps defaults to 8, so a 24 fps cut that says nothing there used to be written three times slow against audio of the right length - From 430550dbb878a90d57a35e55c741d1d94b9eda96 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 05:27:27 -0500 Subject: [PATCH 051/251] style(mcp): #562 - satisfy ruff format on test_concat_videos.py Co-Authored-By: Claude Sonnet 5 --- tests/test_concat_videos.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_concat_videos.py b/tests/test_concat_videos.py index 48969800..713876ff 100644 --- a/tests/test_concat_videos.py +++ b/tests/test_concat_videos.py @@ -826,9 +826,7 @@ def test_a_middle_inputs_own_shortfall_lands_the_next_seam_on_grid(self): frames(4), numpy.full((2, 70), 0.5, dtype=numpy.float32), 100 ) - result = concat_videos( - [audio_video(4, 0.5), short, audio_video(4, 0.5)], fps=4 - ) + result = concat_videos([audio_video(4, 0.5), short, audio_video(4, 0.5)], fps=4) third_shot = result.shots[-1] assert third_shot["start_sample"] == frames_to_samples(8, 4, 100) From 4126d50e8911d583d72445b242b76061a9647908 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 05:50:00 -0500 Subject: [PATCH 052/251] fix(mcp): #561 - conform a composed child's artifacts before they cross the workflow boundary The prior fix (090705d) stamped fps/fitted audio onto artifacts cached inside conform_artifacts's own Result instance, but Workflow.run hands result_list - not that cache - to the parent step's Step.run, which folds it into a fresh Result via add_result. That Result's own get_artifact_list() re-extracted a brand new, unstamped AudioVideo from the raw pipeline output and fell back to DEFAULT_VIDEO_FPS (8), so for_each members composing templates/ltx2/image-to-video kept writing at 8fps with truncated audio and no warning. conform_artifacts now replaces result_list with the extracted, conformed artifacts themselves, so the parent receives the already- stamped AudioVideo instances directly - get_artifact_list's isinstance(result, AudioVideo) passthrough hands them back unchanged rather than re-extracting. New regression test drives the real cross-instance handoff: a raw pipeline output through a child Result's conform_artifacts(), folded into a separate parent Result via add_result, then asserts on the fps/ audio actually passed to encode_video on save() - confirmed to fail against the previously deployed fix (8 != 24) and pass with this one. Co-Authored-By: Claude Sonnet 5 --- dw/result.py | 27 +++++++++++++++++++++++---- tests/test_result.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/dw/result.py b/dw/result.py index 1a598303..ed7d6c9f 100644 --- a/dw/result.py +++ b/dw/result.py @@ -1255,15 +1255,34 @@ def conform_artifact(self, artifact): return fps def conform_artifacts(self): - """conform_artifact for every AudioVideo this result holds. + """conform_artifact for every AudioVideo this result holds, and + replace result_list with the extracted, conformed artifacts. Called in place of a skipped save (parent_saves_this, workflow.py) so a composed child that never writes its own file still leaves its declared fps and frame-fitted audio on the artifacts it hands up. + Workflow.run returns this Result's result_list to the parent step's + Step.run, which folds it into its own, separate Result via + add_result - a fresh _artifact_cache, keyed by id() of the raw + pipeline output. Stamping only the artifact get_artifacts() returns + (as before) leaves result_list holding the original raw output + (frames/audio attributes, no fps of its own); the parent's own + get_artifact_list() then rebuilds a brand new, unstamped AudioVideo + from it and falls back to DEFAULT_VIDEO_FPS (#561, the deployed + #561 fix that didn't hold). Replacing result_list's entries with the + conformed artifacts themselves means the parent receives these exact + instances, and get_artifact_list's AudioVideo passthrough + (isinstance(result, AudioVideo): return [result]) hands them back + unchanged instead of re-extracting. """ - for artifact in self.get_artifacts(): - if isinstance(artifact, AudioVideo): - self.conform_artifact(artifact) + conformed = [] + for result in self.result_list: + artifacts = self._artifacts_for(result) + for artifact in artifacts: + if isinstance(artifact, AudioVideo): + self.conform_artifact(artifact) + conformed.extend(artifacts) + self.result_list = conformed def save_audio_video(self, artifact, output_path, content_type): """Write a video and the audio generated with it into a single file. diff --git a/tests/test_result.py b/tests/test_result.py index 10b0c08f..c20bd868 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -1010,6 +1010,44 @@ def test_a_non_audio_video_artifact_is_left_alone(self): result.conform_artifacts() # must not raise + def test_stamped_fps_survives_a_fresh_result_across_the_workflow_boundary(self): + # #561 reopened: Workflow.run hands a composed child's result_list to + # the *parent* step's own, separate Result (Step.run's add_result) - + # a fresh _artifact_cache. Stamping only the artifact get_artifacts() + # returned (the original fix) left result_list holding the raw + # pipeline output; the parent's own get_artifact_list() then rebuilt + # a brand new, unstamped AudioVideo from it and fell back to + # DEFAULT_VIDEO_FPS when it wrote the file - the fix has to survive + # the actual save() call on a *different* Result instance, not just + # a direct get_artifacts()/attribute check on the same one. + frames = ["frame"] * 48 + audio = [torch.zeros((2, 1900))] # 48 frames @ 24fps @ 1000Hz -> 2000 + pipeline_output = types.SimpleNamespace( + frames=[frames], audio=audio, audio_sample_rate=1000 + ) + child_result = Result({"content_type": "video/mp4", "fps": 24}) + child_result.add_result(pipeline_output) + + child_result.conform_artifacts() + + # What Workflow.run returns to the parent's Step.run, folded via + # add_result into a fresh Result with no fps of its own declared - + # the shape of a for_each member's "shot" step result block in #561 + parent_result = Result({"content_type": "video/mp4"}) + parent_result.add_result(child_result.result_list) + + with ( + patch("dw.result.encode_video") as encode, + patch("dw.result.export_to_video") as export, + patch("dw.result.is_av_available", return_value=True), + ): + with tempfile.TemporaryDirectory() as temp_dir: + parent_result.save(temp_dir, "test") + + export.assert_not_called() + assert encode.call_args.kwargs["fps"] == 24 + assert encode.call_args.kwargs["audio"].shape == (2, 2000) + class TestNormalizeAudio: """Test conversion of pipeline audio output into writable waveforms""" From 47e3ebd76c22c6e665292495e55625fb705a0e3c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 06:16:04 -0500 Subject: [PATCH 053/251] fix(mcp): #563 - warn when pair_audio regrids a real measured shot position remeasured_shots recomputes every interior shot's start_sample purely from the frame grid, discarding whatever real position a join (concat_videos) had measured there. That's correct for a genuinely new soundtrack, but assemble-and-score's 'film' step pairs the picture with a track that is a pointwise transform (resample/fade/mix/normalize) of the same audio the join already measured - so a real inner-seam drift the join found gets silently overwritten with an on-grid number. Adds a warning (kind shot_position_regridded) when the regrid moves an interior shot's start_sample by 5ms or more from its prior measured position, pointing the caller at the join's own shots for per-seam sync. The last shot is excluded - it's deliberately pinned to the track's real end (#423), not the frame grid. dissolve_videos's nested-shots handling is a separate, already-correct design decision (#401/#405) and is out of scope here. Co-Authored-By: Claude Sonnet 5 --- dw/tasks/pair_audio.py | 56 +++++++++++++++++++-- tests/test_pair_audio.py | 104 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 3 deletions(-) diff --git a/dw/tasks/pair_audio.py b/dw/tasks/pair_audio.py index 623ba3f1..3915b5fe 100644 --- a/dw/tasks/pair_audio.py +++ b/dw/tasks/pair_audio.py @@ -24,6 +24,40 @@ # the gap matters LENGTH_WARN_MS = 100.0 +# remeasured_shots regrids every interior shot onto the new track's frame +# grid, discarding whatever real position a join (concat_videos, +# dissolve_videos) had measured there. That is correct for a genuinely new +# soundtrack, but wrong for one that is a pointwise transform of the same +# audio the join already measured (#563: assemble-and-score's +# resample/fade/mix/normalize chain). A regridded shot's position is +# expected to differ from its real one by sub-sample rounding only - well +# under a millisecond; this sits far below that and also below +# 'sync_drift''s 40ms finding threshold (dw/assessment_rules.py), so a drift +# this small is reported here rather than waiting on that probe to run +REGRID_DRIFT_WARN_MS = 5.0 + + +def _regridded_drift(old_shots, native_rate, new_shots, rate): + """The interior shots whose real measured position pair_audio's regrid + overwrote by more than rounding noise, as (name, drift_ms) pairs. + + The last shot is excluded: remeasured_shots pins it to the track's + actual end by design (#423), not to the frame grid, so comparing it here + would flag that intentional difference on every call. + """ + if not old_shots or not new_shots or not native_rate or not rate: + return [] + drifted = [] + for old, new in zip(old_shots[:-1], new_shots[:-1]): + old_sample = old.get("start_sample") + new_sample = new.get("start_sample") + if old_sample is None or new_sample is None: + continue + drift_ms = old_sample / native_rate * 1000.0 - new_sample / rate * 1000.0 + if abs(drift_ms) >= REGRID_DRIFT_WARN_MS: + drifted.append((old.get("name", "shot"), drift_ms)) + return drifted + def _one_video(frames): """The frames of one video, unwrapping a batch of one. @@ -306,12 +340,28 @@ def pair_audio(video, audio, sample_rate=None, fps=None, fit=None): ) # The picture's shots survive; their samples are re-measured on the new # track, which was laid under whole rather than built shot by shot + old_shots = getattr(video, "shots", None) + new_shots = remeasured_shots(old_shots, frame_rate, rate, waveform.shape[1]) + drifted = _regridded_drift( + old_shots, getattr(video, "sample_rate", None), new_shots, rate + ) + if drifted: + named = ", ".join(f"{name} ({drift_ms:+.2f} ms)" for name, drift_ms in drifted) + emit_warning( + f"pair_audio: the video's own recorded shots placed {named} off " + f"the frame grid - a real seam position an earlier join measured. " + f"Regridding them onto this track discards that: the saved " + f"shots table will read as on-grid even though this track was " + f"not re-sliced at those seams. Trust the join's own shots for " + f"per-seam sync on this file, not this step's.", + kind="shot_position_regridded", + command="pair_audio", + shots=[name for name, _ in drifted], + ) return AudioVideo( frames, waveform, rate, fps=getattr(video, "fps", None), - shots=remeasured_shots( - getattr(video, "shots", None), frame_rate, rate, waveform.shape[1] - ), + shots=new_shots, ) diff --git a/tests/test_pair_audio.py b/tests/test_pair_audio.py index fde74d8a..83aa4cf5 100644 --- a/tests/test_pair_audio.py +++ b/tests/test_pair_audio.py @@ -23,6 +23,25 @@ def _waveform(samples=100, channels=2): return numpy.zeros((channels, samples), dtype=numpy.float32) +@pytest.fixture +def warnings_emitted(): + """The messages of the warning events a call emits - what a consumer over + the API or MCP actually reads, rather than the server's log (#82).""" + from dw.events import RunContext, activate_context, deactivate_context + + messages = [] + + def record(event): + if event["event"] == "warning": + messages.append(event["message"]) + + token = activate_context(RunContext(on_event=record)) + try: + yield messages + finally: + deactivate_context(token) + + def test_pairs_frames_with_a_soundtrack_read_from_a_file(tmp_path): import soundfile @@ -184,6 +203,91 @@ def test_registered_as_a_task_command(): assert "pair_audio" in _COMMAND_REGISTRY +def test_warns_when_a_real_measured_shot_position_is_regridded(warnings_emitted): + """#563: assemble-and-score's 'film' step pairs the picture from + concat_videos (real measured shots.start_sample) with an audio track that + is a pointwise transform of that same audio (resample/fade/mix/normalize) + rather than a track built shot by shot. remeasured_shots regrids every + interior shot onto the frame grid regardless, silently discarding a real + drift a join measured - shot 'b' here sits 31.67 ms off the grid, the + kind of number an inner concat_videos seam actually reported.""" + fps = 24 + native_rate = 48000 + shots = [ + {"name": "a", "start_frame": 0, "num_frames": 40, "start_sample": 0}, + { + "name": "b", + "start_frame": 40, + "num_frames": 40, + # On-grid would be round(40 / 24 * 48000) == 80000; this is a + # real measured position 1520 samples (31.67 ms) off it + "start_sample": 78480, + }, + {"name": "c", "start_frame": 80, "num_frames": 40, "start_sample": 160000}, + ] + video = AudioVideo( + [f"frame{i}" for i in range(120)], + _waveform(samples=5 * native_rate), + native_rate, + fps=fps, + shots=shots, + ) + target_rate = 44100 + track = _waveform(samples=5 * target_rate) + + paired = pair_audio(video, track, sample_rate=target_rate) + + assert paired.shots[1]["name"] == "b" + assert paired.shots[1]["start_sample"] == round(40 / fps * target_rate) + regridded = [w for w in warnings_emitted if "off the frame grid" in w] + assert len(regridded) == 1 + assert "b (-31.67 ms)" in regridded[0] + + +def test_a_shot_already_on_the_grid_is_not_warned_about(warnings_emitted): + """The ordinary case - a join's shots already sit where the frame grid + puts them - is not a false positive.""" + fps = 24 + rate = 44100 + shots = [ + { + "name": "a", + "start_frame": 0, + "num_frames": 40, + "start_sample": round(0 / fps * rate), + }, + { + "name": "b", + "start_frame": 40, + "num_frames": 40, + "start_sample": round(40 / fps * rate), + }, + { + "name": "c", + "start_frame": 80, + "num_frames": 40, + "start_sample": round(80 / fps * rate), + }, + ] + video = AudioVideo( + [f"frame{i}" for i in range(120)], + _waveform(samples=5 * rate), + rate, + fps=fps, + shots=shots, + ) + + pair_audio(video, _waveform(samples=5 * rate), sample_rate=rate) + + assert warnings_emitted == [] + + +def test_a_video_with_no_shots_is_not_warned_about(warnings_emitted): + pair_audio(["frame1", "frame2"], _waveform(), sample_rate=24000) + + assert warnings_emitted == [] + + def test_get_task_says_what_the_frame_rate_does(): """#104's fix is only useful where an author authoring a task step looks, and that is `get_task` - which reads the implementation's own docstring.""" From 50c8b0e72dc580316b233560bedc4cb63c7c7120 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 07:47:19 -0500 Subject: [PATCH 054/251] feat(prompts): add night tram scene description with tags and intended model --- prompts/ltx2/night-tram.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 prompts/ltx2/night-tram.json diff --git a/prompts/ltx2/night-tram.json b/prompts/ltx2/night-tram.json new file mode 100644 index 00000000..3ad03626 --- /dev/null +++ b/prompts/ltx2/night-tram.json @@ -0,0 +1,10 @@ +{ + "text": "A yellow vintage tram is gliding into frame from the left along a rain-wet city street at night, a wide shot from a low angle in side view as the camera remains static. Its polished body is catching the streetlamps in a long warm band of light, its windows are glowing from the cabin within, and the overhead wire is throwing short blue sparks at every joint. Initially the electric whine of the motor is steady and bright, the rails are clattering a soft rhythm under the wheels, and rain is falling on the asphalt in a constant hiss, the puddles mirroring the tram's headlights so the street doubles in amber. A moment later the tram is slowing toward a small shelter, its brake note bending into a small three-note melody, and the camera is pushing forward into a medium shot from the same low angle, the shelter's glass filling the right of frame. Simultaneously a small tabby cat is turning on the bench inside as the doors slide open on a soft two-tone chime, warm light is spilling across the wet platform, and rain is running in bright threads down the shelter's glass, the street settling to the sound of the water alone. The tram's lights are flaring off the dark asphalt in crisp detail as the cat is watching the tram pull on into the night.", + "description": "Night tram arrives at an empty shelter with a cat; brake note bends into a three-note melody, rain and reflections", + "intended_model": "ltx-2.5", + "tags": [ + "t2v", + "urban", + "night" + ] +} From 95f2625b874300b840d70fe0a337dbc958fc581c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 08:11:59 -0500 Subject: [PATCH 055/251] fix(api): streamline savePrompt function call in PromptEditorPage test: update expected number of LTX prompts to reflect recent additions --- tests/test_ltx_prompt_library.py | 2 +- ui/src/lib/api.ts | 31 ++++++++++++------------ ui/src/lib/pages/PromptEditorPage.svelte | 5 +--- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/tests/test_ltx_prompt_library.py b/tests/test_ltx_prompt_library.py index 3ec8a5d0..9625c289 100644 --- a/tests/test_ltx_prompt_library.py +++ b/tests/test_ltx_prompt_library.py @@ -54,7 +54,7 @@ def _prompt(path): def test_there_are_ltx_prompts(): - assert len(PROMPTS) == 10 + assert len(PROMPTS) == 11 @pytest.mark.parametrize("path", PROMPTS, ids=os.path.basename) diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index 34056720..e355a494 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -565,14 +565,16 @@ export const api = { workflowDownloadUrl: (name: string) => withToken(`/api/workflows/${encodePath(name)}/download`), saveWorkflow: (name: string, workflow: WorkflowDefinition) => - request<{ name: string; workspace: string; origin: string; warnings: string[] }>( - `/api/workflows/${encodePath(name)}`, - { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ workflow }), - }, - ), + request<{ + name: string + workspace: string + origin: string + warnings: string[] + }>(`/api/workflows/${encodePath(name)}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ workflow }), + }), listPrompts: () => request<{ /** The writable library - where a save lands. */ @@ -596,14 +598,11 @@ export const api = { }), ), savePrompt: (name: string, prompt: PromptDefinition) => - request<{ name: string }>( - `/api/prompts/${encodePath(name)}`, - { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ prompt }), - }, - ), + request<{ name: string }>(`/api/prompts/${encodePath(name)}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ prompt }), + }), deletePrompt: (name: string) => request<{ name: string; deleted: boolean }>( `/api/prompts/${encodePath(name)}`, diff --git a/ui/src/lib/pages/PromptEditorPage.svelte b/ui/src/lib/pages/PromptEditorPage.svelte index 46673f67..7325ee62 100644 --- a/ui/src/lib/pages/PromptEditorPage.svelte +++ b/ui/src/lib/pages/PromptEditorPage.svelte @@ -390,10 +390,7 @@ const path = savePath()! busy = true try { - await api.savePrompt( - path, - $state.snapshot(doc) as PromptDefinition, - ) + await api.savePrompt(path, $state.snapshot(doc) as PromptDefinition) if (folder === '__new__') { folder = newFolder.trim() newFolder = '' From 7cddd4159c9c5485ca3ef0e2e2edba7e3e37e296 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 28 Sep 2026 09:17:11 -0500 Subject: [PATCH 056/251] fix(tests): update test functions to use temporary directories for output paths --- CLAUDE.md | 4 +- tests/conftest.py | 9 +++ tests/test_events.py | 24 ++++---- tests/test_ltx_prompt_library.py | 3 +- tests/test_mcp_media.py | 10 +--- tests/test_modular_progress.py | 4 +- tests/test_phase_events.py | 20 +++---- tests/test_pipeline_caching.py | 70 +++++++++++------------ tests/test_template_subfolders.py | 27 ++------- tests/test_worker_execute.py | 4 +- tests/test_workflow_step_cache.py | 95 ++++++++++++++++--------------- 11 files changed, 128 insertions(+), 142 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5afa2816..992a1bd6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -432,8 +432,8 @@ same reason - default setup cannot load a pack. `-.` base rather than prefixing it, so two steps in one subfolder that set the same one collide onto `output_file_path`'s `-2` counter. - Every `workflows/templates/**` file with two or more saving steps - marks each one `final`/`intermediate` (`tests/test_template_subfolders.py` pins the rule; + Every `workflows/templates/**` file saves at least one step and + marks each saving step `final`/`intermediate`, at least one `final` (`tests/test_template_subfolders.py` pins the rule; `dw/workflows/` builtins stay unmarked - a role is the parent's to assign). A template's outputs land in `/final/` and `/intermediate/`, so gallery names read `