diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index 51a2fb63b8..342c116a82 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -72,6 +72,9 @@ "packages/engine/tests/**", "skills/**/test-corpus/**", "skills/**/scripts/**", + // The relocated media-use .mjs files are agent-invoked scripts outside the + // import graph; keep the exemption scoped to that shipped script type. + "packages/cli/src/media-use/**/*.mjs", // Agent-invoked motion-graphics tools co-located with their docs (run via // `node ` per grounding/PROTOCOL.md / categories/maps/module.md // prose), not import-graph reachable. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7489c3f14..7449e587a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: catalog_index: - "registry/registry.json" - "registry/catalog-artifact/**" + - "skills/media-use/audio/assets/sfx/**" - "scripts/catalog/check-artifact-coverage.ts" docs_catalog: - "docs/**" @@ -375,8 +376,8 @@ jobs: # `node:` built-in imports. They aren't part of any workspace package, and # the main `Test` job's `code` path filter excludes `skills/**`, so without # this dedicated job they'd never run in CI. Examples: - # * skills/media-use/scripts/resolve.test.mjs - # * skills/media-use/scripts/lib/manifest.test.mjs + # * packages/cli/src/media-use/resolve.test.mjs + # * packages/cli/src/media-use/lib/manifest.test.mjs # Several of these are regression guards (e.g. shell-injection cases), so # the whole point is that they fire on PRs that touch skills/. test-skills: @@ -396,11 +397,12 @@ jobs: # would defeat the whole point of this job). run: | set -euo pipefail - mapfile -t SKILLS_TESTS < <(find skills -type f -name "*.test.mjs" | sort) + mapfile -t SKILLS_TESTS < <(find skills packages/cli/src/media-use -type f -name "*.test.mjs" | sort) if [ "${#SKILLS_TESTS[@]}" -eq 0 ]; then echo "::error::No skills/**/*.test.mjs files found. Did the layout change?" exit 1 fi + SKILLS_TESTS+=(scripts/check-media-use-copy-parity.test.mjs) printf 'Running %d skills test file(s):\n' "${#SKILLS_TESTS[@]}" printf ' * %s\n' "${SKILLS_TESTS[@]}" node --test "${SKILLS_TESTS[@]}" diff --git a/package.json b/package.json index 22606600b1..d812a90a72 100644 --- a/package.json +++ b/package.json @@ -51,9 +51,9 @@ "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", - "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts && vitest run scripts/catalog/", + "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts && vitest run scripts/catalog/", "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json", - "test:skills": "node --test 'skills/**/*.test.mjs'", + "test:skills": "node --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'", "generate:previews": "tsx scripts/generate-template-previews.ts", "generate:catalog-previews": "tsx scripts/generate-catalog-previews.ts", "package:codex-plugin": "node scripts/package-codex-plugin.mjs", diff --git a/packages/cli/scripts/build-copy.mjs b/packages/cli/scripts/build-copy.mjs index 847a6211aa..84c78fd6e1 100644 --- a/packages/cli/scripts/build-copy.mjs +++ b/packages/cli/scripts/build-copy.mjs @@ -1,7 +1,7 @@ // Cross-platform replacement for the previous `mkdir -p … && cp -r …` shell // chain, which failed on Windows because `cp` doesn't accept `-r` there. -import { cpSync, existsSync, mkdirSync, readdirSync } from "node:fs"; +import { cpSync, existsSync, mkdirSync, readdirSync, rmSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { setTimeout as sleep } from "node:timers/promises"; @@ -80,15 +80,33 @@ async function main() { // Skills bundled into the published CLI. Branches don't all carry the same // skills/ tree (it gets restructured), so each entry is existsSync-guarded: // a missing skill dir warns + skips instead of crashing the build. - for (const skill of ["hyperframes", "hyperframes-cli", "gsap"]) { + for (const skill of ["hyperframes", "hyperframes-cli", "gsap", "media-use"]) { const src = join(REPO_ROOT, "skills", skill); if (!existsSync(src)) { console.warn(`[build-copy] skill not found, skipping: skills/${skill}`); continue; } - copyDir(src, join(DIST, "skills", skill)); + const destination = join(DIST, "skills", skill); + rmSync(destination, { recursive: true, force: true }); + copyDir(src, destination); } + // The media-use engine lives with the CLI source, but keeps its published + // skill-relative layout so the moved .mjs tree can run without a rewrite. + const mediaEngine = join(CLI_ROOT, "src", "media-use"); + const publishedMediaLib = join(DIST, "skills", "media-use", "scripts", "lib"); + rmSync(publishedMediaLib, { recursive: true, force: true }); + mkdirSync(publishedMediaLib, { recursive: true }); + copyDirContents(join(mediaEngine, "lib"), publishedMediaLib); + cpSync( + join(mediaEngine, "resolve.mjs"), + join(DIST, "skills", "media-use", "scripts", "resolve.mjs"), + ); + mkdirSync(join(DIST, "skills", "registry"), { recursive: true }); + copyDirContents(join(DIST, "registry"), join(DIST, "skills", "registry")); + mkdirSync(join(DIST, "skills", "media-use", "registry"), { recursive: true }); + copyDirContents(join(DIST, "registry"), join(DIST, "skills", "media-use", "registry")); + const dockerfile = join(CLI_ROOT, "src", "docker", "Dockerfile.render"); if (existsSync(dockerfile)) { cpSync(dockerfile, join(DIST, "docker", "Dockerfile.render")); diff --git a/packages/cli/src/audio/scripts b/packages/cli/src/audio/scripts new file mode 120000 index 0000000000..cd39d7d4a2 --- /dev/null +++ b/packages/cli/src/audio/scripts @@ -0,0 +1 @@ +../../../../skills/media-use/audio/scripts \ No newline at end of file diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 357782ff7f..2e7cd0e2f3 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -143,6 +143,7 @@ const commandLoaders = { init: () => import("./commands/init.js").then((m) => m.default), add: () => import("./commands/add.js").then((m) => m.default), catalog: () => import("./commands/catalog.js").then((m) => m.default), + "media-use": () => import("./commands/media-use.js").then((m) => m.default), play: () => import("./commands/play.js").then((m) => m.default), present: () => import("./commands/present.js").then((m) => m.default), preview: () => diff --git a/packages/cli/src/commands/catalog.test.ts b/packages/cli/src/commands/catalog.test.ts index 1b11016554..ea5faba8b1 100644 --- a/packages/cli/src/commands/catalog.test.ts +++ b/packages/cli/src/commands/catalog.test.ts @@ -143,6 +143,7 @@ vi.mock("../registry/localEmbedder.js", () => ({ })); vi.mock("../registry/localSemantic.js", () => ({ + mediaSemanticRanking: async () => null, localSemanticRanking: async () => { if (state.rankingError) throw state.rankingError; return state.ranking; diff --git a/packages/cli/src/commands/media-use.test.ts b/packages/cli/src/commands/media-use.test.ts new file mode 100644 index 0000000000..ade7b0ae19 --- /dev/null +++ b/packages/cli/src/commands/media-use.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + createMediaUseCommand, + MEDIA_USE_VERBS, + mediaUsePassthroughArgs, + mediaUseVerbFlags, + resolveMediaUseEnginePath, +} from "./media-use.js"; + +function tempCommandDir(): string { + return mkdtempSync(join(tmpdir(), "hyperframes-media-use-command-")); +} + +describe("media-use command wiring", () => { + it("uses the bundled engine when the first candidate exists", () => { + const here = tempCommandDir(); + const engine = join(here, "..", "media-use", "resolve.mjs"); + try { + expect(resolveMediaUseEnginePath(here, (candidate) => candidate === engine)).toBe(engine); + } finally { + rmSync(here, { recursive: true, force: true }); + } + }); + + it("falls back to the source-tree skill engine", () => { + const here = tempCommandDir(); + const engine = join(here, "skills", "media-use", "scripts", "resolve.mjs"); + try { + expect(resolveMediaUseEnginePath(here, (candidate) => candidate === engine)).toBe(engine); + } finally { + rmSync(here, { recursive: true, force: true }); + } + }); + + it("explains how to recover when the engine is absent", () => { + const here = tempCommandDir(); + try { + expect(() => resolveMediaUseEnginePath(here)).toThrow( + "media-use engine is missing from this CLI build; reinstall the CLI or run from a source checkout", + ); + } finally { + rmSync(here, { recursive: true, force: true }); + } + }); + + it("passes flags after the media-use verb through unchanged", () => { + expect( + mediaUsePassthroughArgs([ + "/usr/bin/node", + "cli.js", + "media-use", + "resolve", + "--type", + "sfx", + "--intent", + "cat", + ]), + ).toEqual(["--type", "sfx", "--intent", "cat"]); + }); + + it("maps every verb to the engine flag, leaving resolve unflagged", () => { + for (const verb of MEDIA_USE_VERBS) { + expect(mediaUseVerbFlags(verb)).toEqual(verb === "resolve" ? [] : [`--${verb}`]); + } + }); + + it("wires every subcommand to invoke its matching verb", () => { + const invoked: string[] = []; + const command = createMediaUseCommand((verb) => { + invoked.push(verb); + throw new Error("stop after dispatch"); + }); + + for (const verb of MEDIA_USE_VERBS) { + const factory = (command.subCommands as Record | undefined)?.[verb]; + expect(factory).toBeTypeOf("function"); + const child = (factory as () => { run?: () => never })(); + expect(() => child.run?.()).toThrow("stop after dispatch"); + } + + expect(invoked).toEqual(MEDIA_USE_VERBS); + }); +}); diff --git a/packages/cli/src/commands/media-use.ts b/packages/cli/src/commands/media-use.ts new file mode 100644 index 0000000000..0c222ee341 --- /dev/null +++ b/packages/cli/src/commands/media-use.ts @@ -0,0 +1,107 @@ +import { defineCommand } from "citty"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { finishCommand } from "../utils/commandResult.js"; + +const MEDIA_USE_ARGS = { + type: { type: "string" }, + intent: { type: "string" }, + entity: { type: "string" }, + project: { type: "string", alias: "p" }, + adopt: { type: "boolean" }, + candidates: { type: "boolean" }, + doctor: { type: "boolean" }, + stats: { type: "boolean" }, + days: { type: "string" }, + "dry-run": { type: "boolean" }, + reuse: { type: "string" }, + from: { type: "string" }, + params: { type: "string" }, + for: { type: "string" }, + analyze: { type: "boolean" }, + "local-only": { type: "boolean" }, + provider: { type: "string" }, + "avatar-id": { type: "string" }, + "voice-id": { type: "string" }, + json: { type: "boolean" }, + help: { type: "boolean", alias: "h" }, +} as const; + +export const MEDIA_USE_VERBS = [ + "resolve", + "doctor", + "stats", + "adopt", + "candidates", + "reuse", + "from", + "params", + "analyze", +] as const; +export type MediaUseVerb = (typeof MEDIA_USE_VERBS)[number]; + +export function resolveMediaUseEnginePath( + here: string, + fileExists: (path: string) => boolean = existsSync, +): string { + const candidates = [ + join(here, "..", "media-use", "resolve.mjs"), + join(here, "skills", "media-use", "scripts", "resolve.mjs"), + ]; + const engine = candidates.find((candidate) => fileExists(candidate)); + if (!engine) { + throw new Error( + "media-use engine is missing from this CLI build; reinstall the CLI or run from a source checkout", + ); + } + return engine; +} + +export function mediaUsePassthroughArgs(argv: readonly string[]): string[] { + const commandIndex = argv.indexOf("media-use"); + return argv.slice(commandIndex + 2); +} + +export function mediaUseVerbFlags(verb: MediaUseVerb): string[] { + return verb === "resolve" ? [] : [`--${verb}`]; +} + +type InvokeMediaUse = (verb: MediaUseVerb) => never; + +function invokeEngine(verb: MediaUseVerb): never { + const here = dirname(fileURLToPath(import.meta.url)); + const passed = mediaUsePassthroughArgs(process.argv); + const flag = mediaUseVerbFlags(verb); + const result = spawnSync( + process.execPath, + [resolveMediaUseEnginePath(here), ...flag, ...passed], + { + stdio: "inherit", + }, + ); + if (result.error) throw result.error; + finishCommand(result.status ?? 1); +} + +function subcommand(name: MediaUseVerb, invoke: InvokeMediaUse) { + return defineCommand({ + meta: { name, description: `media-use ${name}` }, + args: MEDIA_USE_ARGS, + run: () => invoke(name), + }); +} + +export function createMediaUseCommand(invoke: InvokeMediaUse = invokeEngine) { + const subCommands = Object.fromEntries( + MEDIA_USE_VERBS.map((name) => [name, () => subcommand(name, invoke)]), + ); + return defineCommand({ + meta: { name: "media-use", description: "Resolve and operate on project media" }, + subCommands, + run: () => console.log("Run `hyperframes media-use --help`"), + }); +} + +export default createMediaUseCommand(); diff --git a/skills/media-use/scripts/lib/adopt.mjs b/packages/cli/src/media-use/lib/adopt.mjs similarity index 100% rename from skills/media-use/scripts/lib/adopt.mjs rename to packages/cli/src/media-use/lib/adopt.mjs diff --git a/skills/media-use/scripts/lib/adopt.test.mjs b/packages/cli/src/media-use/lib/adopt.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/adopt.test.mjs rename to packages/cli/src/media-use/lib/adopt.test.mjs diff --git a/skills/media-use/scripts/lib/bgm-provider.mjs b/packages/cli/src/media-use/lib/bgm-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/bgm-provider.mjs rename to packages/cli/src/media-use/lib/bgm-provider.mjs diff --git a/skills/media-use/scripts/lib/brand-provider.mjs b/packages/cli/src/media-use/lib/brand-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/brand-provider.mjs rename to packages/cli/src/media-use/lib/brand-provider.mjs diff --git a/skills/media-use/scripts/lib/bundled-sfx-provider.mjs b/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs similarity index 77% rename from skills/media-use/scripts/lib/bundled-sfx-provider.mjs rename to packages/cli/src/media-use/lib/bundled-sfx-provider.mjs index 7011b45fcd..6522da28a9 100644 --- a/skills/media-use/scripts/lib/bundled-sfx-provider.mjs +++ b/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs @@ -1,8 +1,25 @@ import { existsSync, readFileSync } from "node:fs"; import { extname, join } from "node:path"; +import { rankMediaRows } from "./media-search.mjs"; const LIB_DIR = process.env.HYPERFRAMES_MEDIA_USE_SFX_DIR || + [ + join(import.meta.dirname, "..", "..", "audio", "assets", "sfx"), + join( + import.meta.dirname, + "..", + "..", + "..", + "..", + "..", + "skills", + "media-use", + "audio", + "assets", + "sfx", + ), + ].find((candidate) => existsSync(candidate)) || join(import.meta.dirname, "..", "..", "audio", "assets", "sfx"); export const BUNDLED_SFX_RECOVERY_COMMAND = "npx hyperframes skills update media-use"; @@ -59,25 +76,10 @@ export function inspectBundledSfxAssets(libraryDir = LIB_DIR) { }; } -const normalize = (value) => - String(value) - .toLowerCase() - .replace(/[^a-z0-9]+/g, " ") - .trim(); - export function extensionForBundledSfxFile(filename) { return extname(filename) || ".mp3"; } -function score(intent, key, entry) { - const query = normalize(intent); - const name = normalize(key); - if (query === name) return 100; - if (query.includes(name) || name.includes(query)) return 50; - const haystack = new Set(normalize(`${key} ${entry.description || ""}`).split(/\s+/)); - return query.split(/\s+/).filter((token) => token && haystack.has(token)).length; -} - export const bundledSfxProvider = { async search(intent, ctx = {}) { const libraryDir = ctx.libraryDir || LIB_DIR; @@ -85,11 +87,20 @@ export const bundledSfxProvider = { if (!health.ok) throw new BundledSfxAssetsError(health); const manifest = JSON.parse(readFileSync(join(libraryDir, "manifest.json"), "utf8")); - const ranked = Object.entries(manifest) - .map(([key, entry]) => ({ key, entry, score: score(intent, key, entry) })) - .filter(({ entry, score }) => entry?.file && score > 0) - .sort((a, b) => b.score - a.score || a.key.localeCompare(b.key)); - const best = ranked[0]; + const rows = Object.entries(manifest).map(([key, entry]) => ({ + id: key, + kind: "sfx", + title: key, + description: entry?.description || key, + tags: ["sfx"], + file: entry?.file || "", + duration: entry?.duration, + })); + const bestRow = rankMediaRows(intent, rows)[0]; + const best = + bestRow && manifest[bestRow.id]?.file + ? { key: bestRow.id, entry: manifest[bestRow.id] } + : null; if (!best) return null; const localPath = join(libraryDir, best.entry.file); diff --git a/skills/media-use/scripts/lib/bundled-sfx-provider.test.mjs b/packages/cli/src/media-use/lib/bundled-sfx-provider.test.mjs similarity index 64% rename from skills/media-use/scripts/lib/bundled-sfx-provider.test.mjs rename to packages/cli/src/media-use/lib/bundled-sfx-provider.test.mjs index 869fe896f3..7609e57549 100644 --- a/skills/media-use/scripts/lib/bundled-sfx-provider.test.mjs +++ b/packages/cli/src/media-use/lib/bundled-sfx-provider.test.mjs @@ -10,6 +10,7 @@ import { extensionForBundledSfxFile, inspectBundledSfxAssets, } from "./bundled-sfx-provider.mjs"; +import { rankMediaRows } from "./media-search.mjs"; test("derives bundled SFX extension from the manifest filename", () => { assert.equal(extensionForBundledSfxFile("impact.wav"), ".wav"); @@ -83,3 +84,56 @@ test("accepts a complete bundled SFX library", () => { rmSync(libraryDir, { recursive: true, force: true }); } }); + +test("prefers an exact key over a longer key with the same words", async () => { + const libraryDir = mkdtempSync(join(tmpdir(), "media-use-sfx-exact-key-")); + try { + writeFileSync( + join(libraryDir, "manifest.json"), + JSON.stringify({ + "whoosh-cinematic": { file: "whoosh-cinematic.mp3", description: "long whoosh" }, + whoosh: { file: "whoosh.mp3", description: "short whoosh" }, + }), + ); + writeFileSync(join(libraryDir, "whoosh-cinematic.mp3"), "cinematic audio"); + writeFileSync(join(libraryDir, "whoosh.mp3"), "exact audio"); + + const result = await bundledSfxProvider.search("whoosh", { libraryDir }); + assert.equal(result?.localPath, join(libraryDir, "whoosh.mp3")); + assert.equal(result?.metadata.provenance.library_key, "whoosh"); + + const stemmed = await bundledSfxProvider.search("whooshes", { libraryDir }); + assert.equal(stemmed?.localPath, join(libraryDir, "whoosh.mp3")); + assert.equal(stemmed?.metadata.provenance.library_key, "whoosh"); + } finally { + rmSync(libraryDir, { recursive: true, force: true }); + } +}); + +test("literal ids outrank distinct ids with the same stem", () => { + const rows = [ + { + id: "cats", + title: "Cats Fighting", + description: "fighting cats", + tags: ["yowl", "fight"], + kind: "sfx", + }, + { + id: "cat", + title: "Cat", + description: "single cat", + tags: ["meow"], + kind: "sfx", + }, + ]; + + assert.deepEqual( + rankMediaRows("cat", rows).map((row) => row.id), + ["cat", "cats"], + ); + assert.deepEqual( + rankMediaRows("cats", rows).map((row) => row.id), + ["cats", "cat"], + ); +}); diff --git a/skills/media-use/scripts/lib/cache.mjs b/packages/cli/src/media-use/lib/cache.mjs similarity index 100% rename from skills/media-use/scripts/lib/cache.mjs rename to packages/cli/src/media-use/lib/cache.mjs diff --git a/skills/media-use/scripts/lib/candidates.mjs b/packages/cli/src/media-use/lib/candidates.mjs similarity index 100% rename from skills/media-use/scripts/lib/candidates.mjs rename to packages/cli/src/media-use/lib/candidates.mjs diff --git a/skills/media-use/scripts/lib/candidates.test.mjs b/packages/cli/src/media-use/lib/candidates.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/candidates.test.mjs rename to packages/cli/src/media-use/lib/candidates.test.mjs diff --git a/skills/media-use/scripts/lib/codex-provider.mjs b/packages/cli/src/media-use/lib/codex-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/codex-provider.mjs rename to packages/cli/src/media-use/lib/codex-provider.mjs diff --git a/skills/media-use/scripts/lib/codex-provider.test.mjs b/packages/cli/src/media-use/lib/codex-provider.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/codex-provider.test.mjs rename to packages/cli/src/media-use/lib/codex-provider.test.mjs diff --git a/skills/media-use/scripts/lib/coverage.test.mjs b/packages/cli/src/media-use/lib/coverage.test.mjs similarity index 93% rename from skills/media-use/scripts/lib/coverage.test.mjs rename to packages/cli/src/media-use/lib/coverage.test.mjs index ed4ffbde93..79d58568bb 100644 --- a/skills/media-use/scripts/lib/coverage.test.mjs +++ b/packages/cli/src/media-use/lib/coverage.test.mjs @@ -10,7 +10,16 @@ import { CAPABILITIES, listModels } from "./local-models.mjs"; // test enforces the weakness→owner matrix in references/meta.md so a claim can't rot — if // a capability's entrypoint disappears, this fails. -const SKILL = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const SKILL = join( + dirname(fileURLToPath(import.meta.url)), + "..", + "..", + "..", + "..", + "..", + "skills", + "media-use", +); test("weakness: audio-only → media-use resolves image + icon", () => { for (const t of ["image", "icon"]) { @@ -43,7 +52,10 @@ test("weakness: no media-ops → ops guidance reference exists", () => { test("weakness: no transcript-driven cutting → cut compiler entrypoints exist", async () => { assert.ok(existsSync(join(SKILL, "scripts", "transcript-cut.mjs")), "transcript-cut missing"); - assert.ok(existsSync(join(SKILL, "scripts", "lib", "cutlist.mjs")), "cutlist lib missing"); + assert.ok( + existsSync(join(dirname(fileURLToPath(import.meta.url)), "cutlist.mjs")), + "cutlist lib missing", + ); const cutlist = await import("./cutlist.mjs"); assert.equal(typeof cutlist.compileCutList, "function"); }); @@ -60,7 +72,10 @@ test("weakness: whisper.cpp is weak → better local ASR (Parakeet) entrypoint e test("weakness: no auto-duck/loudness → duck compiler and recipes exist", async () => { assert.ok(existsSync(join(SKILL, "scripts", "audio-duck.mjs")), "audio-duck missing"); - assert.ok(existsSync(join(SKILL, "scripts", "lib", "duck.mjs")), "duck lib missing"); + assert.ok( + existsSync(join(dirname(fileURLToPath(import.meta.url)), "duck.mjs")), + "duck lib missing", + ); assert.ok(existsSync(join(SKILL, "references", "operations.md")), "operations.md missing"); const duck = await import("./duck.mjs"); assert.equal(typeof duck.speechSpans, "function"); diff --git a/skills/media-use/scripts/lib/cube-build.mjs b/packages/cli/src/media-use/lib/cube-build.mjs similarity index 100% rename from skills/media-use/scripts/lib/cube-build.mjs rename to packages/cli/src/media-use/lib/cube-build.mjs diff --git a/skills/media-use/scripts/lib/cube-build.test.mjs b/packages/cli/src/media-use/lib/cube-build.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/cube-build.test.mjs rename to packages/cli/src/media-use/lib/cube-build.test.mjs diff --git a/skills/media-use/scripts/lib/cube-validate.mjs b/packages/cli/src/media-use/lib/cube-validate.mjs similarity index 100% rename from skills/media-use/scripts/lib/cube-validate.mjs rename to packages/cli/src/media-use/lib/cube-validate.mjs diff --git a/skills/media-use/scripts/lib/cube-validate.test.mjs b/packages/cli/src/media-use/lib/cube-validate.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/cube-validate.test.mjs rename to packages/cli/src/media-use/lib/cube-validate.test.mjs diff --git a/packages/cli/src/media-use/lib/cutlist.mjs b/packages/cli/src/media-use/lib/cutlist.mjs new file mode 100644 index 0000000000..07d0d34159 --- /dev/null +++ b/packages/cli/src/media-use/lib/cutlist.mjs @@ -0,0 +1,184 @@ +import { normalizeWords } from "./words.mjs"; + +const MIN_SEGMENT_SECONDS = 0.2; +const SILENCE_PAD_SECONDS = 0.15; + +export function compileCutList(transcript, opts = {}) { + const words = normalizeWords(transcript); + if (opts.keep != null && hasRemovalSource(opts)) { + throw new Error("--keep is mutually exclusive with removal options"); + } + + if (opts.keep != null) { + const duration = durationFrom(words, opts); + const ranges = parseTimeRanges(opts.keep); + return finalizeKept(duration != null ? clampRanges(ranges, duration) : ranges); + } + + const duration = durationFrom(words, opts); + if (!duration) return []; + + const removals = [ + ...parseTimeRanges(opts.remove), + ...wordIndexRanges(words, opts.removeWords), + ...fillerRanges(words, opts.removeFillers), + ...silenceRanges(words, opts.cutSilence), + ]; + const mergedRemovals = mergeRanges(clampRanges(removals, duration)); + return finalizeKept(invertRanges(mergedRemovals, duration)); +} + +function hasRemovalSource(opts) { + return ( + opts.remove != null || + opts.removeWords != null || + opts.removeFillers != null || + opts.cutSilence != null + ); +} + +function durationFrom(words, opts) { + const explicit = Number(opts.duration ?? opts.totalDuration); + if (Number.isFinite(explicit) && explicit > 0) return explicit; + const last = words.at(-1); + return last && Number.isFinite(last.end) && last.end > 0 ? last.end : null; +} + +function parseTimeRanges(value) { + if (value == null || value === false || value === "") return []; + if (typeof value === "string") { + return value + .split(",") + .map((part) => part.trim()) + .filter(Boolean) + .map(parseRangeString); + } + if (!Array.isArray(value)) throw new Error("range list must be a string or array"); + return value.map((range) => { + if (Array.isArray(range)) return cleanRange(Number(range[0]), Number(range[1])); + return cleanRange(Number(range?.start), Number(range?.end)); + }); +} + +function parseRangeString(value) { + const match = value.match(/^([0-9]*\.?[0-9]+)\s*-\s*([0-9]*\.?[0-9]+)$/); + if (!match) throw new Error(`invalid range: ${value}`); + return cleanRange(Number(match[1]), Number(match[2])); +} + +function cleanRange(start, end) { + if (!Number.isFinite(start) || !Number.isFinite(end)) { + throw new Error("range start/end must be finite numbers"); + } + if (end < start) throw new Error(`range end ${end} is before start ${start}`); + return { start, end }; +} + +function wordIndexRanges(words, value) { + if (value == null || value === false || value === "") return []; + const ranges = typeof value === "string" ? value.split(",") : value; + if (!Array.isArray(ranges)) throw new Error("--remove-words must be a string or array"); + return ranges + .map((range) => (typeof range === "string" ? range.trim() : range)) + .filter(Boolean) + .map((range) => { + const [first, last = first] = + typeof range === "string" ? range.split("-").map((n) => n.trim()) : range; + const startIndex = Number(first); + const endIndex = Number(last); + if (!Number.isInteger(startIndex) || !Number.isInteger(endIndex)) { + throw new Error(`invalid word range: ${range}`); + } + if (startIndex < 0 || endIndex < startIndex || endIndex >= words.length) { + throw new Error(`word range out of bounds: ${range}`); + } + return { start: words[startIndex].start, end: words[endIndex].end }; + }); +} + +function fillerRanges(words, value) { + if (value == null || value === false || value === "") return []; + const fillers = Array.isArray(value) + ? value + : String(value) + .split(",") + .map((s) => s.trim()); + const set = new Set(fillers.filter(Boolean).map(bareToken)); + if (set.size === 0) return []; + // Whisper emits words with attached punctuation and arbitrary case + // ("UM," / "Um."), so compare bare tokens. + return words + .filter((word) => set.has(bareToken(word.text))) + .map((word) => ({ start: word.start, end: word.end })); +} + +function bareToken(text) { + return String(text) + .toLowerCase() + .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""); +} + +function silenceRanges(words, value) { + if (value == null || value === false || value === "") return []; + const threshold = Number(value); + if (!Number.isFinite(threshold) || threshold <= 0) { + throw new Error("--cut-silence must be a positive number"); + } + const ranges = []; + for (let i = 0; i < words.length - 1; i++) { + const current = words[i]; + const next = words[i + 1]; + const gap = next.start - current.end; + if (gap <= threshold) continue; + const start = current.end + SILENCE_PAD_SECONDS; + const end = next.start - SILENCE_PAD_SECONDS; + if (end > start) ranges.push({ start, end }); + } + return ranges; +} + +function clampRanges(ranges, duration) { + return ranges + .map((range) => ({ + start: Math.max(0, Math.min(duration, range.start)), + end: Math.max(0, Math.min(duration, range.end)), + })) + .filter((range) => range.end > range.start); +} + +function mergeRanges(ranges) { + const sorted = ranges + .map((range) => ({ start: round3(range.start), end: round3(range.end) })) + .sort((a, b) => a.start - b.start || a.end - b.end); + const merged = []; + for (const range of sorted) { + const prev = merged.at(-1); + if (prev && range.start <= prev.end) { + prev.end = Math.max(prev.end, range.end); + } else { + merged.push({ ...range }); + } + } + return merged; +} + +function invertRanges(removals, duration) { + const kept = []; + let cursor = 0; + for (const range of removals) { + if (range.start > cursor) kept.push({ start: cursor, end: range.start }); + cursor = Math.max(cursor, range.end); + } + if (cursor < duration) kept.push({ start: cursor, end: duration }); + return kept; +} + +function finalizeKept(ranges) { + return mergeRanges(ranges) + .map((range) => ({ start: round3(range.start), end: round3(range.end) })) + .filter((range) => round3(range.end - range.start) >= MIN_SEGMENT_SECONDS); +} + +function round3(n) { + return Math.round(Number(n) * 1000) / 1000; +} diff --git a/skills/media-use/scripts/lib/cutlist.test.mjs b/packages/cli/src/media-use/lib/cutlist.test.mjs similarity index 91% rename from skills/media-use/scripts/lib/cutlist.test.mjs rename to packages/cli/src/media-use/lib/cutlist.test.mjs index 72e6f2b0df..98e7710a27 100644 --- a/skills/media-use/scripts/lib/cutlist.test.mjs +++ b/packages/cli/src/media-use/lib/cutlist.test.mjs @@ -8,7 +8,21 @@ import { test } from "node:test"; import { compileCutList } from "./cutlist.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); -const SCRIPT = join(HERE, "..", "transcript-cut.mjs"); +// These legacy wrappers remain under skills/media-use/scripts because the +// skill's public audio entrypoints still invoke them; the moved engine tests +// their implementation here while preserving that compatibility surface. +const SCRIPT = join( + HERE, + "..", + "..", + "..", + "..", + "..", + "skills", + "media-use", + "scripts", + "transcript-cut.mjs", +); test("explicit --remove ranges invert to kept segments", () => { const transcript = [ diff --git a/packages/cli/src/media-use/lib/duck.mjs b/packages/cli/src/media-use/lib/duck.mjs new file mode 100644 index 0000000000..9bbe7afda0 --- /dev/null +++ b/packages/cli/src/media-use/lib/duck.mjs @@ -0,0 +1,95 @@ +import { wordListsFromMediaMeta } from "./words.mjs"; + +// audio_meta.json word times are relative to each line's own file. +export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false, gap = 0 } = {}) { + const merge = Number(mergeGap); + const lists = wordListsFromMediaMeta(meta); + const voices = Array.isArray(meta?.voices) ? meta.voices : []; + if (lists.length > 1 && !offsets && !sequential) { + throw new Error( + "audio_meta has multiple voice lines with file-relative times; pass --sequential or --offsets so spans land at composition time", + ); + } + const intervals = []; + let cursor = 0; + for (let i = 0; i < lists.length; i++) { + const voice = voices[i]; + let offset = 0; + if (offsets) { + const id = voice?.id ?? String(i); + if (!(id in offsets)) throw new Error(`--offsets is missing voice "${id}"`); + offset = Number(offsets[id]) || 0; + } else if (sequential) { + offset = cursor; + const lineDuration = Number(voice?.duration_s) || Math.max(...lists[i].map((w) => w.end), 0); + cursor += lineDuration + (Number(gap) || 0); + } + for (const word of lists[i]) { + if (word.end > word.start) + intervals.push({ start: word.start + offset, end: word.end + offset }); + } + } + return mergeIntervals(intervals, Number.isFinite(merge) && merge >= 0 ? merge : 0.6); +} + +export function duckKeyframes( + spans, + { duck = 0.25, attack = 0.15, release = 0.4, baseVolume = 1 } = {}, +) { + const base = finiteOr(baseVolume, 1); + const ducked = round3(base * finiteOr(duck, 0.25)); + const keyframes = []; + for (const span of spans) { + keyframes.push({ + time: round3(Math.max(0, finiteOr(span.start, 0))), + volume: ducked, + duration: round3(finiteOr(attack, 0.15)), + }); + keyframes.push({ + time: round3(Math.max(0, finiteOr(span.end, 0))), + volume: round3(base), + duration: round3(finiteOr(release, 0.4)), + }); + } + return keyframes.sort((a, b) => a.time - b.time); +} + +/** Volume lane for `data-automation`: composition-time keyframes as clip-local ramps. */ +export function duckLane(keyframes, { clipStart = 0, baseVolume = 1 } = {}) { + const start = finiteOr(clipStart, 0); + const points = [{ t: 0, v: round3(finiteOr(baseVolume, 1)) }]; + const push = (t, v) => { + if (t > points.at(-1).t) points.push({ t: round3(t), v }); + }; + for (const kf of keyframes) { + const t = Math.max(0, kf.time - start); + push(t, points.at(-1).v); + push(t + kf.duration, kf.volume); + } + return { version: 1, lanes: [{ target: "volume", points }] }; +} + +function mergeIntervals(intervals, mergeGap) { + const sorted = intervals + .map((range) => ({ start: round3(range.start), end: round3(range.end) })) + .sort((a, b) => a.start - b.start || a.end - b.end); + const merged = []; + for (const range of sorted) { + const prev = merged.at(-1); + if (prev && (range.start <= prev.end || range.start - prev.end < mergeGap)) { + prev.end = Math.max(prev.end, range.end); + } else { + merged.push({ ...range }); + } + } + return merged; +} + +function finiteOr(value, fallback) { + const n = Number(value); + return Number.isFinite(n) ? n : fallback; +} + +function round3(n) { + return Math.round(Number(n) * 1000) / 1000; +} diff --git a/skills/media-use/scripts/lib/duck.test.mjs b/packages/cli/src/media-use/lib/duck.test.mjs similarity index 94% rename from skills/media-use/scripts/lib/duck.test.mjs rename to packages/cli/src/media-use/lib/duck.test.mjs index 398601f0b9..471eafff6d 100644 --- a/skills/media-use/scripts/lib/duck.test.mjs +++ b/packages/cli/src/media-use/lib/duck.test.mjs @@ -8,7 +8,20 @@ import { test } from "node:test"; import { duckKeyframes, duckLane, speechSpans } from "./duck.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); -const SCRIPT = join(HERE, "..", "audio-duck.mjs"); +// Keep the skill-owned wrapper path under test until the compatibility layer +// is removed in a later release. +const SCRIPT = join( + HERE, + "..", + "..", + "..", + "..", + "..", + "skills", + "media-use", + "scripts", + "audio-duck.mjs", +); test("speechSpans bridges gaps smaller than mergeGap", () => { const meta = { diff --git a/packages/cli/src/media-use/lib/error-diffusion.mjs b/packages/cli/src/media-use/lib/error-diffusion.mjs new file mode 100644 index 0000000000..e59a9c5ab6 --- /dev/null +++ b/packages/cli/src/media-use/lib/error-diffusion.mjs @@ -0,0 +1,230 @@ +export const ERROR_DIFFUSION_ALGORITHMS = { + "floyd-steinberg": { + kernel: [ + [1, 0, 7], + [-1, 1, 3], + [0, 1, 5], + [1, 1, 1], + ], + divisor: 16, + }, + atkinson: { + kernel: [ + [1, 0, 1], + [2, 0, 1], + [-1, 1, 1], + [0, 1, 1], + [1, 1, 1], + [0, 2, 1], + ], + divisor: 8, + }, + "jarvis-judice-ninke": { + kernel: [ + [1, 0, 7], + [2, 0, 5], + [-2, 1, 3], + [-1, 1, 5], + [0, 1, 7], + [1, 1, 5], + [2, 1, 3], + [-2, 2, 1], + [-1, 2, 3], + [0, 2, 5], + [1, 2, 3], + [2, 2, 1], + ], + divisor: 48, + }, + stucki: { + kernel: [ + [1, 0, 8], + [2, 0, 4], + [-2, 1, 2], + [-1, 1, 4], + [0, 1, 8], + [1, 1, 4], + [2, 1, 2], + [-2, 2, 1], + [-1, 2, 2], + [0, 2, 4], + [1, 2, 2], + [2, 2, 1], + ], + divisor: 42, + }, + burkes: { + kernel: [ + [1, 0, 8], + [2, 0, 4], + [-2, 1, 2], + [-1, 1, 4], + [0, 1, 8], + [1, 1, 4], + [2, 1, 2], + ], + divisor: 32, + }, + sierra: { + kernel: [ + [1, 0, 5], + [2, 0, 3], + [-2, 1, 2], + [-1, 1, 4], + [0, 1, 5], + [1, 1, 4], + [2, 1, 2], + [-1, 2, 2], + [0, 2, 3], + [1, 2, 2], + ], + divisor: 32, + }, + "sierra-lite": { + kernel: [ + [1, 0, 2], + [-1, 1, 1], + [0, 1, 1], + ], + divisor: 4, + }, + "two-row-sierra": { + kernel: [ + [1, 0, 4], + [2, 0, 3], + [-2, 1, 1], + [-1, 1, 2], + [0, 1, 3], + [1, 1, 2], + [2, 1, 1], + ], + divisor: 16, + }, +}; + +const DEFAULTS = { + algorithm: "floyd-steinberg", + brightness: 1, + contrast: 1.2, + detail: 1, + palette: ["#000000", "#ffffff"], + pointSize: 3, +}; + +export function errorDiffusionBufferLength(width, height, pointSize) { + return Math.ceil(width / pointSize) * Math.ceil(height / pointSize) * 3; +} + +export function applyErrorDiffusionRgba(data, width, height, options = {}, errorBuffer) { + if (!Number.isInteger(width) || width < 1 || !Number.isInteger(height) || height < 1) { + throw new Error("width and height must be positive integers"); + } + if (!data || data.length !== width * height * 4) { + throw new Error(`RGBA data must contain ${width * height * 4} bytes`); + } + + const algorithm = options.algorithm ?? DEFAULTS.algorithm; + const diffusion = ERROR_DIFFUSION_ALGORITHMS[algorithm]; + if (!diffusion) throw new Error(`unknown error-diffusion algorithm: ${algorithm}`); + + const pointSize = integerInRange(options.pointSize ?? DEFAULTS.pointSize, 1, 20, "pointSize"); + const brightness = numberInRange(options.brightness ?? DEFAULTS.brightness, 0.5, 2, "brightness"); + const contrast = numberInRange(options.contrast ?? DEFAULTS.contrast, 0.5, 2, "contrast"); + const detail = numberInRange(options.detail ?? DEFAULTS.detail, 0.1, 1, "detail"); + const palette = parsePalette(options.palette ?? DEFAULTS.palette); + const blockColumns = Math.ceil(width / pointSize); + const blockRows = Math.ceil(height / pointSize); + const errorLength = errorDiffusionBufferLength(width, height, pointSize); + const errors = errorBuffer ?? new Float32Array(errorLength); + if (!(errors instanceof Float32Array) || errors.length !== errorLength) { + throw new Error(`errorBuffer must be a Float32Array of length ${errorLength}`); + } + errors.fill(0); + + const centerOffset = Math.floor(pointSize / 2); + for (let blockRow = 0; blockRow < blockRows; blockRow++) { + const blockY = blockRow * pointSize; + for (let blockColumn = 0; blockColumn < blockColumns; blockColumn++) { + const blockX = blockColumn * pointSize; + const centerX = Math.min(blockX + centerOffset, width - 1); + const centerY = Math.min(blockY + centerOffset, height - 1); + const rgbaIndex = (centerY * width + centerX) * 4; + const errorIndex = (blockRow * blockColumns + blockColumn) * 3; + const red = correctedChannel(data[rgbaIndex], errors[errorIndex], brightness, contrast); + const green = correctedChannel( + data[rgbaIndex + 1], + errors[errorIndex + 1], + brightness, + contrast, + ); + const blue = correctedChannel( + data[rgbaIndex + 2], + errors[errorIndex + 2], + brightness, + contrast, + ); + const luminance = 0.299 * red + 0.587 * green + 0.114 * blue; + const output = palette[Math.min(palette.length - 1, Math.floor(luminance * palette.length))]; + + for (let y = blockY; y < Math.min(blockY + pointSize, height); y++) { + for (let x = blockX; x < Math.min(blockX + pointSize, width); x++) { + const outputIndex = (y * width + x) * 4; + data[outputIndex] = Math.round(output[0] * 255); + data[outputIndex + 1] = Math.round(output[1] * 255); + data[outputIndex + 2] = Math.round(output[2] * 255); + } + } + + for (const [dx, dy, weight] of diffusion.kernel) { + const targetColumn = blockColumn + dx; + const targetRow = blockRow + dy; + if ( + targetColumn < 0 || + targetColumn >= blockColumns || + targetRow < 0 || + targetRow >= blockRows + ) { + continue; + } + const target = (targetRow * blockColumns + targetColumn) * 3; + const scale = (weight / diffusion.divisor) * detail; + errors[target] += (red - output[0]) * scale; + errors[target + 1] += (green - output[1]) * scale; + errors[target + 2] += (blue - output[2]) * scale; + } + } + } + return data; +} + +function correctedChannel(byte, error, brightness, contrast) { + return Math.min(1, Math.max(0, ((byte / 255 - 0.5) * contrast + 0.5) * brightness + error)); +} + +function parsePalette(colors) { + if (!Array.isArray(colors) || colors.length < 2 || colors.length > 6) { + throw new Error("palette must contain 2 to 6 colors"); + } + return colors.map((color) => { + const match = /^#([0-9a-f]{6})$/i.exec(color); + if (!match) throw new Error(`palette color must use #rrggbb: ${color}`); + const value = Number.parseInt(match[1], 16); + return [(value >> 16) / 255, ((value >> 8) & 255) / 255, (value & 255) / 255]; + }); +} + +function numberInRange(value, min, max, name) { + const number = Number(value); + if (!Number.isFinite(number) || number < min || number > max) { + throw new Error(`${name} must be between ${min} and ${max}`); + } + return number; +} + +function integerInRange(value, min, max, name) { + const number = Number(value); + if (!Number.isInteger(number) || number < min || number > max) { + throw new Error(`${name} must be an integer between ${min} and ${max}`); + } + return number; +} diff --git a/skills/media-use/scripts/lib/error-diffusion.test.mjs b/packages/cli/src/media-use/lib/error-diffusion.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/error-diffusion.test.mjs rename to packages/cli/src/media-use/lib/error-diffusion.test.mjs diff --git a/skills/media-use/scripts/lib/freeze.mjs b/packages/cli/src/media-use/lib/freeze.mjs similarity index 100% rename from skills/media-use/scripts/lib/freeze.mjs rename to packages/cli/src/media-use/lib/freeze.mjs diff --git a/skills/media-use/scripts/lib/freeze.test.mjs b/packages/cli/src/media-use/lib/freeze.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/freeze.test.mjs rename to packages/cli/src/media-use/lib/freeze.test.mjs diff --git a/skills/media-use/scripts/lib/grade-analyzer.mjs b/packages/cli/src/media-use/lib/grade-analyzer.mjs similarity index 100% rename from skills/media-use/scripts/lib/grade-analyzer.mjs rename to packages/cli/src/media-use/lib/grade-analyzer.mjs diff --git a/skills/media-use/scripts/lib/grade-analyzer.test.mjs b/packages/cli/src/media-use/lib/grade-analyzer.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/grade-analyzer.test.mjs rename to packages/cli/src/media-use/lib/grade-analyzer.test.mjs diff --git a/skills/media-use/scripts/lib/heygen-cli.mjs b/packages/cli/src/media-use/lib/heygen-cli.mjs similarity index 100% rename from skills/media-use/scripts/lib/heygen-cli.mjs rename to packages/cli/src/media-use/lib/heygen-cli.mjs diff --git a/skills/media-use/scripts/lib/heygen-cli.test.mjs b/packages/cli/src/media-use/lib/heygen-cli.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/heygen-cli.test.mjs rename to packages/cli/src/media-use/lib/heygen-cli.test.mjs diff --git a/skills/media-use/scripts/lib/heygen-search.mjs b/packages/cli/src/media-use/lib/heygen-search.mjs similarity index 100% rename from skills/media-use/scripts/lib/heygen-search.mjs rename to packages/cli/src/media-use/lib/heygen-search.mjs diff --git a/skills/media-use/scripts/lib/heygen-search.test.mjs b/packages/cli/src/media-use/lib/heygen-search.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/heygen-search.test.mjs rename to packages/cli/src/media-use/lib/heygen-search.test.mjs diff --git a/skills/media-use/scripts/lib/heygen-video-provider.mjs b/packages/cli/src/media-use/lib/heygen-video-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/heygen-video-provider.mjs rename to packages/cli/src/media-use/lib/heygen-video-provider.mjs diff --git a/skills/media-use/scripts/lib/heygen-video-provider.test.mjs b/packages/cli/src/media-use/lib/heygen-video-provider.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/heygen-video-provider.test.mjs rename to packages/cli/src/media-use/lib/heygen-video-provider.test.mjs diff --git a/skills/media-use/scripts/lib/image-provider.mjs b/packages/cli/src/media-use/lib/image-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/image-provider.mjs rename to packages/cli/src/media-use/lib/image-provider.mjs diff --git a/packages/cli/src/media-use/lib/index-gen.mjs b/packages/cli/src/media-use/lib/index-gen.mjs new file mode 100644 index 0000000000..65b019a92e --- /dev/null +++ b/packages/cli/src/media-use/lib/index-gen.mjs @@ -0,0 +1,63 @@ +import { writeFileSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { readManifest, indexPath } from "./manifest.mjs"; + +function pad(str, len) { + return String(str ?? "").padEnd(len); +} + +function formatDur(record) { + if (record.duration == null) return "—"; + return `${record.duration}s`; +} + +function formatDims(record) { + if (record.width && record.height) return `${record.width}×${record.height}`; + if (record.type === "icon" && record.transparent) return "svg"; + return "—"; +} + +export function generateIndexContent(records) { + const count = records.length; + const header = `# .media · ${count} asset${count === 1 ? "" : "s"}\n`; + if (count === 0) return header; + + const cols = { id: 4, type: 5, dur: 4, dims: 5, path: 5, desc: 11 }; + for (const r of records) { + cols.id = Math.max(cols.id, (r.id ?? "").length); + cols.type = Math.max(cols.type, (r.type ?? "").length); + cols.dur = Math.max(cols.dur, formatDur(r).length); + cols.dims = Math.max(cols.dims, formatDims(r).length); + cols.path = Math.max(cols.path, (r.path ?? "").length); + } + + const heading = + pad("id", cols.id + 2) + + pad("type", cols.type + 2) + + pad("dur", cols.dur + 2) + + pad("dims", cols.dims + 2) + + pad("path", cols.path + 2) + + "description"; + + const lines = [header, heading]; + for (const r of records) { + lines.push( + pad(r.id, cols.id + 2) + + pad(r.type, cols.type + 2) + + pad(formatDur(r), cols.dur + 2) + + pad(formatDims(r), cols.dims + 2) + + pad(r.path, cols.path + 2) + + (r.description ?? ""), + ); + } + return lines.join("\n") + "\n"; +} + +export function regenerateIndex(projectDir) { + const records = readManifest(projectDir); + const content = generateIndexContent(records); + const p = indexPath(projectDir); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, content); + return content; +} diff --git a/packages/cli/src/media-use/lib/local-media-search.mjs b/packages/cli/src/media-use/lib/local-media-search.mjs new file mode 100644 index 0000000000..3e0d31be22 --- /dev/null +++ b/packages/cli/src/media-use/lib/local-media-search.mjs @@ -0,0 +1,32 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +const DEFAULT_DIRECTORY = join(homedir(), ".hyperframes", "catalog"); + +function directory() { + return process.env.HYPERFRAMES_CATALOG_ARTIFACT_DIR || DEFAULT_DIRECTORY; +} + +export async function fetchMediaVectors(registryBaseUrl, options = {}) { + const target = options.directory || directory(); + const base = registryBaseUrl.replace(/\/+$/, ""); + mkdirSync(target, { recursive: true, mode: 0o700 }); + const fetched = []; + for (const file of ["media-vectors.json", "media-vectors.bin"]) { + const response = await fetch(`${base}/catalog-artifact/${file}`); + if (!response.ok) return false; + fetched.push([file, Buffer.from(await response.arrayBuffer())]); + } + for (const [file, bytes] of fetched) writeFileSync(join(target, file), bytes, { mode: 0o600 }); + return true; +} + +export function mediaVectorRows(target = directory()) { + const metadataPath = join(target, "media-vectors.json"); + if (!existsSync(metadataPath)) return []; + const metadata = JSON.parse(readFileSync(metadataPath, "utf8")); + return Array.isArray(metadata.rows) ? metadata.rows : []; +} + +export { rankMediaRowsWithVectors } from "./media-search.mjs"; diff --git a/skills/media-use/scripts/lib/local-models.mjs b/packages/cli/src/media-use/lib/local-models.mjs similarity index 100% rename from skills/media-use/scripts/lib/local-models.mjs rename to packages/cli/src/media-use/lib/local-models.mjs diff --git a/skills/media-use/scripts/lib/local-models.test.mjs b/packages/cli/src/media-use/lib/local-models.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/local-models.test.mjs rename to packages/cli/src/media-use/lib/local-models.test.mjs diff --git a/skills/media-use/scripts/lib/local-run.mjs b/packages/cli/src/media-use/lib/local-run.mjs similarity index 100% rename from skills/media-use/scripts/lib/local-run.mjs rename to packages/cli/src/media-use/lib/local-run.mjs diff --git a/skills/media-use/scripts/lib/local-run.test.mjs b/packages/cli/src/media-use/lib/local-run.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/local-run.test.mjs rename to packages/cli/src/media-use/lib/local-run.test.mjs diff --git a/skills/media-use/scripts/lib/logo-provider.mjs b/packages/cli/src/media-use/lib/logo-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/logo-provider.mjs rename to packages/cli/src/media-use/lib/logo-provider.mjs diff --git a/skills/media-use/scripts/lib/logo-provider.test.mjs b/packages/cli/src/media-use/lib/logo-provider.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/logo-provider.test.mjs rename to packages/cli/src/media-use/lib/logo-provider.test.mjs diff --git a/skills/media-use/scripts/lib/ltx-video-provider.mjs b/packages/cli/src/media-use/lib/ltx-video-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/ltx-video-provider.mjs rename to packages/cli/src/media-use/lib/ltx-video-provider.mjs diff --git a/skills/media-use/scripts/lib/ltx-video-provider.test.mjs b/packages/cli/src/media-use/lib/ltx-video-provider.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/ltx-video-provider.test.mjs rename to packages/cli/src/media-use/lib/ltx-video-provider.test.mjs diff --git a/skills/media-use/scripts/lib/lut-preset-provider.mjs b/packages/cli/src/media-use/lib/lut-preset-provider.mjs similarity index 96% rename from skills/media-use/scripts/lib/lut-preset-provider.mjs rename to packages/cli/src/media-use/lib/lut-preset-provider.mjs index a6c907accc..9b318fa935 100644 --- a/skills/media-use/scripts/lib/lut-preset-provider.mjs +++ b/packages/cli/src/media-use/lib/lut-preset-provider.mjs @@ -6,7 +6,11 @@ import { tokenOverlap } from "./match.mjs"; import { buildCube } from "./cube-build.mjs"; import { validateCube, validateCubeFile } from "./cube-validate.mjs"; -const SKILL_DIR = join(import.meta.dirname, "..", ".."); +const SKILL_DIR = [ + join(import.meta.dirname, ".."), + join(import.meta.dirname, "..", "..", "..", "..", "skills", "media-use"), + join(import.meta.dirname, "..", "..", "..", "..", "..", "skills", "media-use"), +].find((candidate) => existsSync(join(candidate, "luts", "index.json"))); const LUT_DIR = join(SKILL_DIR, "luts"); const LUT_INDEX = join(LUT_DIR, "index.json"); export const LIBRARY_LUT_OFFLINE_CODE = "MEDIA_USE_LIBRARY_LUT_OFFLINE"; diff --git a/skills/media-use/scripts/lib/lut-preset-provider.test.mjs b/packages/cli/src/media-use/lib/lut-preset-provider.test.mjs similarity index 99% rename from skills/media-use/scripts/lib/lut-preset-provider.test.mjs rename to packages/cli/src/media-use/lib/lut-preset-provider.test.mjs index 7cfcd209b1..84afebf18b 100644 --- a/skills/media-use/scripts/lib/lut-preset-provider.test.mjs +++ b/packages/cli/src/media-use/lib/lut-preset-provider.test.mjs @@ -22,7 +22,7 @@ import { import { buildCube } from "./cube-build.mjs"; import { validateCube, validateCubeFile } from "./cube-validate.mjs"; -const REPO_ROOT = join(import.meta.dirname, "..", "..", "..", ".."); +const REPO_ROOT = join(import.meta.dirname, "..", "..", "..", "..", ".."); function corePresetIdsFromSource() { const src = readFileSync(join(REPO_ROOT, "packages/core/src/colorGrading.ts"), "utf8"); diff --git a/packages/cli/src/media-use/lib/manifest.mjs b/packages/cli/src/media-use/lib/manifest.mjs new file mode 100644 index 0000000000..0bd2f7b3da --- /dev/null +++ b/packages/cli/src/media-use/lib/manifest.mjs @@ -0,0 +1,229 @@ +import { + readFileSync, + appendFileSync, + mkdirSync, + existsSync, + readdirSync, + openSync, + closeSync, + writeFileSync, + rmSync, + statSync, +} from "node:fs"; +import { join } from "node:path"; + +const MANIFEST_FILE = "manifest.jsonl"; +const INDEX_FILE = "index.md"; + +const TYPE_DIRS = { + bgm: "audio/bgm", + sfx: "audio/sfx", + voice: "audio/voice", + image: "images", + icon: "images", + logo: "images", + brand: "images", + video: "video", + grade: "luts", + lut: "luts", + recipe: "recipes", +}; + +export function mediaDir(projectDir) { + return join(projectDir, ".media"); +} + +export function manifestPath(projectDir) { + return join(mediaDir(projectDir), MANIFEST_FILE); +} + +export function indexPath(projectDir) { + return join(mediaDir(projectDir), INDEX_FILE); +} + +export function typeSubdir(type) { + const sub = TYPE_DIRS[type]; + if (!sub) throw new Error(`unknown media type: ${type}`); + return sub; +} + +export function typeDirPath(projectDir, type) { + return join(mediaDir(projectDir), typeSubdir(type)); +} + +export function readManifest(projectDir) { + const p = manifestPath(projectDir); + if (!existsSync(p)) return []; + const raw = readFileSync(p, "utf8"); + const records = []; + for (const line of raw.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + records.push(JSON.parse(trimmed)); + } catch { + // ponytail: skip malformed lines, don't crash + } + } + return records; +} + +export function appendRecord(projectDir, record) { + const dir = mediaDir(projectDir); + mkdirSync(dir, { recursive: true }); + const typeDir = typeDirPath(projectDir, record.type); + mkdirSync(typeDir, { recursive: true }); + + const p = manifestPath(projectDir); + const line = JSON.stringify(record) + "\n"; + appendFileSync(p, line); +} + +// Match prompts forgivingly. Agents rarely re-emit a byte-identical intent, so +// keying cache lookups on exact equality meant "Calm piano" and "calm piano" +// re-searched and re-downloaded. Normalize (trim, lowercase, collapse internal +// whitespace) on both sides; the raw prompt is still stored for audit. +export function normalizePrompt(prompt) { + return String(prompt ?? "") + .trim() + .toLowerCase() + .replace(/\s+/g, " "); +} + +export function findByPrompt(projectDir, prompt, type) { + const key = normalizePrompt(prompt); + if (!key) return null; + const records = readManifest(projectDir); + return ( + records.find( + (r) => normalizePrompt(r.provenance?.prompt) === key && (type == null || r.type === type), + ) || null + ); +} + +export function findByEntity(projectDir, entity) { + const lower = entity.toLowerCase(); + const records = readManifest(projectDir); + return records.find((r) => r.entity && r.entity.toLowerCase() === lower) || null; +} + +export function nextId(projectDir, type) { + const records = readManifest(projectDir); + const prefix = type; + let max = 0; + for (const r of records) { + if (r.type !== type) continue; + const m = r.id?.match(new RegExp(`^${prefix}_(\\d+)$`)); + if (m) max = Math.max(max, parseInt(m[1], 10)); + } + return `${prefix}_${String(max + 1).padStart(3, "0")}`; +} + +// Sync sleep (no busy-spin) for the allocation lock retry. +function sleepMs(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +// Coarse per-project lock so concurrent resolves don't race on id allocation. +// ponytail: one lock file with a 15s stale-steal (a crashed holder can't wedge +// the project); fine for agent-scale concurrency — revisit if throughput needs +// finer locking. Date.now() is available here (a normal Node CLI, not a +// workflow DSL), so mtime-based staleness is safe. +const LOCK_STALE_MS = 15000; +const LOCK_TIMEOUT_MS = 20000; + +function withLock(dir, fn) { + const lock = join(dir, ".lock"); + const start = Date.now(); + for (;;) { + try { + closeSync(openSync(lock, "wx")); // O_EXCL: atomic acquire + break; + } catch (err) { + if (err.code !== "EEXIST") throw err; + try { + if (Date.now() - statSync(lock).mtimeMs > LOCK_STALE_MS) { + rmSync(lock, { force: true }); // steal a stale lock from a dead holder + continue; + } + } catch { + continue; // lock vanished between check and stat — retry the acquire + } + if (Date.now() - start > LOCK_TIMEOUT_MS) { + throw new Error("media-use: timed out acquiring .media/.lock"); + } + sleepMs(25); + } + } + try { + return fn(); + } finally { + rmSync(lock, { force: true }); + } +} + +// Atomically allocate the next free id for `type` AND reserve its file, so a +// slow download/copy between allocation and appendRecord can't let a concurrent +// caller grab the same id (the MU-23 clobber). Under the lock we take the max id +// across BOTH the manifest and any already-reserved files in the type dir, then +// O_EXCL-create an empty placeholder at the target path; freeze/copy overwrites +// it. Returns { id, localPath }. +export function allocateId(projectDir, type, ext) { + mkdirSync(mediaDir(projectDir), { recursive: true }); + const typeDir = typeDirPath(projectDir, type); + mkdirSync(typeDir, { recursive: true }); + return withLock(mediaDir(projectDir), () => { + const re = new RegExp(`^${type}_(\\d+)`); + let max = 0; + for (const r of readManifest(projectDir)) { + if (r.type !== type) continue; + const m = r.id?.match(re); + if (m) max = Math.max(max, parseInt(m[1], 10)); + } + for (const f of readdirSync(typeDir)) { + const m = f.match(re); + if (m) max = Math.max(max, parseInt(m[1], 10)); // skip ids reserved but not yet appended + } + const id = `${type}_${String(max + 1).padStart(3, "0")}`; + const localPath = `.media/${typeSubdir(type)}/${id}${ext}`; + writeFileSync(join(projectDir, localPath), "", { flag: "wx" }); // durable reservation + return { id, localPath }; + }); +} + +function reservedFile(projectDir, type, ext) { + const allocation = allocateId(projectDir, type, ext); + return { ...allocation, fullPath: join(projectDir, allocation.localPath) }; +} + +function rollbackReservation(reservation) { + rmSync(reservation.fullPath, { force: true }); +} + +// A reservation is committed only when populate returns a non-null value. +// Throwing/rejecting or returning null means no usable asset was produced, so +// the placeholder must be released. Keeping this transaction beside allocateId +// prevents individual provider/cache/LUT paths from forgetting the rollback. +export function withReservedFileSync(projectDir, type, ext, populate) { + const reservation = reservedFile(projectDir, type, ext); + try { + const result = populate(reservation); + if (result == null) rollbackReservation(reservation); + return result; + } catch (error) { + rollbackReservation(reservation); + throw error; + } +} + +export async function withReservedFile(projectDir, type, ext, populate) { + const reservation = reservedFile(projectDir, type, ext); + try { + const result = await populate(reservation); + if (result == null) rollbackReservation(reservation); + return result; + } catch (error) { + rollbackReservation(reservation); + throw error; + } +} diff --git a/skills/media-use/scripts/lib/manifest.test.mjs b/packages/cli/src/media-use/lib/manifest.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/manifest.test.mjs rename to packages/cli/src/media-use/lib/manifest.test.mjs diff --git a/skills/media-use/scripts/lib/match.mjs b/packages/cli/src/media-use/lib/match.mjs similarity index 100% rename from skills/media-use/scripts/lib/match.mjs rename to packages/cli/src/media-use/lib/match.mjs diff --git a/packages/cli/src/media-use/lib/media-fetch.mjs b/packages/cli/src/media-use/lib/media-fetch.mjs new file mode 100644 index 0000000000..830e41618a --- /dev/null +++ b/packages/cli/src/media-use/lib/media-fetch.mjs @@ -0,0 +1,71 @@ +// Media downloads use public HTTP(S) URLs. Validate every redirect target; +// a provider result must meet the same host policy as a direct ingest URL. +// Public HTTPS-to-HTTP redirects are allowed, matching direct HTTP support. +// This is a literal-host policy, not DNS pinning: DNS resolution remains trusted. + +import { BlockList, isIP } from "node:net"; + +const blocked = new BlockList(); +for (const [network, prefix] of [ + ["0.0.0.0", 8], + ["10.0.0.0", 8], + ["100.64.0.0", 10], + ["127.0.0.0", 8], + ["169.254.0.0", 16], + ["172.16.0.0", 12], + ["192.0.0.0", 24], + ["192.0.2.0", 24], + ["192.88.99.0", 24], + ["192.168.0.0", 16], + ["198.18.0.0", 15], + ["198.51.100.0", 24], + ["203.0.113.0", 24], + ["224.0.0.0", 4], + ["240.0.0.0", 4], +]) + blocked.addSubnet(network, prefix, "ipv4"); +for (const [network, prefix] of [ + ["::", 128], + ["::1", 128], + ["fc00::", 7], + ["fe80::", 10], + ["fec0::", 10], + ["ff00::", 8], + ["2001:db8::", 32], +]) + blocked.addSubnet(network, prefix, "ipv6"); + +export function isPublicMediaUrl(value) { + try { + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") return false; + const host = url.hostname.replace(/\.$/, ""); + if ( + host === "localhost" || + host.endsWith(".localhost") || + host.endsWith(".local") || + host.endsWith(".internal") + ) + return false; + const address = host.replace(/^\[|\]$/g, ""); + const family = isIP(address); + return family === 0 || !blocked.check(address, family === 4 ? "ipv4" : "ipv6"); + } catch { + return false; + } +} + +export async function fetchMedia(url, { method = "GET", signal, fetchImpl = fetch } = {}) { + let current = String(url); + for (let hop = 0; hop <= 5; hop++) { + if (!isPublicMediaUrl(current)) + throw new Error("Media download blocked: URL is not public HTTP(S)"); + const response = await fetchImpl(current, { method, signal, redirect: "manual" }); + if (!(response.status >= 300 && response.status < 400)) return response; + const location = response.headers.get("location"); + if (!location) return response; + await response.body?.cancel(); + current = new URL(location, current).href; + } + throw new Error("Media download exceeded redirect limit"); +} diff --git a/skills/media-use/scripts/lib/media-fetch.test.mjs b/packages/cli/src/media-use/lib/media-fetch.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/media-fetch.test.mjs rename to packages/cli/src/media-use/lib/media-fetch.test.mjs diff --git a/packages/cli/src/media-use/lib/media-search.d.mts b/packages/cli/src/media-use/lib/media-search.d.mts new file mode 100644 index 0000000000..854d715d97 --- /dev/null +++ b/packages/cli/src/media-use/lib/media-search.d.mts @@ -0,0 +1,9 @@ +export function rankMediaRows< + T extends { + id: string; + kind: string; + title: string; + description: string; + tags: string[]; + }, +>(query: string, rows: T[]): T[]; diff --git a/packages/cli/src/media-use/lib/media-search.mjs b/packages/cli/src/media-use/lib/media-search.mjs new file mode 100644 index 0000000000..ab1c6b31e5 --- /dev/null +++ b/packages/cli/src/media-use/lib/media-search.mjs @@ -0,0 +1,107 @@ +const STOP = new Set( + ( + "the a an and or of to in on at is are be it its for with that this as by from into " + + "one two must not no all over under across while when where which who whom whose they " + + "them their we our you your he she his her but if then than so such can may might will " + + "would should each other another same both few more most some any every" + ).split(" "), +); + +const PLURAL_RULES = [ + [/([^aeiou])ies$/, "$1y"], + [/(ch|sh|ss|x)es$/, "$1"], + [/(ss|us|is)$/, "$&"], + [/s$/, ""], +]; +const STRONG_FIELD_WEIGHT = 3; +const INFERRED_TOKEN_WEIGHT = 0.35; + +function singularize(word) { + if (word.length <= 3) return word; + for (const [pattern, replacement] of PLURAL_RULES) { + if (pattern.test(word)) return word.replace(pattern, replacement); + } + return word; +} + +function tokenize(text) { + const words = + String(text) + .toLowerCase() + .match(/[a-z]+/g) || []; + return words.filter((word) => word.length > 2 && !STOP.has(word)).map(singularize); +} + +export function rankMediaRows(query, rows) { + const asked = tokenize(query); + const normalizedLiteralQuery = String(query).trim().toLowerCase(); + const normalizedQuery = asked.join(" "); + const want = new Map(asked.map((token) => [token, 1])); + if (want.size === 0) return []; + + const parsed = rows.map((row) => { + const strongTokens = new Set(tokenize(`${row.id} ${row.title}`)); + const allTokens = new Set([ + ...strongTokens, + ...tokenize(`${row.description} ${row.tags.join(" ")} ${row.kind}`), + ]); + return { + row, + strongTokens, + allTokens, + literalId: row.id.trim().toLowerCase() === normalizedLiteralQuery, + stemmedId: tokenize(row.id).join(" ") === normalizedQuery, + }; + }); + const vocabulary = new Set(parsed.flatMap(({ allTokens }) => [...allTokens])); + for (const token of asked) { + if (token.length < 6) continue; + for (let cut = 3; cut <= token.length - 3; cut++) { + const head = token.slice(0, cut); + const tail = token.slice(cut); + if (vocabulary.has(head) && vocabulary.has(tail)) { + want.set(head, INFERRED_TOKEN_WEIGHT); + want.set(tail, INFERRED_TOKEN_WEIGHT); + break; + } + } + } + for (let index = 0; index < asked.length - 1; index++) { + const joined = `${asked[index]}${asked[index + 1]}`; + if (vocabulary.has(joined)) want.set(joined, INFERRED_TOKEN_WEIGHT); + } + + const idf = new Map(); + for (const token of want.keys()) { + const documentFrequency = parsed.filter(({ allTokens }) => allTokens.has(token)).length; + idf.set(token, Math.log((parsed.length + 1) / (documentFrequency + 1)) + 1); + } + return parsed + .map(({ row, strongTokens, allTokens, literalId, stemmedId }) => { + let score = 0; + for (const [token, asking] of want) { + const weight = (idf.get(token) || 1) * asking; + if (strongTokens.has(token)) score += STRONG_FIELD_WEIGHT * weight; + else if (allTokens.has(token)) score += weight; + } + return { row, score, literalId, stemmedId }; + }) + .filter(({ score }) => score > 0) + .sort( + (a, b) => + Number(b.literalId) - Number(a.literalId) || + Number(b.stemmedId) - Number(a.stemmedId) || + b.score - a.score, + ) + .map(({ row }) => row); +} + +export async function rankMediaRowsWithVectors(query, rows, semanticRanking) { + const words = rankMediaRows(query, rows); + if (words.length > 0) return { rows: words, tier: "words" }; + if (!semanticRanking) return { rows: [], tier: "words" }; + const semantic = await semanticRanking(query, rows); + return semantic + ? { rows: semantic.map(({ row }) => row), tier: "on-device" } + : { rows: [], tier: "words" }; +} diff --git a/skills/media-use/scripts/lib/mflux-provider.mjs b/packages/cli/src/media-use/lib/mflux-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/mflux-provider.mjs rename to packages/cli/src/media-use/lib/mflux-provider.mjs diff --git a/skills/media-use/scripts/lib/mflux-provider.test.mjs b/packages/cli/src/media-use/lib/mflux-provider.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/mflux-provider.test.mjs rename to packages/cli/src/media-use/lib/mflux-provider.test.mjs diff --git a/skills/media-use/scripts/lib/misses.mjs b/packages/cli/src/media-use/lib/misses.mjs similarity index 100% rename from skills/media-use/scripts/lib/misses.mjs rename to packages/cli/src/media-use/lib/misses.mjs diff --git a/skills/media-use/scripts/lib/misses.test.mjs b/packages/cli/src/media-use/lib/misses.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/misses.test.mjs rename to packages/cli/src/media-use/lib/misses.test.mjs diff --git a/packages/cli/src/media-use/lib/npx-sync.mjs b/packages/cli/src/media-use/lib/npx-sync.mjs new file mode 100644 index 0000000000..8ada152d05 --- /dev/null +++ b/packages/cli/src/media-use/lib/npx-sync.mjs @@ -0,0 +1,31 @@ +import { existsSync } from "node:fs"; +import { resolveSpawnCommand } from "../../audio/scripts/lib/tts.mjs"; + +// Sync-spawn analog of the audio engine's spawnP, for execFileSync call sites +// that must hard-fail (rather than fall through to another provider) when npx +// cannot be resolved. On Windows a bare "npx" is npx.cmd, which +// execFileSync/spawnSync cannot exec (spawnSync npx ENOENT) — +// resolveSpawnCommand reroutes it through node + npx-cli.js, no shell:true. +// +// `platform`/`env`/`pathExists` params (defaulting to the real values) exist +// so tests can exercise the win32 branch without mocking node:child_process +// (its ESM exports are non-configurable) — same idiom as spawnP and +// localTtsGenerate. +export function resolveNpxInvocation( + argv, + opts, + platform = process.platform, + env = process.env, + pathExists = existsSync, +) { + const resolved = resolveSpawnCommand("npx", argv, opts, platform, env, pathExists); + if (!resolved) { + // npx-on-win32 with no resolvable npx-cli.js — same terminal condition + // spawnP warns about, surfaced as a throw for callers with no fallback. + throw new Error( + "cannot run npx on Windows: npm's npx-cli.js was not found " + + "(install npm with Node, or run via npx/npm run so npm_execpath is set)", + ); + } + return resolved; +} diff --git a/skills/media-use/scripts/lib/npx-sync.test.mjs b/packages/cli/src/media-use/lib/npx-sync.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/npx-sync.test.mjs rename to packages/cli/src/media-use/lib/npx-sync.test.mjs diff --git a/packages/cli/src/media-use/lib/parakeet-words.mjs b/packages/cli/src/media-use/lib/parakeet-words.mjs new file mode 100644 index 0000000000..04df79c7d4 --- /dev/null +++ b/packages/cli/src/media-use/lib/parakeet-words.mjs @@ -0,0 +1,27 @@ +// Merge Parakeet-MLX token timestamps into word timestamps. +// +// parakeet-mlx JSON emits SUB-WORD tokens (" H", "ello", ...) with per-token +// start/end. Captions + transcript-cut need WORD timestamps, so join tokens +// into words on the space boundary: a token whose text starts with a space +// (or the very first token) begins a new word; the rest append. Output matches +// the { words: [{ text, start, end }] } shape the rest of media-use consumes +// (see words.mjs / cutlist.mjs). + +export function mergeTokensToWords(parakeet) { + const sentences = Array.isArray(parakeet?.sentences) ? parakeet.sentences : []; + const words = []; + for (const s of sentences) { + for (const t of s.tokens ?? []) { + const raw = typeof t.text === "string" ? t.text : ""; + const startsWord = raw.startsWith(" ") || words.length === 0; + if (startsWord) { + words.push({ text: raw.trim(), start: t.start, end: t.end }); + } else { + const w = words[words.length - 1]; + w.text += raw; + w.end = t.end; + } + } + } + return { text: (parakeet?.text ?? "").trim(), words: words.filter((w) => w.text.length > 0) }; +} diff --git a/skills/media-use/scripts/lib/parakeet-words.test.mjs b/packages/cli/src/media-use/lib/parakeet-words.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/parakeet-words.test.mjs rename to packages/cli/src/media-use/lib/parakeet-words.test.mjs diff --git a/packages/cli/src/media-use/lib/prefs-store.mjs b/packages/cli/src/media-use/lib/prefs-store.mjs new file mode 100644 index 0000000000..6bc3f7345f --- /dev/null +++ b/packages/cli/src/media-use/lib/prefs-store.mjs @@ -0,0 +1,180 @@ +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; + +/** + * Remembered defaults — the lightweight tier of HyperFrames user memory. + * + * Two files, same shape as the rest of media-use's storage split: + * - project `.media/preferences.json` — committed with the repo, so the whole + * team inherits it; written every time a brief answer is confirmed. + * - user `~/.media/preferences.json` — personal, cross-repo. A key is promoted + * here only once the same value has been confirmed in two different projects + * (`PROMOTE_AT`), so a one-off choice never pollutes the global defaults. + * Pre-promotion evidence accumulates in the user file's `sightings` ledger — + * project files can't see each other, so the cross-project count has to live + * user-side. + * + * Consumption contract (brief-contract § 2, Remembered defaults): a remembered + * value becomes the recommended option with a receipt naming its source — it + * never skips a question, and explicit request content always wins. + */ + +const PREFS_FILE = "preferences.json"; + +/** Keys the brief contract records; `style_preset` is stored per workflow. */ +export const PREFERENCE_KEYS = [ + "destination", + "aspect", + "language", + "flow", + "storyboard", + "voice", + "style_preset", +]; + +/** A value must be confirmed in this many distinct projects to go user-tier. */ +export const PROMOTE_AT = 2; + +export function projectPrefsPath(projectDir) { + return join(resolve(projectDir), ".media", PREFS_FILE); +} + +export function userPrefsPath() { + return join(homedir(), ".media", PREFS_FILE); +} + +function emptyFile() { + return { version: 1, preferences: {}, sightings: {} }; +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** Tolerant read — a missing or malformed file counts as empty. */ +function readPrefsFile(path) { + try { + if (!existsSync(path)) return emptyFile(); + const parsed = JSON.parse(readFileSync(path, "utf8")); + if (!isRecord(parsed)) return emptyFile(); + return { + version: 1, + preferences: isRecord(parsed.preferences) ? parsed.preferences : {}, + sightings: isRecord(parsed.sightings) ? parsed.sightings : {}, + }; + } catch { + return emptyFile(); + } +} + +/** Atomic write (tmp + rename) so a crash never leaves a torn file. */ +function writePrefsFile(path, file) { + mkdirSync(dirname(path), { recursive: true }); + const tmp = `${path}.tmp`; + writeFileSync(tmp, `${JSON.stringify(file, null, 2)}\n`); + renameSync(tmp, path); +} + +/** `style_preset` entries are stored per workflow as `style_preset.`. */ +export function preferenceKeyFor(key, workflow) { + return key === "style_preset" && workflow ? `style_preset.${workflow}` : key; +} + +function validEntry(entry) { + return isRecord(entry) && typeof entry.value === "string" && entry.value.length > 0; +} + +/** + * The merged view the brief reads: user-tier promoted entries first, project + * entries on top (project wins). Each entry carries `source` plus the receipt + * material (`confirmed_in`, `updated_at`). + */ +export function mergedPreferences(projectDir) { + const user = readPrefsFile(userPrefsPath()); + const project = readPrefsFile(projectPrefsPath(projectDir)); + const merged = {}; + for (const [key, entry] of Object.entries(user.preferences)) { + if (validEntry(entry)) merged[key] = { ...entry, source: "user" }; + } + for (const [key, entry] of Object.entries(project.preferences)) { + if (validEntry(entry)) merged[key] = { ...entry, source: "project" }; + } + return merged; +} + +function dedupe(list) { + return [...new Set(list)]; +} + +/** + * Project tier: same value accumulates confirmations; a changed value starts + * provenance over (the old confirmations vouched for the old value). + */ +function recordProjectTier(projectDir, fullKey, value, projectName, now) { + const path = projectPrefsPath(projectDir); + const file = readPrefsFile(path); + const previous = file.preferences[fullKey]; + const keepProvenance = validEntry(previous) && previous.value === value; + const confirmedIn = keepProvenance + ? dedupe([...(Array.isArray(previous.confirmed_in) ? previous.confirmed_in : []), projectName]) + : [projectName]; + file.preferences[fullKey] = { value, confirmed_in: confirmedIn, updated_at: now }; + writePrefsFile(path, file); + return confirmedIn; +} + +/** + * User tier: accumulate this sighting in the ledger, and promote the key once + * the same value has been confirmed in PROMOTE_AT distinct projects. + */ +function recordUserSighting(fullKey, value, projectName, now) { + const path = userPrefsPath(); + const file = readPrefsFile(path); + const keySightings = isRecord(file.sightings[fullKey]) ? file.sightings[fullKey] : {}; + const seenIn = dedupe([ + ...(Array.isArray(keySightings[value]) ? keySightings[value] : []), + projectName, + ]); + keySightings[value] = seenIn; + file.sightings[fullKey] = keySightings; + const promoted = seenIn.length >= PROMOTE_AT; + if (promoted) { + file.preferences[fullKey] = { value, confirmed_in: seenIn, updated_at: now }; + } + writePrefsFile(path, file); + return promoted; +} + +/** + * Record one confirmed brief answer. Always writes the project tier; feeds the + * user tier's sightings ledger and promotes once the same value has been + * confirmed in PROMOTE_AT distinct projects. Idempotent per project. + */ +export function recordPreference({ projectDir, key, value, workflow }) { + if (!PREFERENCE_KEYS.includes(key)) { + throw new Error(`unknown preference key: "${key}" (known: ${PREFERENCE_KEYS.join(", ")})`); + } + if (typeof value !== "string" || !value.trim()) { + throw new Error("a preference needs a non-empty string value"); + } + if (key === "style_preset" && (!workflow || !String(workflow).trim())) { + throw new Error("style_preset is stored per workflow — pass --workflow "); + } + const fullKey = preferenceKeyFor(key, workflow); + const projectName = basename(resolve(projectDir)); + const trimmed = value.trim(); + const now = new Date().toISOString(); + + const confirmedIn = recordProjectTier(projectDir, fullKey, trimmed, projectName, now); + + // Best-effort — a read-only home directory must never fail a brief. + let promoted = false; + try { + promoted = recordUserSighting(fullKey, trimmed, projectName, now); + } catch { + // The project record already landed; promotion just waits for next time. + } + + return { key: fullKey, value: trimmed, confirmed_in: confirmedIn, promoted }; +} diff --git a/skills/media-use/scripts/lib/prefs-store.test.mjs b/packages/cli/src/media-use/lib/prefs-store.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/prefs-store.test.mjs rename to packages/cli/src/media-use/lib/prefs-store.test.mjs diff --git a/skills/media-use/scripts/lib/probe.mjs b/packages/cli/src/media-use/lib/probe.mjs similarity index 100% rename from skills/media-use/scripts/lib/probe.mjs rename to packages/cli/src/media-use/lib/probe.mjs diff --git a/skills/media-use/scripts/lib/probe.test.mjs b/packages/cli/src/media-use/lib/probe.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/probe.test.mjs rename to packages/cli/src/media-use/lib/probe.test.mjs diff --git a/skills/media-use/scripts/lib/providers.mjs b/packages/cli/src/media-use/lib/providers.mjs similarity index 100% rename from skills/media-use/scripts/lib/providers.mjs rename to packages/cli/src/media-use/lib/providers.mjs diff --git a/packages/cli/src/media-use/lib/recipe-store.mjs b/packages/cli/src/media-use/lib/recipe-store.mjs new file mode 100644 index 0000000000..e11ec96a31 --- /dev/null +++ b/packages/cli/src/media-use/lib/recipe-store.mjs @@ -0,0 +1,367 @@ +import { + cpSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { appendRecord, mediaDir, nextId } from "./manifest.mjs"; +import { regenerateIndex } from "./index-gen.mjs"; +import { mergedPreferences } from "./prefs-store.mjs"; + +/** + * Recipes — the heavyweight tier of HyperFrames user memory. + * + * A recipe is the full confirmed bundle for one video type: the frozen design + * spec (`frame.md`), the storyboard skeleton (structure with the content + * blanked), and the confirmed brief values — frozen after the run's final + * approval, reused to start the next video of the same type from everything + * already approved. + * + * Storage is **named folders**, not content-addressed cache entries: a recipe + * is an evolving bundle with a `version`, so re-freezing the same name bumps + * the version and archives the old folder as `@v`. Two tiers, same + * split as everything else in media-use: project `.media/recipes//` + * (committed) and user `~/.media/recipes//` (a freeze is already a + * confirmed bundle, so it promotes immediately — no two-project rule here). + */ + +/** Frontmatter keys that describe THIS video, not the reusable type. */ +const FRONTMATTER_CONTENT_KEYS = new Set(["message", "audience", "mode"]); + +/** BRIEF.md frontmatter keys that describe this run, not the reusable type — + * a recipe never locks the run's shape, so the intent layer always re-asks. */ +const BRIEF_CONTENT_KEYS = new Set(["flow", "storyboard", "message", "audience"]); + +/** Per-frame metadata that is content, not structure. */ +const FRAME_CONTENT_KEYS = new Set([ + "voiceover", + "vo", + "voice_over", + "narration", + "scene", + "description", + "summary", + "caption", + "asset_candidates", +]); + +const FRAME_HEADING_RE = /^(#{2,3})\s+(?:frame|beat|scene)\s+\d+/i; + +export function projectRecipesDir(projectDir) { + return join(mediaDir(projectDir), "recipes"); +} + +export function userRecipesDir() { + return join(homedir(), ".media", "recipes"); +} + +export function slugifyRecipeName(name) { + const slug = String(name ?? "") + .trim() + .toLowerCase() + .replace(/[\s_]+/g, "-") + .replace(/[^a-z0-9-]/g, "") + .replace(/-+/g, "-") + .replace(/^-|-$/g, ""); + if (!slug) throw new Error(`recipe name "${name}" has no usable characters`); + return slug; +} + +function frameTitle(headingLine) { + const dash = headingLine.split(/\s+—\s+/)[1]; + if (dash && dash.trim()) return dash.trim(); + return headingLine.replace(/^#+\s*/, "").trim(); +} + +/** Frontmatter: drop the content keys, keep structure/style keys verbatim. */ +function skeletonFrontmatter(lines, out, contentKeys = FRONTMATTER_CONTENT_KEYS) { + if (lines[0]?.trim() !== "---") return 0; + out.push(lines[0]); + let i = 1; + while (i < lines.length && lines[i].trim() !== "---") { + const key = lines[i].match(/^(\w+)\s*:/)?.[1]?.toLowerCase(); + if (!key || !contentKeys.has(key)) out.push(lines[i]); + i++; + } + if (i < lines.length) { + out.push(lines[i]); // closing --- + i++; + } + return i; +} + +/** One line inside a frame section — returns the replacement lines (may be none). */ +function skeletonFrameLine(line, state, out) { + const bulletKey = line.match(/^-\s+(\w+)\s*:/)?.[1]?.toLowerCase(); + if (bulletKey) { + if (bulletKey === "status") out.push("- status: outline"); + else if (!FRAME_CONTENT_KEYS.has(bulletKey)) out.push(line); + return; + } + if (!line.trim()) { + out.push(line); + return; + } + // Frame prose: one placeholder per frame in place of the narrative. + if (!state.proseReplaced) { + out.push( + ``, + ); + state.proseReplaced = true; + } +} + +/** + * Skeletonize a STORYBOARD.md: keep the reusable structure (frame count, + * durations, transitions, src paths, the Video direction block, style-ish + * frontmatter), reset every status to `outline`, and blank the content + * (message/audience, narration guides, per-frame prose) down to a fill-in + * placeholder that names the frame's role. + */ +/** + * Skeletonize a BRIEF.md: keep the frontmatter's reusable keys (workflow, + * destination, aspect, language, length, angle…), drop the run-shape and + * content keys (flow, storyboard, message, audience), and blank each body + * section down to a fill-in placeholder under its kept heading. + */ +export function skeletonizeBrief(source) { + const lines = String(source ?? "").split(/\r?\n/); + const out = []; + let i = skeletonFrontmatter(lines, out, BRIEF_CONTENT_KEYS); + for (; i < lines.length; i++) { + const heading = lines[i].match(/^##\s+(.+)$/); + if (heading) { + out.push(lines[i], ""); + out.push( + ``, + ); + out.push(""); + } + } + return out.join("\n").replace(/\n{3,}/g, "\n\n"); +} + +export function skeletonizeStoryboard(source) { + const lines = String(source ?? "").split(/\r?\n/); + const out = []; + const state = { inFrame: false, proseReplaced: false, title: "" }; + for (let i = skeletonFrontmatter(lines, out); i < lines.length; i++) { + const line = lines[i]; + if (/^#{2,3}\s/.test(line)) { + state.inFrame = FRAME_HEADING_RE.test(line); + state.proseReplaced = false; + state.title = state.inFrame ? frameTitle(line) : ""; + out.push(line); + } else if (!state.inFrame) { + out.push(line); + } else { + skeletonFrameLine(line, state, out); + } + } + return out.join("\n").replace(/\n{3,}/g, "\n\n"); +} + +/** The run's workflow as BRIEF.md records it — the source of truth a freeze + * must not contradict. Undefined when no BRIEF.md (or no `workflow:`) exists. */ +function briefWorkflow(root) { + const brief = join(root, "BRIEF.md"); + if (!existsSync(brief)) return undefined; + const lines = readFileSync(brief, "utf8").split(/\r?\n/); + if (lines[0]?.trim() !== "---") return undefined; + for (let i = 1; i < lines.length && lines[i].trim() !== "---"; i++) { + const match = lines[i].match(/^workflow\s*:\s*(.+?)\s*$/); + if (match) return match[1].replace(/^["']|["']$/g, "") || undefined; + } + return undefined; +} + +function readRecipeJson(dir) { + try { + const parsed = JSON.parse(readFileSync(join(dir, "recipe.json"), "utf8")); + if (typeof parsed !== "object" || parsed === null || typeof parsed.name !== "string") { + return null; + } + return parsed; + } catch { + return null; + } +} + +function prefValue(prefs, key) { + return prefs[key]?.value; +} + +/** + * Freeze the current project's approved run as a named recipe. Writes the + * project-tier folder + a manifest record, then copies to the user tier (a + * freeze is already confirmed — it promotes immediately). + */ +export function freezeRecipe({ projectDir, name, workflow, blocks }) { + const slug = slugifyRecipeName(name); + const root = resolve(projectDir); + const fromBrief = briefWorkflow(root); + const fromFlag = workflow && String(workflow).trim() ? String(workflow).trim() : undefined; + // BRIEF.md decides; the flag only covers projects briefed before it existed. + const resolvedWorkflow = fromBrief ?? fromFlag; + if (!resolvedWorkflow) { + throw new Error("no workflow found — BRIEF.md names none and no --workflow was given"); + } + const frameSpec = join(root, "frame.md"); + const storyboard = join(root, "STORYBOARD.md"); + if (!existsSync(frameSpec)) throw new Error("no frame.md to freeze — run the design step first"); + if (!existsSync(storyboard)) throw new Error("no STORYBOARD.md to freeze"); + + const dir = join(projectRecipesDir(root), slug); + let version = 1; + const previous = existsSync(dir) ? readRecipeJson(dir) : null; + if (previous) { + version = (Number.isInteger(previous.version) ? previous.version : 1) + 1; + const archive = `${dir}@v${previous.version ?? 1}`; + rmSync(archive, { recursive: true, force: true }); + renameSync(dir, archive); + } + mkdirSync(dir, { recursive: true }); + + const prefs = mergedPreferences(root); + const recipe = { + version, + name: slug, + workflow: resolvedWorkflow, + approved_at: new Date().toISOString(), + source_project: basename(root), + destination: prefValue(prefs, "destination"), + aspect: prefValue(prefs, "aspect"), + language: prefValue(prefs, "language"), + voice: prefValue(prefs, "voice"), + // The bare-key fallback tolerates records made before the store required + // style_preset to be workflow-scoped. + style_preset: + prefValue(prefs, `style_preset.${resolvedWorkflow}`) ?? prefValue(prefs, "style_preset"), + blocks: Array.isArray(blocks) && blocks.length > 0 ? blocks : undefined, + }; + + writeFileSync(join(dir, "recipe.json"), `${JSON.stringify(recipe, null, 2)}\n`); + cpSync(frameSpec, join(dir, "frame.md")); + writeFileSync( + join(dir, "storyboard-skeleton.md"), + `${skeletonizeStoryboard(readFileSync(storyboard, "utf8")).trimEnd()}\n`, + ); + + // Best-effort fourth artifact — projects briefed before BRIEF.md existed + // (or by workflows that don't write one) freeze fine without it. + const brief = join(root, "BRIEF.md"); + const briefSkeleton = existsSync(brief); + if (briefSkeleton) { + writeFileSync( + join(dir, "brief-skeleton.md"), + `${skeletonizeBrief(readFileSync(brief, "utf8")).trimEnd()}\n`, + ); + } + + const id = nextId(root, "recipe"); + appendRecord(root, { + id, + type: "recipe", + path: `.media/recipes/${slug}/recipe.json`, + entity: slug, + description: `recipe: ${slug} (${recipe.workflow}, v${version})`, + provenance: { provider: "recipe.freeze", version, source_project: recipe.source_project }, + }); + regenerateIndex(root); + + // User tier — best-effort, like every other promotion. + try { + const userDir = join(userRecipesDir(), slug); + mkdirSync(userDir, { recursive: true }); + cpSync(dir, userDir, { recursive: true, force: true }); + } catch { + // The project-tier freeze already landed. + } + + return { + id, + slug, + version, + dir, + briefSkeleton, + workflow: resolvedWorkflow, + workflowOverridden: Boolean(fromBrief && fromFlag && fromBrief !== fromFlag), + }; +} + +function scanRecipesDir(dir, source) { + if (!existsSync(dir)) return []; + const found = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (!entry.isDirectory() || entry.name.includes("@v")) continue; + const recipe = readRecipeJson(join(dir, entry.name)); + if (recipe) found.push({ ...recipe, source, dir: join(dir, entry.name) }); + } + return found; +} + +/** Two-tier merged listing (project wins), newest approval first. */ +export function listRecipes({ projectDir, workflow }) { + const merged = new Map(); + for (const recipe of scanRecipesDir(userRecipesDir(), "user")) merged.set(recipe.name, recipe); + for (const recipe of scanRecipesDir(projectRecipesDir(resolve(projectDir)), "project")) { + merged.set(recipe.name, recipe); + } + let list = [...merged.values()]; + if (workflow) list = list.filter((r) => r.workflow === workflow); + return list.sort((a, b) => + String(b.approved_at ?? "").localeCompare(String(a.approved_at ?? "")), + ); +} + +/** + * Adopt a recipe into the current project: import the folder from the user + * tier when the project doesn't have it, copy its frame.md over the project's, + * and hand back the values + the skeleton path for the storyboard draft. + */ +export function useRecipe({ projectDir, name }) { + const slug = slugifyRecipeName(name); + const root = resolve(projectDir); + let dir = join(projectRecipesDir(root), slug); + + if (!readRecipeJson(dir)) { + const userDir = join(userRecipesDir(), slug); + if (!readRecipeJson(userDir)) { + const known = listRecipes({ projectDir: root }).map((r) => r.name); + throw new Error( + `no recipe named "${slug}"${known.length ? ` (known: ${known.join(", ")})` : ""}`, + ); + } + mkdirSync(dir, { recursive: true }); + cpSync(userDir, dir, { recursive: true, force: true }); + const imported = readRecipeJson(dir); + appendRecord(root, { + id: nextId(root, "recipe"), + type: "recipe", + path: `.media/recipes/${slug}/recipe.json`, + entity: slug, + description: `recipe: ${slug} (${imported.workflow}, v${imported.version})`, + provenance: { provider: "recipe.local", imported_from: "user-tier" }, + }); + regenerateIndex(root); + } + + const recipe = readRecipeJson(dir); + cpSync(join(dir, "frame.md"), join(root, "frame.md")); + return { + recipe, + dir, + frameSpecPath: "frame.md", + skeletonPath: `.media/recipes/${slug}/storyboard-skeleton.md`, + // Recipes frozen before BRIEF.md existed have no brief skeleton — degrade. + briefSkeletonPath: existsSync(join(dir, "brief-skeleton.md")) + ? `.media/recipes/${slug}/brief-skeleton.md` + : undefined, + }; +} diff --git a/skills/media-use/scripts/lib/recipe-store.test.mjs b/packages/cli/src/media-use/lib/recipe-store.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/recipe-store.test.mjs rename to packages/cli/src/media-use/lib/recipe-store.test.mjs diff --git a/skills/media-use/scripts/lib/registry.mjs b/packages/cli/src/media-use/lib/registry.mjs similarity index 100% rename from skills/media-use/scripts/lib/registry.mjs rename to packages/cli/src/media-use/lib/registry.mjs index 8faa919209..eb6a0beadd 100644 --- a/skills/media-use/scripts/lib/registry.mjs +++ b/packages/cli/src/media-use/lib/registry.mjs @@ -51,8 +51,8 @@ const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // rem const REGISTRY = { bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })], sfx: [ - N("heygen.audio.sounds", { search: sfxProvider.search }), A("bundled.sfx", { search: bundledSfxProvider.search }), + N("heygen.audio.sounds", { search: sfxProvider.search }), ], image: [ N("heygen.asset.search", { search: imageProvider.search }), diff --git a/skills/media-use/scripts/lib/registry.test.mjs b/packages/cli/src/media-use/lib/registry.test.mjs similarity index 94% rename from skills/media-use/scripts/lib/registry.test.mjs rename to packages/cli/src/media-use/lib/registry.test.mjs index 08f30351c3..e5078dcc84 100644 --- a/skills/media-use/scripts/lib/registry.test.mjs +++ b/packages/cli/src/media-use/lib/registry.test.mjs @@ -32,12 +32,13 @@ test("listTypes exposes the v2 media types", () => { } }); -test("heygen provider is first for every type it serves", () => { - for (const t of ["bgm", "sfx", "image", "icon"]) { +test("heygen provider is first for every type it serves except bundled sfx", () => { + for (const t of ["bgm", "image", "icon"]) { const first = getProviders(t)[0]; assert.ok(first, `no enabled provider for ${t}`); assert.match(first.name, /^heygen/, `${t} first provider is ${first.name}`); } + assert.equal(getProviders("sfx")[0].name, "bundled.sfx"); }); test("sanctioned providers only: heygen, local mflux/kokoro/ltx, codex, design spec, logo tiers", () => { @@ -84,13 +85,13 @@ test("video cascade: HeyGen first, LTX local fallback, generate-only", async () assert.equal(await runCapability("video", "search", "x", {}), null); }); -test("sfx cascade: HeyGen catalog first, bundled library remains the local fallback", () => { +test("sfx cascade: bundled library first, HeyGen catalog remains the network fallback", () => { const ps = getProviders("sfx"); - assert.equal(ps[0].name, "heygen.audio.sounds"); - assert.ok(ps[0].network, "HeyGen SFX catalog is network-only"); - assert.equal(ps[1].name, "bundled.sfx"); - assert.equal(typeof ps[1].search, "function"); - assert.ok(!ps[1].network, "bundled SFX remain available offline"); + assert.equal(ps[0].name, "bundled.sfx"); + assert.equal(typeof ps[0].search, "function"); + assert.ok(!ps[0].network, "bundled SFX remain available offline"); + assert.equal(ps[1].name, "heygen.audio.sounds"); + assert.ok(ps[1].network, "HeyGen SFX catalog is network-only"); }); test("ctx.provider forces one generator (e.g. 'make an image WITH codex')", async () => { diff --git a/skills/media-use/scripts/lib/search.mjs b/packages/cli/src/media-use/lib/search.mjs similarity index 100% rename from skills/media-use/scripts/lib/search.mjs rename to packages/cli/src/media-use/lib/search.mjs diff --git a/skills/media-use/scripts/lib/search.test.mjs b/packages/cli/src/media-use/lib/search.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/search.test.mjs rename to packages/cli/src/media-use/lib/search.test.mjs diff --git a/skills/media-use/scripts/lib/sfx-provider.mjs b/packages/cli/src/media-use/lib/sfx-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/sfx-provider.mjs rename to packages/cli/src/media-use/lib/sfx-provider.mjs diff --git a/skills/media-use/scripts/lib/specs.mjs b/packages/cli/src/media-use/lib/specs.mjs similarity index 100% rename from skills/media-use/scripts/lib/specs.mjs rename to packages/cli/src/media-use/lib/specs.mjs diff --git a/skills/media-use/scripts/lib/specs.test.mjs b/packages/cli/src/media-use/lib/specs.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/specs.test.mjs rename to packages/cli/src/media-use/lib/specs.test.mjs diff --git a/skills/media-use/scripts/lib/stats.mjs b/packages/cli/src/media-use/lib/stats.mjs similarity index 100% rename from skills/media-use/scripts/lib/stats.mjs rename to packages/cli/src/media-use/lib/stats.mjs diff --git a/skills/media-use/scripts/lib/stats.test.mjs b/packages/cli/src/media-use/lib/stats.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/stats.test.mjs rename to packages/cli/src/media-use/lib/stats.test.mjs diff --git a/packages/cli/src/media-use/lib/telemetry.mjs b/packages/cli/src/media-use/lib/telemetry.mjs new file mode 100644 index 0000000000..cd2911f529 --- /dev/null +++ b/packages/cli/src/media-use/lib/telemetry.mjs @@ -0,0 +1,193 @@ +// Usage tracking shares the CLI and Studio identity. Properties stay coarse and +// never carry intent text, file names, or paths. + +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +const POSTHOG_API_KEY = "phc_zjjbX0PnWxERXrMHhkEJWj9A9BhGVLRReICgsfTMmpx"; +const POSTHOG_HOST = "https://us.i.posthog.com"; +const TIMEOUT_MS = 1500; +let identifiedAccount = false; +let warnedNonDefaultHost = false; + +function isTestOrCiContext() { + return ( + process.env.CI === "true" || + process.env.CI === "1" || + process.env.NODE_ENV === "test" || + process.env.NODE_ENV === "development" + ); +} + +function posthogHost() { + const override = process.env.MEDIA_USE_TELEMETRY_HOST; + if (override && !warnedNonDefaultHost && !isTestOrCiContext()) { + warnedNonDefaultHost = true; + console.error( + `media-use: telemetry is redirected to a non-default host via MEDIA_USE_TELEMETRY_HOST (${override}) — unset it unless this is intentional.`, + ); + } + return override || POSTHOG_HOST; +} + +/** True when telemetry must NOT be sent (opt-out envs, CI, dev). */ +export function optedOut() { + return ( + process.env.HYPERFRAMES_NO_TELEMETRY === "1" || + process.env.DO_NOT_TRACK === "1" || + process.env.CI === "true" || + process.env.CI === "1" || + process.env.NODE_ENV === "development" + ); +} + +// Read and write the shared config so media-use keeps one identity per install. +function sharedConfigPath() { + return join(homedir(), ".hyperframes", "config.json"); +} + +function readSharedConfig() { + try { + const file = sharedConfigPath(); + if (existsSync(file)) { + const parsed = JSON.parse(readFileSync(file, "utf8")); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed; + } + } catch { + // unreadable config → treat as empty; never throw + } + return {}; +} + +function writeSharedConfig(config) { + const dir = join(homedir(), ".hyperframes"); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "config.json"), JSON.stringify(config, null, 2) + "\n"); +} + +// Adopt a pre-existing media-use-only id (~/.media/anon-id from before this +// change) so upgraders keep their PostHog persona instead of resetting to a new +// one — otherwise cross-surface continuity would start over on upgrade. +function legacyMediaAnonId() { + try { + const file = join(homedir(), ".media", "anon-id"); + if (existsSync(file)) { + const id = readFileSync(file, "utf8").trim(); + if (id) return id; + } + } catch { + // ignore + } + return null; +} + +// Stable per-machine id from the shared config; seeds it (adopting a legacy +// media-use id when present) if absent. +function anonymousId() { + try { + const config = readSharedConfig(); + if (typeof config.anonymousId === "string" && config.anonymousId.trim()) { + return config.anonymousId.trim(); + } + const id = legacyMediaAnonId() || randomUUID(); + writeSharedConfig({ ...config, anonymousId: id }); + return id; + } catch { + return "anon"; // best-effort; a shared bucket is fine if the fs is read-only + } +} + +function heygenAccountDistinctId() { + const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials"); + try { + if (!existsSync(file)) return null; + const raw = readFileSync(file, "utf8").trim(); + if (!raw.startsWith("{")) return null; + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; + const user = parsed.user; + if (!user || typeof user !== "object" || Array.isArray(user)) return null; + const id = typeof user.email === "string" && user.email.trim() ? user.email : user.username; + // Lowercased so this joins with the CLI's own identify call regardless of + // the account's stored email casing — two different-case distinct ids + // would otherwise split one person across two PostHog profiles. + return typeof id === "string" && id.trim() ? id.trim().toLowerCase() : null; + } catch { + return null; + } +} + +function showTelemetryNotice() { + if (optedOut()) return; + try { + const config = readSharedConfig(); + // Shared with the CLI (config.telemetryNoticeShown): shown once per person + // across surfaces, not once per tool. + if (config.telemetryNoticeShown === true) return; + console.error( + [ + "media-use sends usage telemetry: media type, resolution source, and provider; never intent text, file names, or paths.", + "If you sign in to HeyGen, usage links to your account email or username. Opt out with HYPERFRAMES_NO_TELEMETRY=1 or DO_NOT_TRACK=1.", + ].join("\n"), + ); + writeSharedConfig({ ...config, telemetryNoticeShown: true }); + } catch { + // notice is best-effort; never surface into the command + } +} + +async function postBatch(batch) { + try { + await fetch(`${posthogHost()}/batch/`, { + method: "POST", + headers: { "Content-Type": "application/json", Connection: "close" }, + body: JSON.stringify({ api_key: POSTHOG_API_KEY, batch }), + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + } catch { + // telemetry is best-effort; never surface into the command + } +} + +async function postEvent(event, properties, distinctId) { + await postBatch([ + { + event, + properties: { ...properties, surface: "media-use", $ip: null }, + distinct_id: distinctId, + timestamp: new Date().toISOString(), + }, + ]); +} + +async function identifyAccount(anonId) { + if (optedOut() || identifiedAccount) return; + const distinctId = heygenAccountDistinctId(); + if (!distinctId) return; + identifiedAccount = true; + await postEvent("$identify", { $anon_distinct_id: anonId }, distinctId); +} + +/** + * Fire-and-forget a single event to PostHog. Best-effort: awaited with a short + * timeout so a short-lived script flushes before exit, but any failure (offline, + * opted out) is swallowed. `properties` must be non-PII (no intent/paths). + */ +export async function track(event, properties = {}) { + if (optedOut()) return; + showTelemetryNotice(); + const anonId = anonymousId(); + await identifyAccount(anonId); + await postEvent(event, properties, anonId); +} + +export function __anonymousIdForTest() { + return anonymousId(); +} + +export function __resetTelemetryForTest() { + identifiedAccount = false; + warnedNonDefaultHost = false; +} diff --git a/skills/media-use/scripts/lib/telemetry.test.mjs b/packages/cli/src/media-use/lib/telemetry.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/telemetry.test.mjs rename to packages/cli/src/media-use/lib/telemetry.test.mjs diff --git a/skills/media-use/scripts/lib/tts-local-provider.mjs b/packages/cli/src/media-use/lib/tts-local-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/tts-local-provider.mjs rename to packages/cli/src/media-use/lib/tts-local-provider.mjs diff --git a/skills/media-use/scripts/lib/tts-local-provider.test.mjs b/packages/cli/src/media-use/lib/tts-local-provider.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/tts-local-provider.test.mjs rename to packages/cli/src/media-use/lib/tts-local-provider.test.mjs diff --git a/skills/media-use/scripts/lib/usage.mjs b/packages/cli/src/media-use/lib/usage.mjs similarity index 100% rename from skills/media-use/scripts/lib/usage.mjs rename to packages/cli/src/media-use/lib/usage.mjs diff --git a/skills/media-use/scripts/lib/usage.test.mjs b/packages/cli/src/media-use/lib/usage.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/usage.test.mjs rename to packages/cli/src/media-use/lib/usage.test.mjs diff --git a/skills/media-use/scripts/lib/voice-provider.mjs b/packages/cli/src/media-use/lib/voice-provider.mjs similarity index 100% rename from skills/media-use/scripts/lib/voice-provider.mjs rename to packages/cli/src/media-use/lib/voice-provider.mjs diff --git a/skills/media-use/scripts/lib/voice-provider.test.mjs b/packages/cli/src/media-use/lib/voice-provider.test.mjs similarity index 100% rename from skills/media-use/scripts/lib/voice-provider.test.mjs rename to packages/cli/src/media-use/lib/voice-provider.test.mjs diff --git a/packages/cli/src/media-use/lib/words.mjs b/packages/cli/src/media-use/lib/words.mjs new file mode 100644 index 0000000000..114cf766ce --- /dev/null +++ b/packages/cli/src/media-use/lib/words.mjs @@ -0,0 +1,18 @@ +export function normalizeWords(input) { + const raw = Array.isArray(input) ? input : Array.isArray(input?.words) ? input.words : []; + return raw + .map((w, index) => { + const text = String(w?.text ?? w?.word ?? "").trim(); + const start = Number(w?.start); + const end = Number(w?.end); + if (!text || !Number.isFinite(start) || !Number.isFinite(end)) return null; + return { id: w?.id ?? `w${index}`, text, start, end }; + }) + .filter(Boolean); +} + +export function wordListsFromMediaMeta(input) { + if (Array.isArray(input) || Array.isArray(input?.words)) return [normalizeWords(input)]; + if (!Array.isArray(input?.voices)) return []; + return input.voices.map((voice) => normalizeWords(voice)).filter((words) => words.length > 0); +} diff --git a/packages/cli/src/media-use/resolve.mjs b/packages/cli/src/media-use/resolve.mjs new file mode 100644 index 0000000000..cc2a4fc955 --- /dev/null +++ b/packages/cli/src/media-use/resolve.mjs @@ -0,0 +1,1287 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { existsSync, statSync, writeFileSync, renameSync, rmSync } from "node:fs"; +import { resolve, join, extname, basename } from "node:path"; +import { parseArgs } from "node:util"; +import { + appendRecord, + findByPrompt, + findByEntity, + nextId, + withReservedFile, + withReservedFileSync, +} from "./lib/manifest.mjs"; +import { regenerateIndex } from "./lib/index-gen.mjs"; +import { cacheGet, cacheGetByEntity, importFromCache, cachePut } from "./lib/cache.mjs"; +import { + runCapability, + listTypes, + providerMatches, + providerNamesFor, + providerTierFor, +} from "./lib/registry.mjs"; +import { freezeUrl, freezeLocalFile, isDirectMediaUrl } from "./lib/freeze.mjs"; +import { findExistingAsset } from "./lib/adopt.mjs"; +import { track } from "./lib/telemetry.mjs"; +import { recordMiss } from "./lib/misses.mjs"; +import { buildStats } from "./lib/stats.mjs"; +import { typesMatch } from "./lib/match.mjs"; +import { listCandidates, formatCandidates, CANDIDATE_CAP } from "./lib/candidates.mjs"; +import { findGlobalBySha } from "./lib/cache.mjs"; +import { heygenAuthMethod } from "../audio/scripts/lib/heygen.mjs"; +import { buildCube, paramsFromIntent } from "./lib/cube-build.mjs"; +import { validateCubeFile } from "./lib/cube-validate.mjs"; +import { analyzeMediaGrade, formatMeasuredNote } from "./lib/grade-analyzer.mjs"; +import { + freezeLibraryLut, + isLibraryLutOfflineMiss, + matchColorLook, +} from "./lib/lut-preset-provider.mjs"; +import { + HEYGEN_AUTH_COMMAND, + HEYGEN_INSTALL_COMMAND, + HEYGEN_MIN_VERSION, + HEYGEN_UPDATE_COMMAND, + consumeHeygenRemediation, + firstSemver, + flushHeygenFailureTracking, + versionLessThan, +} from "./lib/heygen-cli.mjs"; +import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs"; +import { + fetchMediaVectors, + mediaVectorRows, + rankMediaRowsWithVectors, +} from "./lib/local-media-search.mjs"; + +const INGEST_TYPES = listTypes(); +const DEFAULT_EXT = { + bgm: ".wav", + sfx: ".mp3", + voice: ".wav", + image: ".jpg", + icon: ".svg", + logo: ".svg", + brand: ".png", + video: ".mp4", + grade: ".cube", + lut: ".cube", +}; + +// resolve shells `fetch`/`freezeUrl` and modern ESM; 18 is the floor where those +// exist without flags. Named so the --doctor node check verifies something real +// (O2). Declared before the top-level `--doctor` branch that calls runDoctor(). +const MIN_NODE_VERSION = "18.0.0"; + +const { values: args } = parseArgs({ + options: { + type: { type: "string", short: "t" }, + intent: { type: "string", short: "i" }, + entity: { type: "string", short: "e" }, + project: { type: "string", short: "p", default: "." }, + adopt: { type: "boolean", default: false }, + candidates: { type: "boolean", default: false }, + doctor: { type: "boolean", default: false }, + stats: { type: "boolean", default: false }, + days: { type: "string" }, + "dry-run": { type: "boolean", default: false }, + reuse: { type: "string" }, + from: { type: "string" }, + params: { type: "string" }, + for: { type: "string" }, + analyze: { type: "boolean", default: false }, + "local-only": { type: "boolean", default: false }, + provider: { type: "string" }, + "avatar-id": { type: "string" }, + "voice-id": { type: "string" }, + json: { type: "boolean", default: false }, + help: { type: "boolean", short: "h", default: false }, + }, + strict: true, +}); + +if (args.help) { + console.log(`media-use resolve — turn a media need into a frozen local file + +Usage: + node resolve.mjs --type --intent "" [--project ] + +Types: ${listTypes().join(", ")} + +Options: + --type, -t Media type (required) + --intent, -i What you need (required) + --entity, -e Entity name for cache matching (optional) + --project, -p Project directory (default: .) + --adopt Adopt all existing assets/ files into the manifest + --candidates List reusable assets (project + global cache) for --type; no + download, no mutation. Read them and decide reuse yourself. + --doctor Check local CLI dependencies; no manifest changes. + --stats Print local usage stats from .media and ~/.media; no mutation. + --days Limit --stats to records/misses from the last N days when + timestamps are available. + --reuse Import a specific global-cache asset (by content sha/prefix, + from --candidates) into this project + --from Freeze a local file or direct public URL (ingest) + --params Build an explicit parametric LUT (lut/grade only) + --for Analyze a local image/video and add measured grade adjust + suggestions (grade only) + --analyze Return --for grade evidence without recording a candidate + --local-only Offline: skip every network provider + --provider Force one generator (e.g. codex, mflux, kokoro, heygen) + --avatar-id Override the default avatar for heygen.video generation + --voice-id Override the default voice for voice/heygen.video generation + --json Output JSON instead of one-line result + --help, -h Show this help`); + process.exit(0); +} + +const projectDir = resolve(args.project); +const type = args.type; +const intent = args.intent; +const entity = args.entity || null; + +if (args.adopt) { + const { adoptExistingAssets } = await import("./lib/adopt.mjs"); + const adopted = adoptExistingAssets(projectDir); + if (args.json) { + console.log(JSON.stringify({ ok: true, adopted: adopted.length, assets: adopted })); + } else if (adopted.length === 0) { + console.log("no new assets to adopt (assets/ empty or already registered)"); + } else { + console.log(`adopted ${adopted.length} asset${adopted.length === 1 ? "" : "s"} from assets/`); + for (const r of adopted) console.log(` ${r.id} → ${r.path} (${r.type})`); + } + process.exit(0); +} + +// Candidates: side-effect-free listing of reusable assets (project + global +// cache) for --type. No download, no provider, no mutation. The agent reads +// these and decides semantic fit itself. +if (args.candidates || args["dry-run"]) { + await showCandidates(); + process.exit(0); +} + +if (args.doctor) { + const doctor = runDoctor(); + const failed = doctor.checks.filter((check) => !check.ok); + // Non-PII: instrument the exact question the feature exists to answer — how + // often is --doctor run and which check fails most. Awaited so a short-lived + // run flushes before exit. + await track("media_use_doctor_run", { + ok: doctor.ok, + checks_failed: failed.length, + failed: failed.map((check) => check.name), + }); + if (args.json) { + console.log(JSON.stringify({ ok: doctor.ok, checks: doctor.checks })); + } else { + printDoctor(doctor.checks); + } + process.exit(doctor.ok ? 0 : 1); +} + +if (args.stats) { + const report = buildStats({ + projectDir, + days: args.days ? Number(args.days) : undefined, + }); + if (args.json) { + console.log(JSON.stringify(report)); + } else { + printStats(report); + } + process.exit(0); +} + +// Reuse a global-cache asset selected by content sha or prefix. +if (args.reuse !== undefined) { + await reuseGlobal(args.reuse); + process.exit(0); +} + +// Ingest: freeze a user-supplied local file or direct public URL (no search). +if (args.from) { + await ingest(args.from); + process.exit(0); +} + +if (args.analyze) { + if (type !== "grade" || !args.for) { + console.error("error: --analyze requires --type grade and --for "); + process.exit(2); + } + const mediaPath = resolve(args.for); + if (!existsSync(mediaPath)) { + console.error(`error: --for file not found: ${mediaPath}`); + process.exit(2); + } + const analysis = analyzeMediaGrade(mediaPath); + if (args.json) { + console.log(JSON.stringify({ ok: true, type: "grade-analysis", ...analysis })); + } else { + console.log(formatMeasuredNote(mediaPath, analysis.measured)); + console.log(`suggested adjust: ${JSON.stringify(analysis.adjust)}`); + } + process.exit(0); +} + +// Resolve named recipe bundles without a provider. +if (type === "recipe") { + const { useRecipe } = await import("./lib/recipe-store.mjs"); + const name = (entity || intent || "").trim(); + if (!name) exitError("--type recipe needs --entity (or --intent )", 2); + try { + const used = useRecipe({ projectDir, name }); + if (args.json) { + console.log(JSON.stringify({ ok: true, ...used })); + } else { + console.log( + `resolved recipe ${used.recipe.name} (v${used.recipe.version}, ${used.recipe.workflow})`, + ); + console.log(` frame spec → ${used.frameSpecPath} (copied over)`); + console.log(` storyboard skeleton → ${used.skeletonPath}`); + if (used.briefSkeletonPath) console.log(` brief skeleton → ${used.briefSkeletonPath}`); + } + process.exit(0); + } catch (err) { + exitError(err.message, 1); + } +} + +if (args.params !== undefined) { + if (type !== "lut" && type !== "grade") { + exitError( + type + ? `--params only supports --type lut or grade (got ${type})` + : "--params requires --type lut or grade", + 2, + ); + } + try { + await runParams(); + process.exit(0); + } catch (err) { + exitError(err.message, 1); + } +} + +if (!args.type || !args.intent || !args.intent.trim()) { + console.error("error: --type and a non-empty --intent are required"); + process.exit(2); +} + +if (!listTypes().includes(args.type)) { + console.error(`error: unknown media type: ${args.type} (known: ${listTypes().join(", ")})`); + process.exit(2); +} + +// Forced-provider validation: reject an unknown/unavailable provider name up +// front so a typo reads as a typo, not a catalog miss (`no provider could +// resolve`). Match rule mirrors runProviders (full name or dotted prefix). +if (args.provider && !providerMatches(args.type, args.provider)) { + console.error( + `error: unknown provider "${args.provider}" for type ${args.type} (available: ${providerNamesFor(args.type).join(", ")})`, + ); + process.exit(2); +} + +function recordAvailable(projectDir, record) { + if (!record) return false; + if (record.path) return existsSync(join(projectDir, record.path)); + return record.type === "grade" && record.grading; +} + +// Sparse `{ authMethod }` for a heygen-family provider name (e.g. "heygen.tts"), +// else `{}` — keeps auth_method telemetry absent for every non-heygen resolve +// instead of implying an auth method that doesn't apply. +function heygenAuthMethodFor(provider) { + if (!provider || !provider.startsWith("heygen.")) return {}; + const authMethod = heygenAuthMethod(); + return authMethod ? { authMethod } : {}; +} + +function localizeImportedRecord(record, localPath) { + if (record?.type === "grade" && record.grading?.lut) { + record.grading = { + ...record.grading, + lut: { ...record.grading.lut, src: localPath }, + }; + } + return record; +} + +async function run() { + // A forced provider bypasses reuse and pins the provider cascade. + const forced = !!args.provider; + + // 1. project manifest — exact-prompt match + const projectHit = forced ? null : findByPrompt(projectDir, intent, type); + if (recordAvailable(projectDir, projectHit)) { + return result(projectHit, "cached"); + } + + // 1b. entity match in project. icon and image are interchangeable for + // entity hits — both live in images/, and figma-imported brand marks are + // always recorded as type image while agents ask for logos as type icon. + if (!forced && entity) { + const entityHit = findByEntity(projectDir, entity); + if (entityHit && typesMatch(entityHit.type, type) && recordAvailable(projectDir, entityHit)) { + return result(entityHit, "cached"); + } + } + + // 1c. scan existing assets/ directory for unregistered matches + const existingAsset = + forced || type === "grade" || type === "lut" + ? null + : findExistingAsset(projectDir, intent, type); + if (existingAsset) { + const id = nextId(projectDir, type); + const record = { + id, + type: existingAsset.type, + path: existingAsset.relativePath, + source: "existing", + description: existingAsset.name.replace(/[-_]/g, " "), + provenance: { provider: "local", adopted: true, prompt: intent }, + }; + appendRecord(projectDir, record); + regenerateIndex(projectDir); + return result(record, "existing"); + } + + // 2. global cache — exact-prompt or entity match + const cacheHit = forced ? null : cacheGet(intent, type); + if (cacheHit) { + const ext = extname(cacheHit.cached_path); + const imported = withReservedFileSync(projectDir, type, ext, ({ id, localPath }) => + localizeImportedRecord(importFromCache(cacheHit, projectDir, id, localPath), localPath), + ); + if (imported) { + appendRecord(projectDir, imported); + regenerateIndex(projectDir); + return result(imported, "reused"); + } + } + + if (!forced && entity) { + const entityCacheHit = cacheGetByEntity(entity); + if (entityCacheHit && typesMatch(entityCacheHit.type, type)) { + const ext = extname(entityCacheHit.cached_path); + const imported = withReservedFileSync(projectDir, type, ext, ({ id, localPath }) => + localizeImportedRecord( + importFromCache(entityCacheHit, projectDir, id, localPath), + localPath, + ), + ); + if (imported) { + appendRecord(projectDir, imported); + regenerateIndex(projectDir); + return result(imported, "reused"); + } + } + } + + // --local-only skips every remote provider. + const localOnly = args["local-only"]; + const ctx = { + entity, + projectDir, + localOnly, + provider: args.provider, + avatarId: args["avatar-id"], + voiceId: args["voice-id"], + }; + + // Suggest candidates before a new fetch. + try { + const { similar } = listCandidates({ projectDir, type, intent, cap: CANDIDATE_CAP }); + if (similar > 0) { + console.error( + `media-use: ${similar} similar cached asset${similar === 1 ? "" : "s"} already ${similar === 1 ? "exists" : "exist"} — run \`resolve --candidates --type ${type} --intent "${intent}"\` to review and reuse instead of fetching.`, + ); + } + } catch { + // hint is best-effort; never block a resolve + } + + if (type === "grade" || type === "lut") { + return resolveColor(type, intent, { projectDir }); + } + + // SFX search is bundled, local-index, then HeyGen. + let searchResult = null; + let providerFailure = null; + let localIndexFailure = null; + try { + if (type === "sfx" && !args.provider) { + searchResult = await runCapability(type, "search", intent, { + ...ctx, + provider: "bundled.sfx", + }); + if (!searchResult && !localOnly) { + const registry = + process.env.HYPERFRAMES_REGISTRY || + "https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry"; + try { + await fetchMediaVectors(registry); + const ranked = await rankMediaRowsWithVectors( + intent, + mediaVectorRows().filter((row) => row.kind === "sfx"), + ); + const row = ranked.rows[0]; + const candidates = row + ? [resolve(row.file), join(import.meta.dirname, "..", "..", "..", row.file)] + : []; + const localPath = candidates.find((candidate) => existsSync(candidate)); + if (row && localPath) { + searchResult = { + localPath, + ext: extname(localPath), + source: "local-index", + metadata: { + description: row.description, + duration: row.duration ?? null, + provider: "catalog.local", + provenance: { library_key: row.id, tier: ranked.tier }, + }, + }; + } + } catch (error) { + localIndexFailure = error; + } + } + if (!searchResult) searchResult = await runCapability(type, "search", intent, ctx); + // Keep HeyGen remediation diagnostics while bundled remains authoritative. + else if (!localOnly) + await runCapability(type, "search", intent, { + ...ctx, + provider: "heygen.audio.sounds", + }); + } else { + searchResult = await runCapability(type, "search", intent, ctx); + } + } catch (error) { + providerFailure = error; + // search failed, try generate + } + + if (localIndexFailure) { + throw new Error(`local SFX index unavailable: ${localIndexFailure.message}`, { + cause: localIndexFailure, + }); + } + + // 4. generate fallback — same ordered cascade for the generate capability + if (!searchResult) { + try { + searchResult = await runCapability(type, "generate", intent, ctx); + } catch (error) { + providerFailure ??= error; + // generate failed too + } + } + + // Flush provider failure telemetry before the process can exit. + await flushHeygenFailureTracking(); + + if (!searchResult) { + await track("media_use_resolve_miss", { + type, + local_only: !!localOnly, + provider_override: !!args.provider, + }); + recordMiss({ + type, + intent, + provider_override: !!args.provider, + local_only: !!args["local-only"], + }); + // brand stays local: no frame.md/design.md -> upsell the HyperFrames design + // flow rather than reporting a generic miss (B5). + const msg = + providerFailure instanceof BundledSfxAssetsError + ? providerFailure.message + : type === "brand" + ? "no brand spec found — add a frame.md or design.md (colors/font/logo) to this project. Run the HyperFrames design flow to create one; brand tokens are read locally for deterministic rendering." + : args.provider + ? `provider "${args.provider}" could not resolve ${type}: "${intent}"${localOnly ? " (--local-only skips network providers; drop it or the --provider override)" : ""}` + : `no provider could resolve ${type}: "${intent}"`; + if (args.json) { + console.log( + JSON.stringify({ + ok: false, + ...(providerFailure instanceof BundledSfxAssetsError + ? { code: providerFailure.code, fix: providerFailure.fix } + : {}), + error: msg, + }), + ); + } else { + console.error(`error: ${msg}`); + } + process.exit(1); + } + + // 5. freeze and register atomically. + const ext = searchResult.ext || extFromUrl(searchResult.url || "") || defaultExt(type); + const { id, localPath, fullPath } = await withReservedFile( + projectDir, + type, + ext, + async (reservation) => { + if (searchResult.localPath) { + freezeLocalFile(searchResult.localPath, reservation.fullPath); + } else if (searchResult.url) { + await freezeUrl(searchResult.url, reservation.fullPath); + } else { + throw new Error("provider returned no url or localPath"); + } + return reservation; + }, + ); + + const record = { + id, + type, + path: localPath, + source: searchResult.source || "search", + description: searchResult.metadata?.description || intent, + ...(searchResult.metadata?.duration != null && { + duration: Math.round(searchResult.metadata.duration * 10) / 10, + }), + ...(searchResult.metadata?.width != null && { width: searchResult.metadata.width }), + ...(searchResult.metadata?.height != null && { height: searchResult.metadata.height }), + ...(searchResult.metadata?.transparent != null && { + transparent: searchResult.metadata.transparent, + }), + ...(entity && { entity }), + provenance: { + provider: searchResult.metadata?.provider || "unknown", + prompt: intent, + // Keep auth method sparse for non-HeyGen providers. + ...heygenAuthMethodFor(searchResult.metadata?.provider), + ...searchResult.metadata?.provenance, + }, + }; + + const heygenRemediation = consumeHeygenRemediation(); + if ( + searchResult.metadata?.provider === "bundled.sfx" && + !localOnly && + !args.provider && + heygenRemediation + ) { + record.advisory = heygenRemediation; + } + + appendRecord(projectDir, record); + regenerateIndex(projectDir); + // Promote fetched assets into the global cache for reuse. + try { + cachePut(fullPath, record); + } catch { + // promotion is best-effort; a resolve still succeeds locally + } + return result(record, searchResult.source || "search"); +} + +function mergeSmartAdjust(block) { + if (!args.for) return block; + const mediaPath = resolve(args.for); + // Clear upfront error beats an ffmpeg "No such file" stack on a typo'd path. + if (!existsSync(mediaPath)) throw new Error(`--for file not found: ${mediaPath}`); + const analysis = analyzeMediaGrade(mediaPath); + console.error(formatMeasuredNote(mediaPath, analysis.measured)); + return { + ...block, + adjust: { + ...(block.adjust || {}), + ...analysis.adjust, + }, + }; +} + +function freezeGeneratedLut( + params, + { + projectDir, + type, + description = "parametric color grade", + validationErrorPrefix = "generated LUT failed validation", + }, +) { + return withReservedFileSync(projectDir, type, ".cube", ({ id, localPath, fullPath }) => { + const tmpPath = `${fullPath}.tmp`; + try { + // Write + validate at .tmp, then atomic rename, so a crash between write and + // validate can't leave an invalid .cube at the final path. + writeFileSync(tmpPath, buildCube(params)); + const check = validateCubeFile(tmpPath); + if (!check.ok) throw new Error(check.error); + renameSync(tmpPath, fullPath); + } catch (err) { + rmSync(tmpPath, { force: true }); + throw new Error(`${validationErrorPrefix}: ${err.message}`); + } + return { + id, + localPath, + fullPath, + lut: { src: localPath, intensity: 1 }, + source: "generated", + description, + metadata: { + provider: "cube_lut.builder", + provenance: { params }, + }, + }; + }); +} + +function exitError(message, status = 1) { + if (args.json) { + console.log(JSON.stringify({ ok: false, error: message })); + } else { + console.error(`error: ${message}`); + } + process.exit(status); +} + +function parseExplicitParams() { + try { + return JSON.parse(args.params); + } catch (err) { + throw new Error(`invalid --params JSON: ${err.message}`); + } +} + +async function runParams() { + if (type === "lut" && args.for) { + throw new Error("--for is only supported with --type grade"); + } + const params = parseExplicitParams(); + const description = + typeof intent === "string" && intent.trim() + ? intent.trim() + : `custom parametric ${type === "lut" ? "lut" : "grade"}`; + const frozen = freezeGeneratedLut(params, { + projectDir, + type, + description, + validationErrorPrefix: "--params produced an invalid LUT", + }); + const record = { + id: frozen.id, + type, + path: frozen.localPath, + source: frozen.source, + description: frozen.description, + ...(type === "grade" && { grading: mergeSmartAdjust({ intensity: 1, lut: frozen.lut }) }), + provenance: { + provider: frozen.metadata.provider, + ...frozen.metadata.provenance, + }, + }; + return finalizeColorRecord(record, frozen.source, frozen.fullPath); +} + +async function finalizeColorRecord(record, source, fullPath = null) { + appendRecord(projectDir, record); + regenerateIndex(projectDir); + if (fullPath) { + try { + cachePut(fullPath, record); + } catch { + // promotion is best-effort + } + } + return result(record, source); +} + +async function colorMiss(type, intent) { + await track("media_use_resolve_miss", { + type, + local_only: !!args["local-only"], + provider_override: !!args.provider, + }); + recordMiss({ + type, + intent, + provider_override: !!args.provider, + local_only: !!args["local-only"], + }); + const msg = `no local color grade could resolve ${type}: "${intent}"`; + if (args.json) { + console.log(JSON.stringify({ ok: false, error: msg })); + } else { + console.error(`error: ${msg}`); + } + process.exit(1); +} + +async function resolveGrade(intent, { projectDir }) { + const match = matchColorLook(intent); + if (match?.kind === "preset") { + const id = nextId(projectDir, "grade"); + const grading = mergeSmartAdjust({ preset: match.preset, intensity: 1 }); + const record = { + id, + type: "grade", + source: "preset", + description: intent, + grading, + provenance: { + provider: "color_grade.local", + prompt: intent, + preset: match.preset, + }, + }; + return finalizeColorRecord(record, "preset"); + } + + if (match?.kind === "library") { + let frozen; + try { + frozen = await freezeLibraryLut(match, { + projectDir, + type: "grade", + localOnly: args["local-only"], + }); + } catch (err) { + if (isLibraryLutOfflineMiss(err)) return colorMiss("grade", intent); + throw err; + } + const grading = mergeSmartAdjust({ intensity: 1, lut: frozen.lut }); + const record = { + id: frozen.id, + type: "grade", + path: frozen.localPath, + source: frozen.source, + description: frozen.description, + grading, + provenance: { + provider: frozen.metadata.provider, + prompt: intent, + ...frozen.metadata.provenance, + }, + }; + return finalizeColorRecord(record, frozen.source, frozen.fullPath); + } + + const params = paramsFromIntent(intent); + if (!params) { + // No creative look matched. With --for, the measured adjust block is a + // valid grade on its own (footage auto-correction); only a true miss + // (no look AND no analysis) aborts. + if (args.for) { + const grading = mergeSmartAdjust({ intensity: 1 }); + const record = { + id: nextId(projectDir, "grade"), + type: "grade", + source: "measured", + description: intent, + grading, + provenance: { provider: "color_grade.local", prompt: intent, measured: true }, + }; + return finalizeColorRecord(record, "measured"); + } + return colorMiss("grade", intent); + } + const frozen = freezeGeneratedLut(params, { projectDir, type: "grade" }); + const grading = mergeSmartAdjust({ intensity: 1, lut: frozen.lut }); + const record = { + id: frozen.id, + type: "grade", + path: frozen.localPath, + source: frozen.source, + description: intent, + grading, + provenance: { + provider: frozen.metadata.provider, + prompt: intent, + ...frozen.metadata.provenance, + }, + }; + return finalizeColorRecord(record, frozen.source, frozen.fullPath); +} + +async function resolveLut(intent, { projectDir }) { + if (args.for) { + throw new Error("--for is only supported with --type grade"); + } + const match = matchColorLook(intent); + if (match?.kind === "library") { + let frozen; + try { + frozen = await freezeLibraryLut(match, { + projectDir, + type: "lut", + localOnly: args["local-only"], + }); + } catch (err) { + if (isLibraryLutOfflineMiss(err)) return colorMiss("lut", intent); + throw err; + } + const record = { + id: frozen.id, + type: "lut", + path: frozen.localPath, + source: frozen.source, + description: frozen.description, + provenance: { + provider: frozen.metadata.provider, + prompt: intent, + ...frozen.metadata.provenance, + }, + }; + return finalizeColorRecord(record, frozen.source, frozen.fullPath); + } + + const params = paramsFromIntent(intent); + if (!params) return colorMiss("lut", intent); + const frozen = freezeGeneratedLut(params, { projectDir, type: "lut" }); + const record = { + id: frozen.id, + type: "lut", + path: frozen.localPath, + source: frozen.source, + description: intent, + provenance: { + provider: frozen.metadata.provider, + prompt: intent, + ...frozen.metadata.provenance, + }, + }; + return finalizeColorRecord(record, frozen.source, frozen.fullPath); +} + +async function resolveColor(type, intent, options) { + if (type === "grade") return resolveGrade(intent, options); + return resolveLut(intent, options); +} + +async function ingest(src) { + if (!type || !INGEST_TYPES.includes(type)) { + console.error(`error: --from requires --type (one of: ${INGEST_TYPES.join(", ")})`); + process.exit(2); + } + const isUrl = /^https?:\/\//i.test(src); + if (isUrl && !isDirectMediaUrl(src)) { + console.error( + `error: --from takes a direct public media URL or a local file; "${src}" is not a direct media link (no platform pages / yt-dlp)`, + ); + process.exit(2); + } + if (!isUrl && !existsSync(resolve(src))) { + console.error(`error: file not found: ${src}`); + process.exit(2); + } + // Refuse 0-byte input: an empty asset would register clean but fail at render + // (freezeUrl already rejects empty responses; this covers local files). + if (!isUrl && statSync(resolve(src)).size === 0) { + console.error(`error: refusing to ingest a 0-byte file: ${src}`); + process.exit(2); + } + const ext = extname(isUrl ? new URL(src).pathname : src) || defaultExt(type); + const { id, localPath, fullPath } = await withReservedFile( + projectDir, + type, + ext, + async (reservation) => { + if (isUrl) await freezeUrl(src, reservation.fullPath); + else freezeLocalFile(resolve(src), reservation.fullPath); + return reservation; + }, + ); + if (type === "lut" || type === "grade") { + try { + const check = validateCubeFile(fullPath); + if (!check.ok) throw new Error(check.error); + } catch (err) { + rmSync(fullPath, { force: true }); + exitError(`ingested LUT is invalid: ${err.message}`, 1); + } + } + const record = { + id, + type, + path: localPath, + source: "ingested", + description: basename(src.split("?")[0]), + provenance: { provider: "local", from: src }, + }; + appendRecord(projectDir, record); + regenerateIndex(projectDir); + try { + cachePut(fullPath, record); // surface ingested assets globally too (B3) + } catch { + // best-effort + } + await result(record, "ingested"); +} + +async function showCandidates() { + const projectDir = resolve(args.project); + const type = args.type; + if (!type || !listTypes().includes(type)) { + console.error(`error: --candidates requires --type (one of: ${listTypes().join(", ")})`); + process.exit(2); + } + const intent = args.intent || ""; + const { candidates, truncated, total, similar } = listCandidates({ + projectDir, + type, + intent, + cap: CANDIDATE_CAP, + }); + await track("media_use_candidates", { + type, + project_n: total.project, + global_n: total.global, + local_only: !!args["local-only"], + }); + if (args.json) { + console.log(JSON.stringify({ ok: true, candidates, truncated, total, similar })); + } else { + console.log(formatCandidates(candidates, { truncated, total })); + } +} + +// Best-effort latest stable CLI tag from the CDN (the install script's source of +// truth). null on any failure (offline, no curl) — treated as "unknown", never fatal. +function latestHeygenStable() { + const probe = runCommand("curl", [ + "-fsSL", + "--max-time", + "4", + "https://static.heygen.ai/cli/stable", + ]); + return probe.status === 0 ? firstSemver(commandText(probe)) : null; +} + +function heygenAuthCheck() { + // heygen auth status emits JSON by default; parse that output directly. + const authProbe = runCommand("heygen", ["auth", "status"]); + // spawnSync sets .error/.signal on a timeout or spawn failure (status then + // null). A stalled auth endpoint (transient network/DNS) must not be reported + // as an authoritative "not authenticated" with a re-login fix. + const timedOut = authProbe.error?.code === "ETIMEDOUT" || authProbe.signal != null; + const email = authProbe.status === 0 ? emailFromAuthStatus(commandText(authProbe)) : null; + return { + name: "heygen authenticated", + ok: !!email, + detail: email + ? `heygen authenticated as ${email}` + : timedOut + ? "heygen auth status timed out — possible network issue, not proof of sign-out" + : "heygen not authenticated", + fix: email ? "" : timedOut ? "check network, then re-run --doctor" : HEYGEN_AUTH_COMMAND, + }; +} + +function runDoctor() { + const checks = []; + const bundledSfx = inspectBundledSfxAssets(); + checks.push({ + name: "bundled SFX assets", + ok: bundledSfx.ok, + detail: bundledSfx.detail, + fix: bundledSfx.fix, + }); + const heygenVersionProbe = runCommand("heygen", ["--version"]); + const heygenOnPath = heygenVersionProbe.status === 0; + const heygenVersionText = commandText(heygenVersionProbe); + const heygenVersion = firstSemver(heygenVersionText); + + checks.push({ + name: "heygen on PATH", + ok: heygenOnPath, + // Just "is the binary here" — the version row below owns the version string, + // so this row must not also render `heygen v0.3.0` (two byte-identical lines). + detail: heygenOnPath ? "heygen found on PATH" : "heygen not found", + fix: heygenOnPath ? "" : HEYGEN_INSTALL_COMMAND, + }); + + if (!heygenOnPath) { + checks.push({ + name: "heygen version", + ok: false, + detail: "heygen version unavailable", + fix: HEYGEN_INSTALL_COMMAND, + }); + checks.push({ + name: "heygen authenticated", + ok: false, + detail: "heygen auth status unavailable", + fix: HEYGEN_INSTALL_COMMAND, + }); + } else if (heygenVersion) { + const versionOk = !versionLessThan(heygenVersion, HEYGEN_MIN_VERSION); + // Keep it latest: even when the installed version clears the floor, nudge + // `heygen update` if a newer stable exists. Best-effort — silently skipped + // when the CDN is unreachable, so it never blocks the check. + const latest = versionOk ? latestHeygenStable() : null; + const behind = latest && versionLessThan(heygenVersion, latest); + checks.push({ + name: "heygen version", + ok: versionOk, + detail: versionOk + ? `heygen v${heygenVersion}${behind ? ` (latest v${latest} available)` : ""}` + : `heygen v${heygenVersion} (need >= v${HEYGEN_MIN_VERSION})`, + fix: versionOk ? (behind ? HEYGEN_UPDATE_COMMAND : "") : HEYGEN_UPDATE_COMMAND, + }); + + // Older CLI versions cannot provide the auth status used here. + checks.push( + versionOk + ? heygenAuthCheck() + : { + name: "heygen authenticated", + ok: false, + detail: "skipped — update heygen first", + fix: HEYGEN_UPDATE_COMMAND, + }, + ); + } else { + // Fail-open: heygen ran but printed no semver (dev/stripped build). We can't + // verify the version, so we don't block on it — but say so rather than a bare + // green check that implies a real version comparison happened. + checks.push({ + name: "heygen version", + ok: true, + detail: "heygen present; version unverifiable (no semver in --version output)", + fix: "", + }); + + checks.push(heygenAuthCheck()); + } + + const ffmpegProbe = runCommand("ffmpeg", ["-version"]); + checks.push({ + name: "ffmpeg on PATH", + ok: ffmpegProbe.status === 0, + detail: ffmpegProbe.status === 0 ? firstLine(ffmpegProbe.stdout) : "ffmpeg not found", + fix: ffmpegProbe.status === 0 ? "" : "brew install ffmpeg", + }); + + const ffprobeProbe = runCommand("ffprobe", ["-version"]); + checks.push({ + name: "ffprobe on PATH", + ok: ffprobeProbe.status === 0, + detail: ffprobeProbe.status === 0 ? firstLine(ffprobeProbe.stdout) : "ffprobe not found", + fix: ffprobeProbe.status === 0 ? "" : "brew install ffmpeg", + }); + + const nodeOk = !versionLessThan(process.versions.node, MIN_NODE_VERSION); + checks.push({ + name: "node version", + ok: nodeOk, + detail: `${process.version} (need >= v${MIN_NODE_VERSION})`, + fix: nodeOk ? "" : `upgrade Node to >= v${MIN_NODE_VERSION}`, + }); + + // ffmpeg AND ffprobe are both strictly required (see references/setup-providers.md); the exit code + // must reflect that so a script gating on `--doctor` doesn't pass with ffprobe + // missing and then break at the first probe call. + const ffmpeg = checks.find((check) => check.name === "ffmpeg on PATH"); + const ffprobe = checks.find((check) => check.name === "ffprobe on PATH"); + return { ok: bundledSfx.ok && !!ffmpeg?.ok && !!ffprobe?.ok, checks }; +} + +function printDoctor(checks) { + const heygenChecks = new Set(["heygen on PATH", "heygen version", "heygen authenticated"]); + for (const check of checks) { + const prefix = check.ok ? "✓" : "✗"; + const freePath = heygenChecks.has(check.name) + ? " — free-usage path: bgm/image/voice/avatar-video" + : ""; + const fix = check.ok || !check.fix ? "" : ` — fix: ${check.fix}`; + console.log(`${prefix} ${check.detail}${freePath}${fix}`); + } +} + +function printStats(report) { + console.log("media-use stats"); + console.log(`total resolves: ${report.total_resolves}`); + console.log(`misses: ${report.misses}`); + console.log( + `hit rate: ${report.hit_rate == null ? "n/a" : `${Math.round(report.hit_rate * 100)}%`}`, + ); + printMap("by type", report.by_type); + printMap("by source", report.by_source); + printMap("by provider", report.by_provider); + printMap("by via", report.by_via); + console.log(`global cache assets: ${report.global_cache_assets}`); + console.log(`global cache disk: ${report.global_cache_disk_bytes} bytes`); + console.log(`cross-project reuse: ${report.cross_project_reuse}`); + console.log("top missed intents:"); + const entries = Object.entries(report.top_missed_intents); + if (entries.length === 0) { + console.log(" none"); + return; + } + for (const [type, misses] of entries) { + console.log(` ${type}:`); + for (const miss of misses) console.log(` ${miss.count} ${miss.intent}`); + } +} + +function printMap(label, values) { + const entries = Object.entries(values); + console.log(`${label}:`); + if (entries.length === 0) { + console.log(" none"); + return; + } + for (const [key, value] of entries) console.log(` ${key}: ${value}`); +} + +function runCommand(bin, argv) { + return spawnSync(bin, argv, { + encoding: "utf8", + timeout: 15000, + }); +} + +function commandText(result) { + return [result.stdout, result.stderr].filter(Boolean).join("\n").trim(); +} + +function firstLine(text) { + return ( + String(text || "") + .trim() + .split(/\r?\n/)[0] || "" + ); +} + +function emailFromAuthStatus(text) { + // JSON only (auth status emits JSON by default). No prose regex fallback: a + // human-format body like "Session expired. Contact support@heygen.ai" would + // otherwise report the user as authenticated as support@heygen.ai. + const trimmed = String(text || "").trim(); + if (!trimmed.startsWith("{")) return null; + try { + const parsed = JSON.parse(trimmed); + return parsed?.data?.email || parsed?.email || null; + } catch { + return null; + } +} + +async function reuseGlobal(shaArg) { + const projectDir = resolve(args.project); + const type = args.type; + if (!type || !listTypes().includes(type)) { + console.error(`error: --reuse requires --type (one of: ${listTypes().join(", ")})`); + process.exit(2); + } + if (!shaArg || !shaArg.trim()) { + console.error("error: --reuse needs a content sha/prefix (from `resolve --candidates`)"); + process.exit(2); + } + const rec = findGlobalBySha(shaArg); + if (rec && rec.ambiguous) { + console.error( + `error: sha prefix "${shaArg}" is ambiguous (${rec.count} matches) — use more characters`, + ); + process.exit(2); + } + if (!rec) { + console.error(`error: no reusable global asset matches sha "${shaArg}"`); + process.exit(1); + } + // Do not import an asset under the wrong type. + if (!typesMatch(rec.type, type)) { + console.error(`error: sha "${shaArg}" is a ${rec.type} asset, not ${type}`); + process.exit(2); + } + const ext = extname(rec.cached_path || "") || defaultExt(type); + const imported = withReservedFileSync(projectDir, type, ext, ({ id, localPath }) => + localizeImportedRecord(importFromCache(rec, projectDir, id, localPath), localPath), + ); + if (!imported) { + console.error(`error: cache entry for "${shaArg}" is incomplete or missing on disk`); + process.exit(1); + } + // Distinguish an explicit agent reuse from an automatic normalize-exact hit. + imported.source = "reused-explicit"; + imported.provenance = { ...imported.provenance, reused_by: "agent" }; + appendRecord(projectDir, imported); + regenerateIndex(projectDir); + await result(imported, "reused-explicit"); +} + +async function result(record, source) { + // Non-PII usage event: which media type, how it resolved, which provider won. + // Never the intent text or paths. Awaited so a short-lived run flushes it. + await track("media_use_resolve", { + type: record.type, + source, + provider: record.provenance?.provider, + // How a library LUT resolved: "url" (CDN), "params-fallback" (CDN failed → + // parametric), or "params" (offline). Surfaces silent CDN→params downgrades + // in prod, which --doctor can't (it only answers "reachable now?"). + via: record.provenance?.via, + // OAuth vs. API-key HeyGen paths are sparse for non-HeyGen providers. + // signal about the fetch that actually consumed a heygen credit, not + // about the (free, no-credential) act of copying a cached file. + auth_method: record.provenance?.authMethod, + // Provider tiers stay sparse and follow the registry's A/N/P declaration. + provider_tier: providerTierFor(record.provenance?.provider), + local_only: !!args["local-only"], + provider_override: !!args.provider, + }); + if (args.json) { + const grading = record.type === "grade" && record.grading ? record.grading : null; + console.log( + JSON.stringify({ + ok: true, + ...record, + ...(grading || {}), + ...(grading && { grading }), + _source: source, + }), + ); + } else { + const meta = formatMeta(record, source); + console.log(`resolved ${record.id} → ${record.path || "inline"} (${meta})`); + } +} + +function formatMeta(record, source) { + const parts = [record.type]; + if (record.grading?.preset) parts.push(`preset ${record.grading.preset}`); + if (record.grading?.lut) parts.push("lut"); + if (record.duration != null) parts.push(`${record.duration}s`); + if (record.width && record.height) parts.push(`${record.width}×${record.height}`); + if (record.transparent) parts.push("transparent"); + if (source === "reused" || source === "reused-explicit") parts.push("reused"); + if (source === "generated") parts.push("generated"); + return parts.join(", "); +} + +function extFromUrl(url) { + try { + return extname(new URL(url).pathname) || null; + } catch { + return null; + } +} + +function defaultExt(type) { + return DEFAULT_EXT[type] || ".bin"; +} + +run().catch((err) => { + if (args.json) { + console.log(JSON.stringify({ ok: false, error: err.message })); + } else { + console.error(`error: ${err.message}`); + } + process.exit(1); +}); diff --git a/skills/media-use/scripts/resolve.test.mjs b/packages/cli/src/media-use/resolve.test.mjs similarity index 99% rename from skills/media-use/scripts/resolve.test.mjs rename to packages/cli/src/media-use/resolve.test.mjs index 028270b08e..9a4af1510a 100644 --- a/skills/media-use/scripts/resolve.test.mjs +++ b/packages/cli/src/media-use/resolve.test.mjs @@ -21,7 +21,7 @@ import { freezeLocalFile } from "./lib/freeze.mjs"; import { cachePut, cacheGet, importFromCache } from "./lib/cache.mjs"; import { validateCubeFile } from "./lib/cube-validate.mjs"; -const REPO_ROOT = join(import.meta.dirname, "..", "..", ".."); +const REPO_ROOT = join(import.meta.dirname, "..", "..", "..", ".."); const RESOLVE_CLI = join(import.meta.dirname, "resolve.mjs"); // The "Test: skills" CI job has no ffmpeg on PATH (by design). The smart-grade // test shells to ffmpeg, so it's skipped there and runs where ffmpeg exists. diff --git a/packages/cli/src/registry/localSemantic.test.ts b/packages/cli/src/registry/localSemantic.test.ts index 701654dca5..d775f26e48 100644 --- a/packages/cli/src/registry/localSemantic.test.ts +++ b/packages/cli/src/registry/localSemantic.test.ts @@ -3,7 +3,13 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { tmpdir } from "node:os"; import { join } from "node:path"; -import { cachedLocalVectorRevision, fetchLocalVectors } from "./localSemantic.js"; +import { + cachedLocalVectorRevision, + fetchLocalVectors, + isMediaVectorRow, + mediaSemanticRanking, + vectorPairAgrees, +} from "./localSemantic.js"; import { LOCAL_MODEL_DIMENSIONS } from "./localModel.js"; describe("fetchLocalVectors", () => { @@ -29,6 +35,28 @@ describe("fetchLocalVectors", () => { })), ); + const mediaRow = { + id: "click", + kind: "sfx", + title: "click", + description: "short click", + tags: ["ui"], + file: "click.mp3", + duration: 0.2, + }; + + const serveMediaPair = (metadata: unknown, floats = LOCAL_MODEL_DIMENSIONS) => + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => ({ + ok: true, + arrayBuffer: async () => + url.endsWith(".json") + ? new TextEncoder().encode(JSON.stringify(metadata)).buffer + : new Float32Array(floats).buffer, + })), + ); + it("writes both files into the cache directory", async () => { servePair(["whip-pan"], LOCAL_MODEL_DIMENSIONS, LOCAL_MODEL_DIMENSIONS); expect(await fetchLocalVectors("http://registry.test/", { directory: dir })).toBe(true); @@ -107,4 +135,140 @@ describe("fetchLocalVectors", () => { writeFileSync(join(dir, "local-vectors.bin"), new Float32Array(LOCAL_MODEL_DIMENSIONS)); expect(cachedLocalVectorRevision(dir)).toBe("r1"); }); + + it("reports no semantic result without a media-vector cache", async () => { + expect(await mediaSemanticRanking("click", dir)).toBeNull(); + }); + + it("accepts a complete media-vector pair with validated rows", async () => { + serveMediaPair({ + names: ["click"], + dimensions: LOCAL_MODEL_DIMENSIONS, + rows: [mediaRow], + }); + + expect( + await fetchLocalVectors("http://registry.test", { + directory: dir, + artifactBasename: "media-vectors", + }), + ).toBe(true); + expect(existsSync(join(dir, "media-vectors.json"))).toBe(true); + expect(existsSync(join(dir, "media-vectors.bin"))).toBe(true); + }); + + it.each([ + ["missing rows", {}], + ["wrong row count", { rows: [] }], + ["wrong row shape", { rows: [{ ...mediaRow, tags: ["ui", 3] }] }], + ["wrong row id", { rows: [{ ...mediaRow, id: "other" }] }], + ["invalid duration", { rows: [{ ...mediaRow, duration: -1 }] }], + ["invalid dimensions", { rows: [{ ...mediaRow, dimensions: { width: 0, height: 10 } }] }], + ])("rejects media-vector metadata with %s", async (_reason, extra) => { + serveMediaPair({ + names: ["click"], + dimensions: LOCAL_MODEL_DIMENSIONS, + ...extra, + }); + + expect( + await fetchLocalVectors("http://registry.test", { + directory: dir, + artifactBasename: "media-vectors", + }), + ).toBe(false); + expect(existsSync(join(dir, "media-vectors.json"))).toBe(false); + }); + + it("rejects an unreadable media-vector metadata response", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => ({ + ok: true, + arrayBuffer: async () => + url.endsWith(".json") + ? new TextEncoder().encode("not json").buffer + : new Float32Array(LOCAL_MODEL_DIMENSIONS).buffer, + })), + ); + + expect( + await fetchLocalVectors("http://registry.test", { + directory: dir, + artifactBasename: "media-vectors", + }), + ).toBe(false); + }); +}); + +describe("media-vector validation", () => { + const row = { + id: "click", + kind: "sfx", + title: "click", + description: "short click", + tags: ["ui"], + file: "click.mp3", + duration: 0.2, + }; + + it.each([ + ["null", null], + ["missing id", { ...row, id: undefined }], + ["non-string tags", { ...row, tags: ["ui", 3] }], + ["negative duration", { ...row, duration: -1 }], + ["zero width", { ...row, dimensions: { width: 0, height: 10 } }], + ])("rejects %s rows", (_name, candidate) => { + expect(isMediaVectorRow(candidate)).toBe(false); + }); + + it("accepts a complete media row", () => { + expect(isMediaVectorRow(row)).toBe(true); + expect(isMediaVectorRow({ ...row, dimensions: { width: 1920, height: 1080 } })).toBe(true); + }); + + const invalidPairs: Array<[string, Array<[string, Buffer]>]> = [ + ["missing metadata", [["media-vectors.bin", Buffer.alloc(4)]]], + ["missing matrix", [["media-vectors.json", Buffer.from("{}")]]], + [ + "invalid json", + [ + ["media-vectors.json", Buffer.from("not json")], + ["media-vectors.bin", Buffer.alloc(4)], + ], + ], + [ + "invalid media row", + [ + [ + "media-vectors.json", + Buffer.from( + JSON.stringify({ names: ["click"], dimensions: 1, rows: [{ ...row, id: "other" }] }), + ), + ], + ["media-vectors.bin", Buffer.alloc(4)], + ], + ], + ]; + + it.each(invalidPairs)("rejects pairs with %s", (_name, fetched) => { + expect(vectorPairAgrees(fetched, "media-vectors")).toBe(false); + }); + + it("accepts a matching media-vector pair", () => { + const metadata = { + names: ["click"], + dimensions: LOCAL_MODEL_DIMENSIONS, + rows: [row], + }; + expect( + vectorPairAgrees( + [ + ["media-vectors.json", Buffer.from(JSON.stringify(metadata))], + ["media-vectors.bin", Buffer.alloc(4 * LOCAL_MODEL_DIMENSIONS)], + ], + "media-vectors", + ), + ).toBe(true); + }); }); diff --git a/packages/cli/src/registry/localSemantic.ts b/packages/cli/src/registry/localSemantic.ts index 42a444a720..6bcab7281b 100644 --- a/packages/cli/src/registry/localSemantic.ts +++ b/packages/cli/src/registry/localSemantic.ts @@ -12,6 +12,7 @@ * being compared against. */ +// Vector metadata validation is intentionally defensive at this file boundary. import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; @@ -29,11 +30,47 @@ interface LocalVectorMetadata { names: string[]; dimensions: number; revision?: string; + rows?: MediaVectorRow[]; +} + +export interface MediaVectorRow { + id: string; + kind: string; + title: string; + description: string; + tags: string[]; + file: string; + duration?: number; + dimensions?: { width: number; height: number }; +} + +export function isMediaVectorRow(value: unknown): value is MediaVectorRow { + if (!value || typeof value !== "object") return false; + const row = value as Partial; + return ( + typeof row.id === "string" && + typeof row.kind === "string" && + typeof row.title === "string" && + typeof row.description === "string" && + Array.isArray(row.tags) && + row.tags.every((tag) => typeof tag === "string") && + typeof row.file === "string" && + (row.duration === undefined || + (typeof row.duration === "number" && Number.isFinite(row.duration) && row.duration >= 0)) && + (row.dimensions === undefined || + (typeof row.dimensions === "object" && + row.dimensions !== null && + Number.isInteger(row.dimensions.width) && + Number.isInteger(row.dimensions.height) && + row.dimensions.width > 0 && + row.dimensions.height > 0)) + ); } export interface FetchLocalVectorOptions { directory?: string; expectedRevision?: string; + artifactBasename?: "local-vectors" | "media-vectors"; } const CATALOG_ARTIFACT_TIMEOUT_MS = 30_000; @@ -63,16 +100,30 @@ function localVectorDirectory(): string { * contract. A pair that fails it is a truncated download or a different * model, never something worth caching. */ -function vectorPairAgrees(fetched: Array<[string, Buffer]>): boolean { - const meta = fetched.find(([file]) => file === "local-vectors.json")?.[1]; - const bin = fetched.find(([file]) => file === "local-vectors.bin")?.[1]; +export function vectorPairAgrees( + fetched: Array<[string, Buffer]>, + artifactBasename: "local-vectors" | "media-vectors", +): boolean { + const meta = fetched.find(([file]) => file === `${artifactBasename}.json`)?.[1]; + const bin = fetched.find(([file]) => file === `${artifactBasename}.bin`)?.[1]; if (!meta || !bin) return false; try { const parsed = JSON.parse(meta.toString("utf-8")) as { names?: string[]; dimensions?: number; + rows?: Array<{ id?: string }>; }; if (parsed.dimensions !== LOCAL_MODEL_DIMENSIONS) return false; + if ( + artifactBasename === "media-vectors" && + (!parsed.rows || + parsed.rows.length !== (parsed.names?.length ?? -1) || + parsed.rows.some( + (row, index) => !isMediaVectorRow(row) || row.id !== parsed.names?.[index], + )) + ) { + return false; + } return bin.byteLength === (parsed.names?.length ?? -1) * LOCAL_MODEL_DIMENSIONS * 4; } catch { return false; @@ -83,9 +134,10 @@ function vectorPairAgrees(fetched: Array<[string, Buffer]>): boolean { function vectorRevisionAgrees( fetched: Array<[string, Buffer]>, expectedRevision?: string, + artifactBasename: "local-vectors" | "media-vectors" = "local-vectors", ): boolean { if (expectedRevision === undefined) return true; - const meta = fetched.find(([file]) => file === "local-vectors.json")?.[1]; + const meta = fetched.find(([file]) => file === `${artifactBasename}.json`)?.[1]; if (!meta) return false; try { const parsed = JSON.parse(meta.toString("utf-8")) as { revision?: unknown }; @@ -100,6 +152,7 @@ export async function fetchLocalVectors( options: FetchLocalVectorOptions = {}, ): Promise { const directory = options.directory ?? localVectorDirectory(); + const artifactBasename = options.artifactBasename ?? "local-vectors"; const base = registryBaseUrl.replace(/\/+$/, ""); try { mkdirSync(directory, { recursive: true, mode: 0o700 }); @@ -108,7 +161,7 @@ export async function fetchLocalVectors( // must leave the previous pair intact rather than pairing a new name list // with an old matrix, which loads as an error instead of as stale data. const fetched: Array<[string, Buffer]> = []; - for (const file of ["local-vectors.json", "local-vectors.bin"] as const) { + for (const file of [`${artifactBasename}.json`, `${artifactBasename}.bin`] as const) { const response = await fetch(`${base}/catalog-artifact/${file}`, { signal: AbortSignal.timeout(CATALOG_ARTIFACT_TIMEOUT_MS), }); @@ -119,7 +172,10 @@ export async function fetchLocalVectors( // discovering the mismatch at load time leaves a cache that fails every // subsequent search until someone deletes it by hand, and it is the only // point where a truncated or wrong-model response can still be refused. - if (!vectorPairAgrees(fetched) || !vectorRevisionAgrees(fetched, options.expectedRevision)) { + if ( + !vectorPairAgrees(fetched, artifactBasename) || + !vectorRevisionAgrees(fetched, options.expectedRevision, artifactBasename) + ) { return false; } // 0o600: the cache is this user's, and the directory may be world-writable @@ -127,7 +183,9 @@ export async function fetchLocalVectors( for (const [file, bytes] of fetched) { writeFileSync(join(directory, file), bytes, { mode: 0o600 }); } - return hasLocalVectors(directory); + return artifactBasename === "media-vectors" + ? hasMediaVectors(directory) + : hasLocalVectors(directory); } catch { return false; } @@ -140,6 +198,13 @@ export function hasLocalVectors(directory = localVectorDirectory()): boolean { ); } +function hasMediaVectors(directory = localVectorDirectory()): boolean { + return ( + existsSync(join(directory, "media-vectors.bin")) && + existsSync(join(directory, "media-vectors.json")) + ); +} + function loadLocalVectors(directory = localVectorDirectory()): LocalVectorSet { const meta = JSON.parse( readFileSync(join(directory, "local-vectors.json"), "utf-8"), @@ -161,6 +226,34 @@ function loadLocalVectors(directory = localVectorDirectory()): LocalVectorSet { return { names: meta.names, dimensions: meta.dimensions, vectors }; } +function loadMediaVectors(directory = localVectorDirectory()): LocalVectorSet & { + rows: MediaVectorRow[]; +} { + const meta = JSON.parse( + readFileSync(join(directory, "media-vectors.json"), "utf-8"), + ) as LocalVectorMetadata; + const buffer = readFileSync(join(directory, "media-vectors.bin")); + const vectors = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4); + const rows = meta.rows ?? []; + if (rows.length !== meta.names.length) { + throw new Error(`media vectors hold ${rows.length} rows, expected ${meta.names.length}`); + } + if (rows.some((row, index) => !isMediaVectorRow(row) || row.id !== meta.names[index])) { + throw new Error("media vector row order does not match media vector names"); + } + if (vectors.length !== rows.length * meta.dimensions) { + throw new Error( + `media vectors hold ${vectors.length} floats, expected ${rows.length * meta.dimensions}`, + ); + } + if (meta.dimensions !== LOCAL_MODEL_DIMENSIONS) { + throw new Error( + `media vectors are ${meta.dimensions}-dimension, model produces ${LOCAL_MODEL_DIMENSIONS}`, + ); + } + return { names: meta.names, dimensions: meta.dimensions, vectors, rows }; +} + /** * The names the on-device index holds, without loading the vector matrix. * @@ -227,3 +320,23 @@ export async function localSemanticRanking( scored.sort((a, b) => b.score - a.score || b.name.localeCompare(a.name)); return scored; } + +export async function mediaSemanticRanking( + query: string, + directory = localVectorDirectory(), +): Promise | null> { + if (!isLocalModelReady() || !hasMediaVectors(directory)) return null; + const set = loadMediaVectors(directory); + const embedder = await loadLocalEmbedder(); + const [queryVector] = await embedder.embed([query], { isQuery: true }); + if (!queryVector) return null; + return set.rows + .map((row, index) => ({ + row, + score: cosine( + queryVector, + Array.from(set.vectors.subarray(index * set.dimensions, (index + 1) * set.dimensions)), + ), + })) + .sort((a, b) => b.score - a.score || b.row.id.localeCompare(a.row.id)); +} diff --git a/packages/cli/src/registry/registryComponents.test.ts b/packages/cli/src/registry/registryComponents.test.ts index 37537c759b..4f3cfa1969 100644 --- a/packages/cli/src/registry/registryComponents.test.ts +++ b/packages/cli/src/registry/registryComponents.test.ts @@ -22,9 +22,9 @@ async function invalidInstallableMedia(entryName: string): Promise { for (const file of manifest.files) { if (file.type !== "hyperframes:snippet" || !file.path.endsWith(".html")) continue; - const result = await lintHyperframeHtml(readFileSync(join(itemDir, file.path), "utf8"), { - isSubComposition: true, - }); + const html = readFileSync(join(itemDir, file.path), "utf8"); + if (!/<(?:audio|video)\b/i.test(html)) continue; + const result = await lintHyperframeHtml(html, { isSubComposition: true }); for (const finding of result.findings) { if (finding.code !== "media_missing_src") continue; invalidMedia.push(`${entryName}/${file.path}: ${finding.code}`); diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts index 50365fb20f..b9592f72d4 100644 --- a/packages/cli/tsup.config.ts +++ b/packages/cli/tsup.config.ts @@ -17,6 +17,7 @@ export default defineConfig({ runtimeVersion: "src/runtimeVersion.ts", renderSetupWorker: "src/renderSetupWorker.ts", shaderTransitionWorker: "../producer/src/services/shaderTransitionWorker.ts", + "registry/localSemantic": "src/registry/localSemantic.ts", }, format: ["esm"], outDir: "dist", diff --git a/packages/core/src/figma/mediaIndex.test.ts b/packages/core/src/figma/mediaIndex.test.ts index b2d597f8ae..95aedcb9d6 100644 --- a/packages/core/src/figma/mediaIndex.test.ts +++ b/packages/core/src/figma/mediaIndex.test.ts @@ -87,9 +87,10 @@ describe("regenerateIndex", () => { "..", "..", "..", - "skills", + "packages", + "cli", + "src", "media-use", - "scripts", "lib", "index-gen.mjs", ), diff --git a/packages/core/src/mediaGradeAnalyzer.vendoredParity.test.ts b/packages/core/src/mediaGradeAnalyzer.vendoredParity.test.ts index 8eb943574e..7a62e9d079 100644 --- a/packages/core/src/mediaGradeAnalyzer.vendoredParity.test.ts +++ b/packages/core/src/mediaGradeAnalyzer.vendoredParity.test.ts @@ -69,7 +69,7 @@ function productResult(signalStats = SIGNALSTATS) { async function vendoredResult(signalStats = SIGNALSTATS) { const moduleUrl = new URL( - "../../../skills/media-use/scripts/lib/grade-analyzer.mjs", + "../../../packages/cli/src/media-use/lib/grade-analyzer.mjs", import.meta.url, ); const analyzer = await import(moduleUrl.href); diff --git a/registry/catalog-artifact/.gitignore b/registry/catalog-artifact/.gitignore index 5d341d92ca..08cd699642 100644 --- a/registry/catalog-artifact/.gitignore +++ b/registry/catalog-artifact/.gitignore @@ -10,3 +10,4 @@ # half-complete and the offline tier silently unable to rank. *.json !local-vectors.json +!media-vectors.json diff --git a/registry/catalog-artifact/media-vectors.bin b/registry/catalog-artifact/media-vectors.bin new file mode 100644 index 0000000000..e2e7c0b82a Binary files /dev/null and b/registry/catalog-artifact/media-vectors.bin differ diff --git a/registry/catalog-artifact/media-vectors.json b/registry/catalog-artifact/media-vectors.json new file mode 100644 index 0000000000..faa379345d --- /dev/null +++ b/registry/catalog-artifact/media-vectors.json @@ -0,0 +1,205 @@ +{ + "model": "bge-small-en-v1.5", + "modelRevision": "ea104dacec62c0de699686887e3f920caeb4f3e3", + "dimensions": 384, + "revision": "56598f057867f4e070ec88a9d7a65f7168827ce7019dd750e8a253cd9e393fb3", + "names": [ + "chime", + "click", + "click-soft", + "error", + "glitch-1", + "glitch-2", + "glitch-3", + "impact-bass-1", + "impact-bass-2", + "key-press", + "notification", + "ping", + "pop", + "riser", + "sparkle", + "typing", + "whoosh", + "whoosh-cinematic", + "whoosh-short" + ], + "rows": [ + { + "id": "chime", + "kind": "sfx", + "title": "chime", + "description": "Soft melodic chime — gentle positive beat: success/confirmation or a lighthearted transition. Sync to the visual moment.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/chime.mp3", + "duration": 2.5 + }, + { + "id": "click", + "kind": "sfx", + "title": "click", + "description": "Crisp UI click — button press, toggle, selection. Short accent, sync exactly to the on-screen action.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/click.mp3", + "duration": 0.37 + }, + { + "id": "click-soft", + "kind": "sfx", + "title": "click-soft", + "description": "Quiet short click — low-key UI tap / soft selection. Short accent, sync exactly to the on-screen action.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/click-soft.mp3", + "duration": 0.37 + }, + { + "id": "error", + "kind": "sfx", + "title": "error", + "description": "Negative / error tone — failure state, a 'wrong' beat, or a glitchy interruption.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/error.mp3", + "duration": 1.62 + }, + { + "id": "glitch-1", + "kind": "sfx", + "title": "glitch-1", + "description": "Punchy digital glitch — hard-cut accent or sudden reveal. Trigger on the hit; let the decay bleed into the next shot (J-cut).", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/glitch-1.mp3", + "duration": 2.64 + }, + { + "id": "glitch-2", + "kind": "sfx", + "title": "glitch-2", + "description": "Harsh, longer glitch — chaotic / jarring transition or a distorted reveal.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/glitch-2.mp3", + "duration": 3.5 + }, + { + "id": "glitch-3", + "kind": "sfx", + "title": "glitch-3", + "description": "Low-key glitch texture — subtle digital shift, minimal transition that sits under other audio.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/glitch-3.mp3", + "duration": 3.1 + }, + { + "id": "impact-bass-1", + "kind": "sfx", + "title": "impact-bass-1", + "description": "Bass impact hit — logo/hero snap, headline slam. Trigger on the visual landing; decay carries into the next shot (J-cut).", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/impact-bass-1.mp3", + "duration": 2.12 + }, + { + "id": "impact-bass-2", + "kind": "sfx", + "title": "impact-bass-2", + "description": "Bass impact with a short swell — brief anticipation then a deep hit. Place so the peak lands on the reveal.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/impact-bass-2.mp3", + "duration": 2.59 + }, + { + "id": "key-press", + "kind": "sfx", + "title": "key-press", + "description": "Single key press — one keystroke / terminal-input beat. Short accent, sync to the typed character.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/key-press.mp3", + "duration": 0.4 + }, + { + "id": "notification", + "kind": "sfx", + "title": "notification", + "description": "Notification chime — alert, message-in, toast/badge appears. Sync to the element entering.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/notification.mp3", + "duration": 2.46 + }, + { + "id": "ping", + "kind": "sfx", + "title": "ping", + "description": "Sharp electronic ping — punchy accent on a key reveal or data point. Sync to the beat.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/ping.mp3", + "duration": 1.32 + }, + { + "id": "pop", + "kind": "sfx", + "title": "pop", + "description": "Quick pop — element appear/spawn, chip/tag/badge in. Small precise accent, sync to the pop-in.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/pop.mp3", + "duration": 0.72 + }, + { + "id": "riser", + "kind": "sfx", + "title": "riser", + "description": "Long cinematic riser (~10s build, peak at the end). Trigger at (climax_time − 10.03s) so it crests exactly on the reveal.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/riser.mp3", + "duration": 10.03 + }, + { + "id": "sparkle", + "kind": "sfx", + "title": "sparkle", + "description": "Bright sparkle / shimmer — magical reveal or 'shine' highlight on a hero element. Sync to the highlight.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/sparkle.mp3", + "duration": 1.8 + }, + { + "id": "typing", + "kind": "sfx", + "title": "typing", + "description": "Typing burst (~1.5s of keys) — keyboard / code typing reveal, text-being-typed beat. Start as the text begins typing.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/typing.mp3", + "duration": 1.5 + }, + { + "id": "whoosh", + "kind": "sfx", + "title": "whoosh", + "description": "Punchy whoosh/impact — fast reveal or hard transition accent. Sync to the motion.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/whoosh.mp3", + "duration": 0.57 + }, + { + "id": "whoosh-cinematic", + "kind": "sfx", + "title": "whoosh-cinematic", + "description": "Cinematic whoosh build (~5.5s) — sweeping scene transition. Align so the swell peaks on the cut.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/whoosh-cinematic.mp3", + "duration": 5.54 + }, + { + "id": "whoosh-short", + "kind": "sfx", + "title": "whoosh-short", + "description": "Short whoosh — quick swipe/slide accent, fast element move, snappy transition. Sync to the motion.", + "tags": ["sfx"], + "file": "skills/media-use/audio/assets/sfx/whoosh-short.mp3", + "duration": 0.57 + } + ], + "metadataRevision": "3bce17533129b251a93cf1587ea18174bef06e0680611040ee82c24ed2bea03f", + "credits": { + "file": "skills/media-use/audio/assets/sfx/CREDITS.md", + "sha256": "79590c7738b0427b10f362f76b02cfd2643158a40e1d5fd787f9ac6b33bcf4ce" + } +} diff --git a/scripts/catalog/build-local-vectors.ts b/scripts/catalog/build-local-vectors.ts index a13104a966..c78715b890 100644 --- a/scripts/catalog/build-local-vectors.ts +++ b/scripts/catalog/build-local-vectors.ts @@ -1,3 +1,5 @@ +// The vector builder validates two input formats and their artifact invariants. +// fallow-ignore-file complexity /** * Embed the catalog with the on-device model so local search can rank by meaning. * @@ -30,8 +32,21 @@ import { catalogFromRegistry, LOCAL_VECTOR_BATCH_SIZE, localVectorRevision, + mediaMetadataRevision, + sha256Hex, } from "./catalog-artifact.js"; +export interface MediaVectorRow { + id: string; + kind: string; + title: string; + description: string; + tags: string[]; + file: string; + duration?: number; + dimensions?: { width: number; height: number }; +} + /** Distinct from 1 so the pre-commit hook can tell "cannot" from "failed". */ const EXIT_NO_MODEL = 3; @@ -40,6 +55,55 @@ function arg(name: string): string | undefined { return index === -1 ? undefined : process.argv[index + 1]; } +// fallow-ignore-next-line high-crap-score +function mediaRows(manifestPath: string): MediaVectorRow[] { + const parsed = JSON.parse(readFileSync(manifestPath, "utf8")) as + | MediaVectorRow[] + | Record & { tags?: string[] }>; + const bundledSfx = manifestPath.endsWith("skills/media-use/audio/assets/sfx/manifest.json"); + const rootRelativeFile = (file: string): string => + bundledSfx && !file.startsWith("skills/") ? `skills/media-use/audio/assets/sfx/${file}` : file; + const rows = Array.isArray(parsed) + ? parsed.map((row) => ({ ...row, file: rootRelativeFile(row.file) })) + : Object.entries(parsed).map(([id, entry]) => ({ + id, + kind: "sfx", + title: id, + description: entry.description, + tags: entry.tags ?? ["sfx"], + file: rootRelativeFile(entry.file), + ...(entry.duration === undefined ? {} : { duration: entry.duration }), + ...(entry.dimensions === undefined ? {} : { dimensions: entry.dimensions }), + })); + if (rows.length === 0) throw new Error(`media manifest contains no rows: ${manifestPath}`); + for (const row of rows) { + if ( + !row.id || + !row.kind || + !row.file || + !row.title || + !row.description || + !Array.isArray(row.tags) || + !row.tags.every((tag) => typeof tag === "string") + ) { + throw new Error(`media manifest row is missing id, title, description, or file`); + } + if (row.duration !== undefined && (!Number.isFinite(row.duration) || row.duration < 0)) { + throw new Error(`media manifest row ${row.id} has an invalid duration`); + } + if ( + row.dimensions !== undefined && + (!Number.isInteger(row.dimensions.width) || + !Number.isInteger(row.dimensions.height) || + row.dimensions.width <= 0 || + row.dimensions.height <= 0) + ) { + throw new Error(`media manifest row ${row.id} has invalid dimensions`); + } + } + return [...rows].sort((left, right) => left.id.localeCompare(right.id)); +} + /** * Embed in batches, in `names` order. * @@ -81,19 +145,29 @@ function packVectors(names: string[], vectors: number[][]): Float32Array { async function main(): Promise { const dir = arg("artifact") ?? "registry/catalog-artifact"; const registryDir = arg("registry") ?? "registry"; + const manifestPath = arg("manifest"); + const basename = arg("output") ?? (manifestPath ? "media-vectors" : "local-vectors"); // Read the corpus straight from the registry rather than from a catalog.json // built by the hosted-tier script. That file is not in the repo, so the // documented regeneration command used to fail on a missing path, which is // the whole reason the index was allowed to drift. - const catalogMap = catalogFromRegistry( - registryDir, - (path) => readFileSync(path, "utf-8"), - (path) => - readdirSync(path, { withFileTypes: true }) - .filter((e) => e.isDirectory()) - .map((e) => e.name), - ); + const rows = manifestPath ? mediaRows(manifestPath) : undefined; + const catalogMap = rows + ? new Map( + rows.map((row) => [ + row.id, + `${row.title}\n${row.description}\n${row.tags.join(" ")}\n${row.kind}`, + ]), + ) + : catalogFromRegistry( + registryDir, + (path) => readFileSync(path, "utf-8"), + (path) => + readdirSync(path, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => e.name), + ); const catalog = Object.fromEntries(catalogMap); const names = Object.keys(catalog).sort(); if (names.length === 0) throw new Error(`no registry items found under ${registryDir}`); @@ -121,26 +195,35 @@ async function main(): Promise { const vectors = await embedInBatches(names, catalog, embedder); const flat = packVectors(names, vectors); + const credits = + rows && manifestPath?.endsWith("skills/media-use/audio/assets/sfx/manifest.json") + ? { + file: "skills/media-use/audio/assets/sfx/CREDITS.md", + sha256: sha256Hex(readFileSync("skills/media-use/audio/assets/sfx/CREDITS.md", "utf8")), + } + : undefined; - writeFileSync(join(dir, "local-vectors.bin"), Buffer.from(flat.buffer)); - writeFileSync( - join(dir, "local-vectors.json"), - `${JSON.stringify({ model: LOCAL_MODEL_ID, modelRevision: LOCAL_MODEL_REVISION, dimensions: LOCAL_MODEL_DIMENSIONS, revision, names }, null, 2)}\n`, - ); - const registryPath = join(registryDir, "registry.json"); - const registry = JSON.parse(readFileSync(registryPath, "utf-8")) as RegistryManifest; + writeFileSync(join(dir, `${basename}.bin`), Buffer.from(flat.buffer)); writeFileSync( - registryPath, - `${JSON.stringify({ ...registry, catalogArtifact: { revision } }, null, 2)}\n`, + join(dir, `${basename}.json`), + `${JSON.stringify({ model: LOCAL_MODEL_ID, modelRevision: LOCAL_MODEL_REVISION, dimensions: LOCAL_MODEL_DIMENSIONS, revision, names, ...(rows ? { rows, metadataRevision: mediaMetadataRevision(rows) } : {}), ...(credits ? { credits } : {}) }, null, 2)}\n`, ); + if (!rows) { + const registryPath = join(registryDir, "registry.json"); + const registry = JSON.parse(readFileSync(registryPath, "utf-8")) as RegistryManifest; + writeFileSync( + registryPath, + `${JSON.stringify({ ...registry, catalogArtifact: { revision } }, null, 2)}\n`, + ); + } const megabytes = (flat.byteLength / 1024 / 1024).toFixed(2); - console.log(`moves ${names.length}`); + console.log(`${rows ? "media" : "moves"} ${names.length}`); console.log(`model ${LOCAL_MODEL_ID}`); console.log(`dimensions ${LOCAL_MODEL_DIMENSIONS}`); console.log(`revision ${revision}`); console.log(`payload ${megabytes} MB`); - console.log(`written ${dir}`); + console.log(`written ${dir}/${basename}.{json,bin}`); } await main(); diff --git a/scripts/catalog/catalog-artifact.ts b/scripts/catalog/catalog-artifact.ts index 5e49816e2c..fe60417665 100644 --- a/scripts/catalog/catalog-artifact.ts +++ b/scripts/catalog/catalog-artifact.ts @@ -148,6 +148,10 @@ export function localVectorRevision( ); } +export function mediaMetadataRevision(rows: readonly unknown[]): string { + return sha256Hex(JSON.stringify(rows)); +} + /** * Digest the two published files as bytes. * diff --git a/scripts/catalog/check-artifact-coverage.ts b/scripts/catalog/check-artifact-coverage.ts index 6188d15317..4ebcfd49c3 100644 --- a/scripts/catalog/check-artifact-coverage.ts +++ b/scripts/catalog/check-artifact-coverage.ts @@ -18,14 +18,38 @@ import { LOCAL_MODEL_ID, LOCAL_MODEL_REVISION, } from "../../packages/cli/src/registry/localModel.js"; -import { catalogFromRegistry, localVectorRevision } from "./catalog-artifact.js"; +import { + catalogFromRegistry, + localVectorRevision, + mediaMetadataRevision, + sha256Hex, +} from "./catalog-artifact.js"; type RegistryItem = { name: string; type?: string }; type Registry = { items: RegistryItem[]; catalogArtifact?: { revision?: string } }; type Artifact = { model?: string; dimensions?: number; revision?: string; names?: string[] }; +type MediaArtifact = { + model?: string; + modelRevision?: string; + dimensions?: number; + revision?: string; + metadataRevision?: string; + credits?: { file?: string; sha256?: string }; + names?: string[]; + rows?: Array<{ + id?: string; + file?: string; + title?: string; + description?: string; + tags?: string[]; + kind?: string; + }>; +}; const REGISTRY = "registry/registry.json"; const ARTIFACT = "registry/catalog-artifact/local-vectors.json"; +const MEDIA_MANIFEST = "skills/media-use/audio/assets/sfx/manifest.json"; +const MEDIA_ARTIFACT = "registry/catalog-artifact/media-vectors.json"; function read(path: string): T { try { @@ -67,6 +91,70 @@ const expectedRevision = localVectorRevision( ); const artifactRevisionMatches = artifact.revision === expectedRevision; const registryRevisionMatches = registry.catalogArtifact?.revision === expectedRevision; +const mediaSource = + read>(MEDIA_MANIFEST); +const mediaArtifact = read(MEDIA_ARTIFACT); +const mediaRowsFromSource = Object.entries(mediaSource) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([id, source]) => ({ + id, + kind: "sfx", + title: id, + description: source.description ?? "", + tags: ["sfx"], + file: `skills/media-use/audio/assets/sfx/${source.file ?? ""}`, + ...(source.duration === undefined ? {} : { duration: source.duration }), + })); +const mediaEntries = new Map( + mediaRowsFromSource.map((row) => [ + row.id, + `${row.title}\n${row.description}\n${row.tags.join(" ")}\n${row.kind}`, + ]), +); +const expectedMediaRevision = localVectorRevision( + LOCAL_MODEL_ID, + LOCAL_MODEL_REVISION, + LOCAL_MODEL_DIMENSIONS, + mediaEntries, +); +const expectedMediaMetadataRevision = mediaMetadataRevision(mediaRowsFromSource); +const expectedCredits = { + file: "skills/media-use/audio/assets/sfx/CREDITS.md", + sha256: sha256Hex(readFileSync("skills/media-use/audio/assets/sfx/CREDITS.md", "utf8")), +}; +const mediaBin = readFileSync(MEDIA_ARTIFACT.replace(/\.json$/, ".bin")); +const mediaRows = new Map( + (mediaArtifact.rows ?? []) + .filter( + (row): row is { id: string; file: string } => + typeof row.id === "string" && typeof row.file === "string", + ) + .map((row) => [row.id, row.file]), +); +const missingMediaRows = Object.entries(mediaSource) + .filter( + ([id, source]) => + mediaRows.get(id) !== `skills/media-use/audio/assets/sfx/${source.file ?? ""}`, + ) + .map(([id]) => id) + .sort(); +const mediaNames = mediaArtifact.names ?? []; +const mediaRowsMatchNames = + (mediaArtifact.rows ?? []).length === mediaNames.length && + (mediaArtifact.rows ?? []).every((row, index) => row.id === mediaNames[index]); +const mediaRowsMatchSource = + JSON.stringify(mediaArtifact.rows ?? []) === JSON.stringify(mediaRowsFromSource); +const mediaArtifactValid = + mediaArtifact.model === LOCAL_MODEL_ID && + mediaArtifact.modelRevision === LOCAL_MODEL_REVISION && + mediaArtifact.dimensions === LOCAL_MODEL_DIMENSIONS && + mediaArtifact.revision === expectedMediaRevision && + mediaArtifact.metadataRevision === expectedMediaMetadataRevision && + mediaArtifact.credits?.file === expectedCredits.file && + mediaArtifact.credits.sha256 === expectedCredits.sha256 && + mediaRowsMatchSource && + mediaRowsMatchNames && + mediaBin.byteLength === mediaNames.length * LOCAL_MODEL_DIMENSIONS * 4; const show = (names: string[]) => names @@ -76,6 +164,17 @@ const show = (names: string[]) => console.log(`registry: ${registryNames.size} searchable items (blocks + components)`); console.log(`artifact: ${artifactNames.size} vectors (${artifact.model ?? "unknown model"})`); +console.log( + `media: ${mediaRows.size} rows for ${Object.keys(mediaSource).length} bundled SFX files`, +); + +if (missingMediaRows.length > 0) { + console.error(`\n${missingMediaRows.length} bundled SFX file(s) have no matching media row:`); + console.error(show(missingMediaRows)); +} +if (!mediaArtifactValid) { + console.error("\nThe published media vector metadata or binary does not match the SFX manifest."); +} if (dropped.length > 0) { // Not fatal: the CLI filters these before a user ever sees them. @@ -96,7 +195,13 @@ if (!artifactRevisionMatches || !registryRevisionMatches) { console.error(` registry: ${registry.catalogArtifact?.revision ?? "missing"}`); } -if (unindexed.length > 0 || !artifactRevisionMatches || !registryRevisionMatches) { +if ( + unindexed.length > 0 || + !artifactRevisionMatches || + !registryRevisionMatches || + missingMediaRows.length > 0 || + !mediaArtifactValid +) { console.error( "\nMeaning search is stale. Word search still uses the live registry.\n\n" + "If you have the embedding model, regenerate and commit the artifact:\n" + diff --git a/scripts/check-media-use-copy-parity.test.mjs b/scripts/check-media-use-copy-parity.test.mjs new file mode 100644 index 0000000000..a5bedac0c5 --- /dev/null +++ b/scripts/check-media-use-copy-parity.test.mjs @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { describe, it } from "node:test"; + +const skillLibDir = resolve("skills/media-use/scripts/lib"); +const cliLibDir = resolve("packages/cli/src/media-use/lib"); + +export const MEDIA_USE_COPY_NAMES = [ + "cutlist.mjs", + "duck.mjs", + "error-diffusion.mjs", + "index-gen.mjs", + "manifest.mjs", + "media-fetch.mjs", + "npx-sync.mjs", + "parakeet-words.mjs", + "prefs-store.mjs", + "recipe-store.mjs", + "telemetry.mjs", + "words.mjs", +]; + +export const INTENTIONAL_MEDIA_USE_DIVERGENCES = new Map([ + [ + "media-fetch.mjs", + "the standalone skill uses a shim because it cannot import the CLI package tree", + ], + [ + "npx-sync.mjs", + "the standalone skill stays self-contained while the CLI copy uses the shared audio helper", + ], +]); + +function copyPaths(name, skillDir, cliDir) { + return { skillPath: join(skillDir, name), cliPath: join(cliDir, name) }; +} + +export function findMediaUseCopyParityIssues({ skillDir = skillLibDir, cliDir = cliLibDir } = {}) { + const missingNames = MEDIA_USE_COPY_NAMES.filter((name) => { + const { skillPath, cliPath } = copyPaths(name, skillDir, cliDir); + return !existsSync(skillPath) || !existsSync(cliPath); + }); + const missing = missingNames.map((name) => `${name}: both media-use copies must exist`); + const drifted = MEDIA_USE_COPY_NAMES.filter( + (name) => !missingNames.includes(name) && !INTENTIONAL_MEDIA_USE_DIVERGENCES.has(name), + ) + .filter((name) => { + const { skillPath, cliPath } = copyPaths(name, skillDir, cliDir); + return !readFileSync(skillPath).equals(readFileSync(cliPath)); + }) + .map((name) => `${name}: standalone and CLI copies differ without an allowlist reason`); + return [...missing, ...drifted]; +} + +describe("media-use source parity", () => { + it("keeps every standalone copy equal or explicitly allowlisted", () => { + assert.deepEqual(findMediaUseCopyParityIssues(), []); + }); + + it("reports a missing copy without reading it as drift", () => { + const root = mkdtempSync(join(tmpdir(), "media-use-parity-")); + const skillDir = join(root, "skill"); + const cliDir = join(root, "cli"); + try { + const name = MEDIA_USE_COPY_NAMES[0]; + mkdirSync(skillDir, { recursive: true }); + mkdirSync(cliDir, { recursive: true }); + writeFileSync(join(skillDir, name), "same"); + const issues = findMediaUseCopyParityIssues({ skillDir, cliDir }); + assert.equal(issues[0], `${name}: both media-use copies must exist`); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/skills-manifest.json b/skills-manifest.json index 99060c3e75..44ee257b8b 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -30,7 +30,7 @@ "files": 7 }, "hyperframes-cli": { - "hash": "986414090bf6442f", + "hash": "609870a08e979434", "files": 11 }, "hyperframes-core": { @@ -54,8 +54,8 @@ "files": 1 }, "media-use": { - "hash": "8f21655cd07a62ea", - "files": 158 + "hash": "e5376453c0d08ac3", + "files": 88 }, "motion-graphics": { "hash": "32641ae2b94c4a8f", diff --git a/skills/hyperframes-cli/SKILL.md b/skills/hyperframes-cli/SKILL.md index 0e296a27e7..990587d960 100644 --- a/skills/hyperframes-cli/SKILL.md +++ b/skills/hyperframes-cli/SKILL.md @@ -16,7 +16,7 @@ Run commands as `npx hyperframes ...` unless project instructions provide a wrap ## Development loop 1. **Scaffold:** `npx hyperframes init ` (centered blank). Or capture a site. Pass `--example=` only to start from a named example. -2. **Find the move:** before authoring motion by hand, search for a primitive that already does it: `npx hyperframes catalog --query "reveal a headline one line at a time"`. Ask for the effect you want rather than the mechanism you have in mind. Install with `npx hyperframes add ` (see `/hyperframes-registry`). Author by hand only once nothing fits. +2. **Find the move:** if the request names an asset, sound, image, voice or fast visual edit, resolve it through `/media-use` before proposing a plan. Otherwise, before authoring motion by hand, search for a primitive that already does it: `npx hyperframes catalog --query "reveal a headline one line at a time"`. Ask for the effect you want rather than the mechanism you have in mind. Install with `npx hyperframes add ` (see `/hyperframes-registry`). Author by hand only once nothing fits. 3. **Author:** write the composition using `/hyperframes-core`. To know what is on a project's timeline (tracks, clips, starts, ends, what plays), run `npx hyperframes timeline --json` instead of reading `index.html` and every sub-composition file: nested rows carry absolute main-timeline `absStart`/`absEnd` and their owning `file`, not just their local, per-sub-composition time. Prefer `--json` over the text form; it costs fewer tokens for the same or better correctness. See `references/upgrade-info-misc.md` for one-liners that answer common questions without reading the whole output. 4. **Get fast feedback while editing:** run `npx hyperframes lint` after the first HTML pass and after structural changes. 5. **Run the final gate:** run `npx hyperframes check`; it reruns lint before opening the browser. Do not prepend a redundant standalone lint invocation. Add `--snapshots` for annotated overview frames and finding crops. @@ -36,7 +36,7 @@ Run commands as `npx hyperframes ...` unless project instructions provide a wrap - For fade-in/fade-out, crossfade, track gain, volume automation, ducking, voiceover carve, or FX on placed audio, read `/hyperframes-audio`. Load core alongside it when clip placement or picture timing also changes. -- Use `/media-use` only to source/generate media or preprocess a derived asset. +- A request naming an asset, sound, image, voice or fast visual edit resolves through `/media-use` before a plan is proposed. Copy creator edit markup from `/hyperframes-core` → `references/creator-editing-recipes.md`. ```bash diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md index 5655ccb439..d9debca680 100644 --- a/skills/media-use/SKILL.md +++ b/skills/media-use/SKILL.md @@ -7,12 +7,12 @@ description: Agent Media OS, the single skill for every media need in a HyperFra The media OS for HyperFrames: resolve · generate · operate · remember — every media type, one skill, zero context noise. -First run: install and sign in to the `heygen` CLI (the free-usage path), then verify with `node /scripts/resolve.mjs --doctor`. Setup and providers: `references/setup-providers.md`. +First run: install and sign in to the `heygen` CLI (the free-usage path), then verify with `npx hyperframes media-use resolve --doctor`. Setup and providers: `references/setup-providers.md`. ## Resolve — the one verb ```bash -node /scripts/resolve.mjs --type --intent "" --project +npx hyperframes media-use resolve --type --intent "" --project ``` Returns one line: `resolved → (, )`. All search noise stays on disk. diff --git a/skills/media-use/references/audio.md b/skills/media-use/references/audio.md index 504d630332..ec27bc138c 100644 --- a/skills/media-use/references/audio.md +++ b/skills/media-use/references/audio.md @@ -11,7 +11,7 @@ node /audio/scripts/audio.mjs --request ./audio_request.json --out ./ - **Request** `{ provider?, lang?, speed?, lines: [{ id, text, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`. - **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`. -- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. Local/provider-specific generators are explicit alternatives where installed; run `node /scripts/resolve.mjs --doctor` before assuming retrieval or TTS will work. +- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work. - If BGM took the generate path (`bgm_pending: true`), run `audio/scripts/wait-bgm.mjs` before final render. Single-shot helpers: `audio/scripts/heygen-tts.mjs` (one voice file). Transcription / background removal / captions use the `hyperframes` CLI (`transcribe`, `remove-background`), see the per-topic guides in `audio/references/` (`tts.md`, `bgm.md`, `sfx.md`, `transcribe.md`, `remove-background.md`, `captions/`). diff --git a/skills/media-use/references/grading.md b/skills/media-use/references/grading.md index 2292eef785..a86503d3c6 100644 --- a/skills/media-use/references/grading.md +++ b/skills/media-use/references/grading.md @@ -20,7 +20,7 @@ assemble those from a generic LUT plus handmade CSS vignette/grain/opacity. **Never `cat`/read a `.cube` file into context.** A 3D LUT is ~size^3 lines of raw numbers (33^3 ≈ 36k lines at the default size). It bloats context and carries zero human/agent-legible signal. To understand or choose a LUT, use `hyperframes grade-compare` to see it rendered, or `cube-validate.mjs` for a one-line `{ok,size}` check. Read `.media/index.md` or `luts/index.json` for the description. Never read the LUT body itself. ```bash -node /scripts/resolve.mjs --type grade --intent "warm daylight" --project . --json +npx hyperframes media-use resolve --type grade --intent "warm daylight" --project . --json ``` Preset-first output uses the core runtime vocabulary and does not freeze a file: @@ -93,7 +93,7 @@ For a reusable color transform beyond the preset vocabulary, freeze a validated `.cube` under `.media/luts/` and return a block that references it: ```bash -node /scripts/resolve.mjs --type grade --intent "teal orange blockbuster" --project . --json +npx hyperframes media-use resolve --type grade --intent "teal orange blockbuster" --project . --json ``` ```json @@ -106,20 +106,20 @@ node /scripts/resolve.mjs --type grade --intent "teal orange blockbus Use `lut` when you only need the reusable `.cube` file: ```bash -node /scripts/resolve.mjs --type lut --intent "teal orange blockbuster" --project . +npx hyperframes media-use resolve --type lut --intent "teal orange blockbuster" --project . ``` For a describable technical look, author an explicit parametric LUT with `--params`: ```bash -node /scripts/resolve.mjs --type lut --params '{"contrast":0.2,"temperature":-0.3}' --project . -node /scripts/resolve.mjs --type grade --params '{"exposure":0.2}' --project . --json +npx hyperframes media-use resolve --type lut --params '{"contrast":0.2,"temperature":-0.3}' --project . +npx hyperframes media-use resolve --type grade --params '{"exposure":0.2}' --project . --json ``` For a LUT generated by your own script, ingest it with `--from`; media-use validates it before registration and rejects invalid or oversized cubes: ```bash -node /scripts/resolve.mjs --type lut --from custom.cube --project . +npx hyperframes media-use resolve --type lut --from custom.cube --project . ``` Parametric math (`buildCube`) cannot reproduce real film stocks or emulsion @@ -155,6 +155,6 @@ validates generated or downloaded cubes as it freezes them under `.media/luts/`. ```bash -node skills/media-use/scripts/resolve.mjs --type lut --intent "teal orange blockbuster" --project . --json +npx hyperframes media-use resolve --type lut --intent "teal orange blockbuster" --project . --json node skills/media-use/scripts/lib/cube-validate.mjs .media/luts/lut_001.cube ``` diff --git a/skills/media-use/references/media-treatment-recipes.md b/skills/media-use/references/media-treatment-recipes.md index 37903ab58f..86517df083 100644 --- a/skills/media-use/references/media-treatment-recipes.md +++ b/skills/media-use/references/media-treatment-recipes.md @@ -784,7 +784,7 @@ node /scripts/dither.mjs \ --palette '#17121a,#824c50,#e09873,#f7ddb1' \ --point-size 3 -node /scripts/resolve.mjs \ +npx hyperframes media-use resolve \ --from .media/generated/video_001.atkinson.mp4 --type video --project . ``` diff --git a/skills/media-use/references/meta.md b/skills/media-use/references/meta.md index 776d1ed5d5..629963a9ee 100644 --- a/skills/media-use/references/meta.md +++ b/skills/media-use/references/meta.md @@ -26,7 +26,7 @@ HyperFrames owns media _playback_; media-use owns everything else. Each row is e Use `resolve --stats` for a local, shareable report over the current project's `.media/` manifest, the global `~/.media/` cache, and local resolve misses. Human output is compact; add `--json` for a single machine-readable object, and `--days N` to window timestamped records. ```bash -node /scripts/resolve.mjs --stats --project . --days 7 +npx hyperframes media-use resolve --stats --project . --days 7 # media-use stats # total resolves: 12 # misses: 2 diff --git a/skills/media-use/references/operations.md b/skills/media-use/references/operations.md index 0755454dcb..e2909f6c01 100644 --- a/skills/media-use/references/operations.md +++ b/skills/media-use/references/operations.md @@ -79,7 +79,7 @@ node /scripts/dither.mjs \ --palette '#0f380f,#306230,#8bac0f,#9bbc0f' \ --point-size 3 -node /scripts/resolve.mjs \ +npx hyperframes media-use resolve \ --from source.atkinson.mp4 --type video --project . ``` diff --git a/skills/media-use/references/resolve.md b/skills/media-use/references/resolve.md index 091483f4ad..16af9637e2 100644 --- a/skills/media-use/references/resolve.md +++ b/skills/media-use/references/resolve.md @@ -1,7 +1,7 @@ # Resolve — command, flags, reuse, adopt, inventory ```bash -node /scripts/resolve.mjs --type --intent "" --project +npx hyperframes media-use resolve --type --intent "" --project ``` Returns one line: `resolved → (, )` @@ -23,31 +23,31 @@ Returns one line: `resolved → (, )` ```bash # Background music -node /scripts/resolve.mjs --type bgm --intent "upbeat tech launch" --project . +npx hyperframes media-use resolve --type bgm --intent "upbeat tech launch" --project . # → resolved bgm_001 → .media/audio/bgm/bgm_001.mp3 (bgm, 25s) # Sound effect -node /scripts/resolve.mjs --type sfx --intent "whoosh" --project . +npx hyperframes media-use resolve --type sfx --intent "whoosh" --project . # → resolved sfx_001 → .media/audio/sfx/sfx_001.mp3 (sfx, 0.57s) # Image -node /scripts/resolve.mjs --type image --intent "gradient tech background" --project . +npx hyperframes media-use resolve --type image --intent "gradient tech background" --project . # → resolved image_001 → .media/images/image_001.jpg (image) # Icon -node /scripts/resolve.mjs --type icon --intent "rocket" --project . +npx hyperframes media-use resolve --type icon --intent "rocket" --project . # → resolved icon_001 → .media/images/icon_001.png (icon, transparent) # Brand logo (official mark — never redrawn by hand) -node /scripts/resolve.mjs --type logo --entity linkedin --intent "LinkedIn logo" --project . +npx hyperframes media-use resolve --type logo --entity linkedin --intent "LinkedIn logo" --project . # → resolved logo_001 → .media/images/logo_001.svg (logo, official mark) # Color grade block -node /scripts/resolve.mjs --type grade --intent "warm daylight" --project . --json +npx hyperframes media-use resolve --type grade --intent "warm daylight" --project . --json # → {"ok":true,"preset":"warm-daylight","grading":{"preset":"warm-daylight","intensity":1},...} # LUT file -node /scripts/resolve.mjs --type lut --intent "teal orange blockbuster" --project . +npx hyperframes media-use resolve --type lut --intent "teal orange blockbuster" --project . # → resolved lut_001 → .media/luts/lut_001.cube (lut) ``` @@ -76,7 +76,7 @@ node /scripts/resolve.mjs --type lut --intent "teal orange blockbuste Before resolving bgm/sfx/image/icon/logo/grade/lut, **check what already exists and reuse it when it fits.** media-use does not semantically match for you — you are the judge. It surfaces candidates; you decide. ```bash -node /scripts/resolve.mjs --type bgm --intent "upbeat tech launch" --candidates --project . +npx hyperframes media-use resolve --type bgm --intent "upbeat tech launch" --candidates --project . # [project] upbeat tech launch (25s, heygen.audio.sounds) # .media/audio/bgm/bgm_001.wav # [global] energetic tech intro (22s, heygen.audio.sounds) @@ -121,7 +121,7 @@ tracks today. Most HyperFrames projects already have assets in `assets/`. media-use adopts them: ```bash -node /scripts/resolve.mjs --adopt --project . +npx hyperframes media-use resolve --adopt --project . # → adopted 9 assets from assets/ # bgm_001 → assets/bgm/mango-fizz.mp3 (bgm, 146.6s) # image_001 → assets/images/avatar.jpg (image, 400×400) diff --git a/skills/media-use/references/setup-providers.md b/skills/media-use/references/setup-providers.md index ef5329e726..316d524eeb 100644 --- a/skills/media-use/references/setup-providers.md +++ b/skills/media-use/references/setup-providers.md @@ -12,7 +12,7 @@ heygen auth login --oauth # OAuth = free subscription credits; --api-key bills A This unlocks the FREE path for bgm/sfx/image/icon catalog search, TTS (voice), and avatar videos. Sign in with `--oauth` — the free allowance rides on the OAuth session (an API key bills API credits instead). **media-use requires heygen >= v0.3.0 uniformly** (the OAuth free-usage path needs it), so `--doctor` nudges older CLIs to update even for API-key-only use. Before resolving anything, verify setup with: ```bash -node /scripts/resolve.mjs --doctor +npx hyperframes media-use resolve --doctor ``` ## Providers diff --git a/skills/media-use/scripts/compatibility.test.mjs b/skills/media-use/scripts/compatibility.test.mjs new file mode 100644 index 0000000000..7e3cd5bf8e --- /dev/null +++ b/skills/media-use/scripts/compatibility.test.mjs @@ -0,0 +1,27 @@ +import { cpSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +import test from "node:test"; + +function run(scriptsDir, script, args) { + return execFileSync(process.execPath, [join(scriptsDir, script), ...args], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); +} + +test("published memory scripts run without reaching outside the skill", () => { + const scratchDir = mkdtempSync(join(tmpdir(), "media-use-compat-")); + const skillDir = join(scratchDir, "media-use"); + cpSync(new URL("..", import.meta.url), skillDir, { recursive: true }); + const scriptsDir = join(skillDir, "scripts"); + const projectDir = join(scratchDir, "project"); + try { + run(scriptsDir, "prefs.mjs", ["get", "--hyperframes", projectDir, "--json"]); + run(scriptsDir, "recipe.mjs", ["list", "--hyperframes", projectDir, "--json"]); + run(scriptsDir, "transcribe.mjs", ["--help"]); + } finally { + rmSync(scratchDir, { recursive: true, force: true }); + } +}); diff --git a/skills/media-use/scripts/lib/duck.mjs b/skills/media-use/scripts/lib/duck.mjs index 6437f7ff06..9bbe7afda0 100644 --- a/skills/media-use/scripts/lib/duck.mjs +++ b/skills/media-use/scripts/lib/duck.mjs @@ -1,15 +1,6 @@ import { wordListsFromMediaMeta } from "./words.mjs"; -/** - * Speech spans from word timestamps. - * - * audio_meta.json word times are relative to EACH LINE'S OWN FILE, not to the - * composition. Without placement info, multiple lines would overlap at t=0 and - * merge into one bogus span. Placement options: - * offsets: { [voiceId]: startSeconds } explicit composition placement - * sequential: stack lines back to back (plus `gap` seconds between lines) - * A single word list (bare transcript) needs neither. - */ +// audio_meta.json word times are relative to each line's own file. export function speechSpans(meta, { mergeGap = 0.6, offsets, sequential = false, gap = 0 } = {}) { const merge = Number(mergeGap); const lists = wordListsFromMediaMeta(meta); diff --git a/skills/media-use/scripts/lib/media-fetch.mjs b/skills/media-use/scripts/lib/media-fetch.mjs index 830e41618a..5c0e362684 100644 --- a/skills/media-use/scripts/lib/media-fetch.mjs +++ b/skills/media-use/scripts/lib/media-fetch.mjs @@ -1,71 +1,6 @@ -// Media downloads use public HTTP(S) URLs. Validate every redirect target; -// a provider result must meet the same host policy as a direct ingest URL. -// Public HTTPS-to-HTTP redirects are allowed, matching direct HTTP support. -// This is a literal-host policy, not DNS pinning: DNS resolution remains trusted. - -import { BlockList, isIP } from "node:net"; - -const blocked = new BlockList(); -for (const [network, prefix] of [ - ["0.0.0.0", 8], - ["10.0.0.0", 8], - ["100.64.0.0", 10], - ["127.0.0.0", 8], - ["169.254.0.0", 16], - ["172.16.0.0", 12], - ["192.0.0.0", 24], - ["192.0.2.0", 24], - ["192.88.99.0", 24], - ["192.168.0.0", 16], - ["198.18.0.0", 15], - ["198.51.100.0", 24], - ["203.0.113.0", 24], - ["224.0.0.0", 4], - ["240.0.0.0", 4], -]) - blocked.addSubnet(network, prefix, "ipv4"); -for (const [network, prefix] of [ - ["::", 128], - ["::1", 128], - ["fc00::", 7], - ["fe80::", 10], - ["fec0::", 10], - ["ff00::", 8], - ["2001:db8::", 32], -]) - blocked.addSubnet(network, prefix, "ipv6"); - -export function isPublicMediaUrl(value) { - try { - const url = new URL(value); - if (url.protocol !== "http:" && url.protocol !== "https:") return false; - const host = url.hostname.replace(/\.$/, ""); - if ( - host === "localhost" || - host.endsWith(".localhost") || - host.endsWith(".local") || - host.endsWith(".internal") - ) - return false; - const address = host.replace(/^\[|\]$/g, ""); - const family = isIP(address); - return family === 0 || !blocked.check(address, family === 4 ? "ipv4" : "ipv6"); - } catch { - return false; - } -} - -export async function fetchMedia(url, { method = "GET", signal, fetchImpl = fetch } = {}) { - let current = String(url); - for (let hop = 0; hop <= 5; hop++) { - if (!isPublicMediaUrl(current)) - throw new Error("Media download blocked: URL is not public HTTP(S)"); - const response = await fetchImpl(current, { method, signal, redirect: "manual" }); - if (!(response.status >= 300 && response.status < 400)) return response; - const location = response.headers.get("location"); - if (!location) return response; - await response.body?.cancel(); - current = new URL(location, current).href; - } - throw new Error("Media download exceeded redirect limit"); -} +// Compatibility copy for the audio helpers that still import the historical +// skill-relative path. The CLI-owned engine is the source of truth. +export { + fetchMedia, + isPublicMediaUrl, +} from "../../../../packages/cli/src/media-use/lib/media-fetch.mjs"; diff --git a/skills/media-use/scripts/lib/npx-sync.mjs b/skills/media-use/scripts/lib/npx-sync.mjs index 8ada152d05..b1049cd2b8 100644 --- a/skills/media-use/scripts/lib/npx-sync.mjs +++ b/skills/media-use/scripts/lib/npx-sync.mjs @@ -1,5 +1,18 @@ import { existsSync } from "node:fs"; -import { resolveSpawnCommand } from "../../audio/scripts/lib/tts.mjs"; +import { dirname, join } from "node:path"; + +function resolveNpxCliPath(env, pathExists) { + const npmExecPath = env.npm_execpath; + const nodeExecPath = env.npm_node_execpath || process.execPath; + if (npmExecPath) { + const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase(); + const candidate = + fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js"); + if (pathExists(candidate)) return candidate; + } + const besideNode = join(dirname(nodeExecPath), "node_modules", "npm", "bin", "npx-cli.js"); + return pathExists(besideNode) ? besideNode : null; +} // Sync-spawn analog of the audio engine's spawnP, for execFileSync call sites // that must hard-fail (rather than fall through to another provider) when npx @@ -18,7 +31,20 @@ export function resolveNpxInvocation( env = process.env, pathExists = existsSync, ) { - const resolved = resolveSpawnCommand("npx", argv, opts, platform, env, pathExists); + const resolved = + platform !== "win32" + ? { cmd: "npx", args: argv, opts: { stdio: "ignore", ...opts } } + : (() => { + const nodeExecPath = env.npm_node_execpath || process.execPath; + const npxCliPath = resolveNpxCliPath(env, pathExists); + return npxCliPath + ? { + cmd: nodeExecPath, + args: [npxCliPath, ...argv.map((arg) => String(arg))], + opts: { stdio: "ignore", windowsHide: true, ...opts }, + } + : null; + })(); if (!resolved) { // npx-on-win32 with no resolvable npx-cli.js — same terminal condition // spawnP warns about, surfaced as a throw for callers with no fallback. diff --git a/skills/media-use/scripts/lib/telemetry.mjs b/skills/media-use/scripts/lib/telemetry.mjs index 43603d7629..cd2911f529 100644 --- a/skills/media-use/scripts/lib/telemetry.mjs +++ b/skills/media-use/scripts/lib/telemetry.mjs @@ -1,15 +1,5 @@ -// Opt-out usage tracking for media-use, sharing the hyperframes CLI/studio -// identity (packages/cli/src/telemetry): the same install id from -// ~/.hyperframes/config.json, plus a $identify to the HeyGen account on sign-in, -// so a person is one PostHog profile across surfaces — not a fresh id per tool. -// Not fully anonymous by design (it must dedupe): pseudonymous before sign-in, -// account-linked after. Event PROPERTIES stay coarse — media TYPE, resolution -// SOURCE, winning PROVIDER — never the intent text, file names, or paths. -// -// Same public PostHog project key as the CLI (a write-only ingestion key, safe -// to ship), same opt-outs (DO_NOT_TRACK / HYPERFRAMES_NO_TELEMETRY / CI / dev), -// and $ip:null so no IP is recorded. Fire-and-forget: telemetry never blocks a -// resolve and never throws into it. +// Usage tracking shares the CLI and Studio identity. Properties stay coarse and +// never carry intent text, file names, or paths. import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; @@ -22,10 +12,6 @@ const TIMEOUT_MS = 1500; let identifiedAccount = false; let warnedNonDefaultHost = false; -// Same CI/test signals the test suite itself sets (resolve.test.mjs's U7 test -// sets NODE_ENV=test and clears CI to prove the interception seam works) — -// reused here, not a new heuristic, so that deliberate test usage never -// triggers the warning below. function isTestOrCiContext() { return ( process.env.CI === "true" || @@ -35,16 +21,6 @@ function isTestOrCiContext() { ); } -// Test-only interception seam: a real HTTP destination a test can point at, -// so a spawned-child test (resolve.test.mjs) can prove track() never reaches -// production rather than trusting DO_NOT_TRACK alone (a future call site or -// test could forget to set that env var). Falls back to the real production -// host whenever unset — production behavior is unchanged. -// -// Safety net: if this ever leaks into a real user's shell, track() would -// silently redirect to a likely-dead host and postBatch()'s catch{} would -// swallow the failure with zero signal. Surface one stderr warning outside -// test/CI contexts so a real user gets some indication instead of silence. function posthogHost() { const override = process.env.MEDIA_USE_TELEMETRY_HOST; if (override && !warnedNonDefaultHost && !isTestOrCiContext()) { @@ -67,12 +43,7 @@ export function optedOut() { ); } -// CLI + studio share one install identity in ~/.hyperframes/config.json -// (packages/cli/src/telemetry/config.ts — same path, same `anonymousId` / -// `telemetryNoticeShown` fields). Read and write that same file so media-use is -// the same PostHog person and shows the notice once per person, not per tool. -// Computed per call (not a module const) so it honors HOME at runtime — tests -// sandbox HOME, and os.homedir() re-reads it each call. +// Read and write the shared config so media-use keeps one identity per install. function sharedConfigPath() { return join(homedir(), ".hyperframes", "config.json"); } diff --git a/skills/media-use/scripts/resolve.mjs b/skills/media-use/scripts/resolve.mjs index 3e6c36f021..84480705b2 100644 --- a/skills/media-use/scripts/resolve.mjs +++ b/skills/media-use/scripts/resolve.mjs @@ -1,1271 +1,17 @@ #!/usr/bin/env node - import { spawnSync } from "node:child_process"; -import { existsSync, statSync, writeFileSync, renameSync, rmSync } from "node:fs"; -import { resolve, join, extname, basename } from "node:path"; -import { parseArgs } from "node:util"; -import { - appendRecord, - findByPrompt, - findByEntity, - nextId, - withReservedFile, - withReservedFileSync, -} from "./lib/manifest.mjs"; -import { regenerateIndex } from "./lib/index-gen.mjs"; -import { cacheGet, cacheGetByEntity, importFromCache, cachePut } from "./lib/cache.mjs"; -import { - runCapability, - listTypes, - providerMatches, - providerNamesFor, - providerTierFor, -} from "./lib/registry.mjs"; -import { freezeUrl, freezeLocalFile, isDirectMediaUrl } from "./lib/freeze.mjs"; -import { findExistingAsset } from "./lib/adopt.mjs"; -import { track } from "./lib/telemetry.mjs"; -import { recordMiss } from "./lib/misses.mjs"; -import { buildStats } from "./lib/stats.mjs"; -import { typesMatch } from "./lib/match.mjs"; -import { listCandidates, formatCandidates, CANDIDATE_CAP } from "./lib/candidates.mjs"; -import { findGlobalBySha } from "./lib/cache.mjs"; -import { heygenAuthMethod } from "../audio/scripts/lib/heygen.mjs"; -import { buildCube, paramsFromIntent } from "./lib/cube-build.mjs"; -import { validateCubeFile } from "./lib/cube-validate.mjs"; -import { analyzeMediaGrade, formatMeasuredNote } from "./lib/grade-analyzer.mjs"; -import { - freezeLibraryLut, - isLibraryLutOfflineMiss, - matchColorLook, -} from "./lib/lut-preset-provider.mjs"; -import { - HEYGEN_AUTH_COMMAND, - HEYGEN_INSTALL_COMMAND, - HEYGEN_MIN_VERSION, - HEYGEN_UPDATE_COMMAND, - consumeHeygenRemediation, - firstSemver, - flushHeygenFailureTracking, - versionLessThan, -} from "./lib/heygen-cli.mjs"; -import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs"; - -const INGEST_TYPES = listTypes(); -const DEFAULT_EXT = { - bgm: ".wav", - sfx: ".mp3", - voice: ".wav", - image: ".jpg", - icon: ".svg", - logo: ".svg", - brand: ".png", - video: ".mp4", - grade: ".cube", - lut: ".cube", -}; - -// resolve shells `fetch`/`freezeUrl` and modern ESM; 18 is the floor where those -// exist without flags. Named so the --doctor node check verifies something real -// (O2). Declared before the top-level `--doctor` branch that calls runDoctor(). -const MIN_NODE_VERSION = "18.0.0"; - -const { values: args } = parseArgs({ - options: { - type: { type: "string", short: "t" }, - intent: { type: "string", short: "i" }, - entity: { type: "string", short: "e" }, - project: { type: "string", short: "p", default: "." }, - adopt: { type: "boolean", default: false }, - candidates: { type: "boolean", default: false }, - doctor: { type: "boolean", default: false }, - stats: { type: "boolean", default: false }, - days: { type: "string" }, - "dry-run": { type: "boolean", default: false }, - reuse: { type: "string" }, - from: { type: "string" }, - params: { type: "string" }, - for: { type: "string" }, - analyze: { type: "boolean", default: false }, - "local-only": { type: "boolean", default: false }, - provider: { type: "string" }, - "avatar-id": { type: "string" }, - "voice-id": { type: "string" }, - json: { type: "boolean", default: false }, - help: { type: "boolean", short: "h", default: false }, - }, - strict: true, -}); - -if (args.help) { - console.log(`media-use resolve — turn a media need into a frozen local file - -Usage: - node resolve.mjs --type --intent "" [--project ] - -Types: ${listTypes().join(", ")} - -Options: - --type, -t Media type (required) - --intent, -i What you need (required) - --entity, -e Entity name for cache matching (optional) - --project, -p Project directory (default: .) - --adopt Adopt all existing assets/ files into the manifest - --candidates List reusable assets (project + global cache) for --type; no - download, no mutation. Read them and decide reuse yourself. - --doctor Check local CLI dependencies; no manifest changes. - --stats Print local usage stats from .media and ~/.media; no mutation. - --days Limit --stats to records/misses from the last N days when - timestamps are available. - --reuse Import a specific global-cache asset (by content sha/prefix, - from --candidates) into this project - --from Freeze a local file or direct public URL (ingest) - --params Build an explicit parametric LUT (lut/grade only) - --for Analyze a local image/video and add measured grade adjust - suggestions (grade only) - --analyze Return --for grade evidence without recording a candidate - --local-only Offline: skip every network provider - --provider Force one generator (e.g. codex, mflux, kokoro, heygen) - --avatar-id Override the default avatar for heygen.video generation - --voice-id Override the default voice for voice/heygen.video generation - --json Output JSON instead of one-line result - --help, -h Show this help`); - process.exit(0); -} - -const projectDir = resolve(args.project); -const type = args.type; -const intent = args.intent; -const entity = args.entity || null; - -if (args.adopt) { - const { adoptExistingAssets } = await import("./lib/adopt.mjs"); - const adopted = adoptExistingAssets(projectDir); - if (args.json) { - console.log(JSON.stringify({ ok: true, adopted: adopted.length, assets: adopted })); - } else if (adopted.length === 0) { - console.log("no new assets to adopt (assets/ empty or already registered)"); - } else { - console.log(`adopted ${adopted.length} asset${adopted.length === 1 ? "" : "s"} from assets/`); - for (const r of adopted) console.log(` ${r.id} → ${r.path} (${r.type})`); - } - process.exit(0); -} - -// Candidates: side-effect-free listing of reusable assets (project + global -// cache) for --type. No download, no provider, no mutation. The agent reads -// these and decides semantic fit itself. -if (args.candidates || args["dry-run"]) { - await showCandidates(); - process.exit(0); -} - -if (args.doctor) { - const doctor = runDoctor(); - const failed = doctor.checks.filter((check) => !check.ok); - // Non-PII: instrument the exact question the feature exists to answer — how - // often is --doctor run and which check fails most. Awaited so a short-lived - // run flushes before exit. - await track("media_use_doctor_run", { - ok: doctor.ok, - checks_failed: failed.length, - failed: failed.map((check) => check.name), - }); - if (args.json) { - console.log(JSON.stringify({ ok: doctor.ok, checks: doctor.checks })); - } else { - printDoctor(doctor.checks); - } - process.exit(doctor.ok ? 0 : 1); -} - -if (args.stats) { - const report = buildStats({ - projectDir, - days: args.days ? Number(args.days) : undefined, - }); - if (args.json) { - console.log(JSON.stringify(report)); - } else { - printStats(report); - } - process.exit(0); -} - -// Reuse: import a specific global-cache asset (by content sha/prefix, taken -// from --candidates) into this project. `!== undefined` so an empty --reuse "" -// still routes here (and gets a clear empty-sha error) instead of falling -// through to the misleading "--type and --intent are required". -if (args.reuse !== undefined) { - await reuseGlobal(args.reuse); - process.exit(0); -} - -// Ingest: freeze a user-supplied local file or direct public URL (no search). -if (args.from) { - await ingest(args.from); - process.exit(0); -} - -if (args.analyze) { - if (type !== "grade" || !args.for) { - console.error("error: --analyze requires --type grade and --for "); - process.exit(2); - } - const mediaPath = resolve(args.for); - if (!existsSync(mediaPath)) { - console.error(`error: --for file not found: ${mediaPath}`); - process.exit(2); - } - const analysis = analyzeMediaGrade(mediaPath); - if (args.json) { - console.log(JSON.stringify({ ok: true, type: "grade-analysis", ...analysis })); - } else { - console.log(formatMeasuredNote(mediaPath, analysis.measured)); - console.log(`suggested adjust: ${JSON.stringify(analysis.adjust)}`); - } - process.exit(0); -} - -// Recipes: folder-based named bundles resolved by entity name — no providers, -// no content hashing (an evolving versioned bundle, not an immutable file). -// Delegates to lib/recipe-store.mjs the way grade/lut delegate to resolveColor; -// freeze/list live in scripts/recipe.mjs. -if (type === "recipe") { - const { useRecipe } = await import("./lib/recipe-store.mjs"); - const name = (entity || intent || "").trim(); - if (!name) exitError("--type recipe needs --entity (or --intent )", 2); - try { - const used = useRecipe({ projectDir, name }); - if (args.json) { - console.log(JSON.stringify({ ok: true, ...used })); - } else { - console.log( - `resolved recipe ${used.recipe.name} (v${used.recipe.version}, ${used.recipe.workflow})`, - ); - console.log(` frame spec → ${used.frameSpecPath} (copied over)`); - console.log(` storyboard skeleton → ${used.skeletonPath}`); - if (used.briefSkeletonPath) console.log(` brief skeleton → ${used.briefSkeletonPath}`); - } - process.exit(0); - } catch (err) { - exitError(err.message, 1); - } -} - -if (args.params !== undefined) { - if (type !== "lut" && type !== "grade") { - exitError( - type - ? `--params only supports --type lut or grade (got ${type})` - : "--params requires --type lut or grade", - 2, +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +const root = join(fileURLToPath(new URL("../../../", import.meta.url))); +const dist = join(root, "packages/cli/dist/cli.js"); +const result = existsSync(dist) + ? spawnSync(process.execPath, [dist, "media-use", "resolve", ...process.argv.slice(2)], { + stdio: "inherit", + }) + : spawnSync( + "bun", + [join(root, "packages/cli/src/cli.ts"), "media-use", "resolve", ...process.argv.slice(2)], + { stdio: "inherit" }, ); - } - try { - await runParams(); - process.exit(0); - } catch (err) { - exitError(err.message, 1); - } -} - -if (!args.type || !args.intent || !args.intent.trim()) { - console.error("error: --type and a non-empty --intent are required"); - process.exit(2); -} - -if (!listTypes().includes(args.type)) { - console.error(`error: unknown media type: ${args.type} (known: ${listTypes().join(", ")})`); - process.exit(2); -} - -// Forced-provider validation: reject an unknown/unavailable provider name up -// front so a typo reads as a typo, not a catalog miss (`no provider could -// resolve`). Match rule mirrors runProviders (full name or dotted prefix). -if (args.provider && !providerMatches(args.type, args.provider)) { - console.error( - `error: unknown provider "${args.provider}" for type ${args.type} (available: ${providerNamesFor(args.type).join(", ")})`, - ); - process.exit(2); -} - -function recordAvailable(projectDir, record) { - if (!record) return false; - if (record.path) return existsSync(join(projectDir, record.path)); - return record.type === "grade" && record.grading; -} - -// Sparse `{ authMethod }` for a heygen-family provider name (e.g. "heygen.tts"), -// else `{}` — keeps auth_method telemetry absent for every non-heygen resolve -// instead of implying an auth method that doesn't apply. -function heygenAuthMethodFor(provider) { - if (!provider || !provider.startsWith("heygen.")) return {}; - const authMethod = heygenAuthMethod(); - return authMethod ? { authMethod } : {}; -} - -function localizeImportedRecord(record, localPath) { - if (record?.type === "grade" && record.grading?.lut) { - record.grading = { - ...record.grading, - lut: { ...record.grading.lut, src: localPath }, - }; - } - return record; -} - -async function run() { - // A forced --provider means "(re)generate with THIS provider" — it bypasses - // every reuse rung (project/entity/assets/global cache) so it can't silently - // hand back an asset from a different provider. The floor only applies to the - // default (unforced) cascade. - const forced = !!args.provider; - - // 1. project manifest — exact-prompt match - const projectHit = forced ? null : findByPrompt(projectDir, intent, type); - if (recordAvailable(projectDir, projectHit)) { - return result(projectHit, "cached"); - } - - // 1b. entity match in project. icon and image are interchangeable for - // entity hits — both live in images/, and figma-imported brand marks are - // always recorded as type image while agents ask for logos as type icon. - if (!forced && entity) { - const entityHit = findByEntity(projectDir, entity); - if (entityHit && typesMatch(entityHit.type, type) && recordAvailable(projectDir, entityHit)) { - return result(entityHit, "cached"); - } - } - - // 1c. scan existing assets/ directory for unregistered matches - const existingAsset = - forced || type === "grade" || type === "lut" - ? null - : findExistingAsset(projectDir, intent, type); - if (existingAsset) { - const id = nextId(projectDir, type); - const record = { - id, - type: existingAsset.type, - path: existingAsset.relativePath, - source: "existing", - description: existingAsset.name.replace(/[-_]/g, " "), - provenance: { provider: "local", adopted: true, prompt: intent }, - }; - appendRecord(projectDir, record); - regenerateIndex(projectDir); - return result(record, "existing"); - } - - // 2. global cache — exact-prompt or entity match - const cacheHit = forced ? null : cacheGet(intent, type); - if (cacheHit) { - const ext = extname(cacheHit.cached_path); - const imported = withReservedFileSync(projectDir, type, ext, ({ id, localPath }) => - localizeImportedRecord(importFromCache(cacheHit, projectDir, id, localPath), localPath), - ); - if (imported) { - appendRecord(projectDir, imported); - regenerateIndex(projectDir); - return result(imported, "reused"); - } - } - - if (!forced && entity) { - const entityCacheHit = cacheGetByEntity(entity); - if (entityCacheHit && typesMatch(entityCacheHit.type, type)) { - const ext = extname(entityCacheHit.cached_path); - const imported = withReservedFileSync(projectDir, type, ext, ({ id, localPath }) => - localizeImportedRecord( - importFromCache(entityCacheHit, projectDir, id, localPath), - localPath, - ), - ); - if (imported) { - appendRecord(projectDir, imported); - regenerateIndex(projectDir); - return result(imported, "reused"); - } - } - } - - // Offline guard: --local-only skips every remote provider (HeyGen catalog), - // leaving the project + global cache and any local provider. - const localOnly = args["local-only"]; - const ctx = { - entity, - projectDir, - localOnly, - provider: args.provider, - avatarId: args["avatar-id"], - voiceId: args["voice-id"], - }; - - // Adherence nudge (offline, no auto-reuse): the exact-cache floor missed and - // we're about to fetch/generate. If lexically-similar assets already exist, - // point the agent at --candidates so it can reuse instead of fetching. Only a - // fuzzy match ever reaches the agent this way — never auto-applied. Goes to - // stderr so it reaches --json callers without corrupting stdout. Best-effort. - try { - const { similar } = listCandidates({ projectDir, type, intent, cap: CANDIDATE_CAP }); - if (similar > 0) { - console.error( - `media-use: ${similar} similar cached asset${similar === 1 ? "" : "s"} already ${similar === 1 ? "exists" : "exist"} — run \`resolve --candidates --type ${type} --intent "${intent}"\` to review and reuse instead of fetching.`, - ); - } - } catch { - // hint is best-effort; never block a resolve - } - - if (type === "grade" || type === "lut") { - return resolveColor(type, intent, { projectDir }); - } - - // 3. provider search — registry tries providers in order (heygen-CLI first) - let searchResult = null; - let providerFailure = null; - try { - searchResult = await runCapability(type, "search", intent, ctx); - } catch (error) { - providerFailure = error; - // search failed, try generate - } - - // 4. generate fallback — same ordered cascade for the generate capability - if (!searchResult) { - try { - searchResult = await runCapability(type, "generate", intent, ctx); - } catch (error) { - providerFailure ??= error; - // generate failed too - } - } - - // A search/generate attempt against heygen may have fired a fire-and-forget - // media_use_provider_error track (reportHeygenFailure — heygen-search.mjs / - // voice-provider.mjs are sync call sites several layers below here and can't - // await it themselves). Join it now, before any process.exit() below can - // race it: both it and the miss/success telemetry below are separate, - // non-keepalive HTTP connections with no ordering guarantee otherwise. - await flushHeygenFailureTracking(); - - if (!searchResult) { - await track("media_use_resolve_miss", { - type, - local_only: !!localOnly, - provider_override: !!args.provider, - }); - recordMiss({ - type, - intent, - provider_override: !!args.provider, - local_only: !!args["local-only"], - }); - // brand stays local: no frame.md/design.md -> upsell the HyperFrames design - // flow rather than reporting a generic miss (B5). - const msg = - providerFailure instanceof BundledSfxAssetsError - ? providerFailure.message - : type === "brand" - ? "no brand spec found — add a frame.md or design.md (colors/font/logo) to this project. Run the HyperFrames design flow to create one; brand tokens are read locally for deterministic rendering." - : args.provider - ? `provider "${args.provider}" could not resolve ${type}: "${intent}"${localOnly ? " (--local-only skips network providers; drop it or the --provider override)" : ""}` - : `no provider could resolve ${type}: "${intent}"`; - if (args.json) { - console.log( - JSON.stringify({ - ok: false, - ...(providerFailure instanceof BundledSfxAssetsError - ? { code: providerFailure.code, fix: providerFailure.fix } - : {}), - error: msg, - }), - ); - } else { - console.error(`error: ${msg}`); - } - process.exit(1); - } - - // 5. freeze + register (atomic id+file reservation so concurrent resolves - // can't collide on an id during the download — MU-23) - const ext = searchResult.ext || extFromUrl(searchResult.url || "") || defaultExt(type); - const { id, localPath, fullPath } = await withReservedFile( - projectDir, - type, - ext, - async (reservation) => { - if (searchResult.localPath) { - freezeLocalFile(searchResult.localPath, reservation.fullPath); - } else if (searchResult.url) { - await freezeUrl(searchResult.url, reservation.fullPath); - } else { - throw new Error("provider returned no url or localPath"); - } - return reservation; - }, - ); - - const record = { - id, - type, - path: localPath, - source: searchResult.source || "search", - description: searchResult.metadata?.description || intent, - ...(searchResult.metadata?.duration != null && { - duration: Math.round(searchResult.metadata.duration * 10) / 10, // round to 0.1s like probe (voice bypassed it) - }), - ...(searchResult.metadata?.width != null && { width: searchResult.metadata.width }), - ...(searchResult.metadata?.height != null && { height: searchResult.metadata.height }), - ...(searchResult.metadata?.transparent != null && { - transparent: searchResult.metadata.transparent, - }), - ...(entity && { entity }), - provenance: { - provider: searchResult.metadata?.provider || "unknown", - prompt: intent, - // heygenAuthMethodFor spreads first so an explicit authMethod on a - // future provider's own metadata.provenance can still override it below - // -- safe today (no provider sets authMethod itself), but keep this - // ordering if that ever changes. - ...heygenAuthMethodFor(searchResult.metadata?.provider), - ...searchResult.metadata?.provenance, - }, - }; - - const heygenRemediation = consumeHeygenRemediation(); - if ( - searchResult.metadata?.provider === "bundled.sfx" && - !localOnly && - !args.provider && - heygenRemediation - ) { - record.advisory = heygenRemediation; - } - - appendRecord(projectDir, record); - regenerateIndex(projectDir); - // Auto-promote: surface every fetched asset in the global cache so it's - // reusable across all hyperframes projects (B3). Non-fatal; dedup by sha. - // ponytail: promotes search/generate/ingest assets (the ones media-use - // fetched), not bulk --adopt imports — add those if cross-project reuse of - // pre-existing project assets is wanted. - try { - cachePut(fullPath, record); - } catch { - // promotion is best-effort; a resolve still succeeds locally - } - return result(record, searchResult.source || "search"); -} - -function mergeSmartAdjust(block) { - if (!args.for) return block; - const mediaPath = resolve(args.for); - // Clear upfront error beats an ffmpeg "No such file" stack on a typo'd path. - if (!existsSync(mediaPath)) throw new Error(`--for file not found: ${mediaPath}`); - const analysis = analyzeMediaGrade(mediaPath); - console.error(formatMeasuredNote(mediaPath, analysis.measured)); - return { - ...block, - adjust: { - ...(block.adjust || {}), - ...analysis.adjust, - }, - }; -} - -function freezeGeneratedLut( - params, - { - projectDir, - type, - description = "parametric color grade", - validationErrorPrefix = "generated LUT failed validation", - }, -) { - return withReservedFileSync(projectDir, type, ".cube", ({ id, localPath, fullPath }) => { - const tmpPath = `${fullPath}.tmp`; - try { - // Write + validate at .tmp, then atomic rename, so a crash between write and - // validate can't leave an invalid .cube at the final path. - writeFileSync(tmpPath, buildCube(params)); - const check = validateCubeFile(tmpPath); - if (!check.ok) throw new Error(check.error); - renameSync(tmpPath, fullPath); - } catch (err) { - rmSync(tmpPath, { force: true }); - throw new Error(`${validationErrorPrefix}: ${err.message}`); - } - return { - id, - localPath, - fullPath, - lut: { src: localPath, intensity: 1 }, - source: "generated", - description, - metadata: { - provider: "cube_lut.builder", - provenance: { params }, - }, - }; - }); -} - -function exitError(message, status = 1) { - if (args.json) { - console.log(JSON.stringify({ ok: false, error: message })); - } else { - console.error(`error: ${message}`); - } - process.exit(status); -} - -function parseExplicitParams() { - try { - return JSON.parse(args.params); - } catch (err) { - throw new Error(`invalid --params JSON: ${err.message}`); - } -} - -async function runParams() { - if (type === "lut" && args.for) { - throw new Error("--for is only supported with --type grade"); - } - const params = parseExplicitParams(); - const description = - typeof intent === "string" && intent.trim() - ? intent.trim() - : `custom parametric ${type === "lut" ? "lut" : "grade"}`; - const frozen = freezeGeneratedLut(params, { - projectDir, - type, - description, - validationErrorPrefix: "--params produced an invalid LUT", - }); - const record = { - id: frozen.id, - type, - path: frozen.localPath, - source: frozen.source, - description: frozen.description, - ...(type === "grade" && { grading: mergeSmartAdjust({ intensity: 1, lut: frozen.lut }) }), - provenance: { - provider: frozen.metadata.provider, - ...frozen.metadata.provenance, - }, - }; - return finalizeColorRecord(record, frozen.source, frozen.fullPath); -} - -async function finalizeColorRecord(record, source, fullPath = null) { - appendRecord(projectDir, record); - regenerateIndex(projectDir); - if (fullPath) { - try { - cachePut(fullPath, record); - } catch { - // promotion is best-effort - } - } - return result(record, source); -} - -async function colorMiss(type, intent) { - await track("media_use_resolve_miss", { - type, - local_only: !!args["local-only"], - provider_override: !!args.provider, - }); - recordMiss({ - type, - intent, - provider_override: !!args.provider, - local_only: !!args["local-only"], - }); - const msg = `no local color grade could resolve ${type}: "${intent}"`; - if (args.json) { - console.log(JSON.stringify({ ok: false, error: msg })); - } else { - console.error(`error: ${msg}`); - } - process.exit(1); -} - -async function resolveGrade(intent, { projectDir }) { - const match = matchColorLook(intent); - if (match?.kind === "preset") { - const id = nextId(projectDir, "grade"); - const grading = mergeSmartAdjust({ preset: match.preset, intensity: 1 }); - const record = { - id, - type: "grade", - source: "preset", - description: intent, - grading, - provenance: { - provider: "color_grade.local", - prompt: intent, - preset: match.preset, - }, - }; - return finalizeColorRecord(record, "preset"); - } - - if (match?.kind === "library") { - let frozen; - try { - frozen = await freezeLibraryLut(match, { - projectDir, - type: "grade", - localOnly: args["local-only"], - }); - } catch (err) { - if (isLibraryLutOfflineMiss(err)) return colorMiss("grade", intent); - throw err; - } - const grading = mergeSmartAdjust({ intensity: 1, lut: frozen.lut }); - const record = { - id: frozen.id, - type: "grade", - path: frozen.localPath, - source: frozen.source, - description: frozen.description, - grading, - provenance: { - provider: frozen.metadata.provider, - prompt: intent, - ...frozen.metadata.provenance, - }, - }; - return finalizeColorRecord(record, frozen.source, frozen.fullPath); - } - - const params = paramsFromIntent(intent); - if (!params) { - // No creative look matched. With --for, the measured adjust block is a - // valid grade on its own (footage auto-correction); only a true miss - // (no look AND no analysis) aborts. - if (args.for) { - const grading = mergeSmartAdjust({ intensity: 1 }); - const record = { - id: nextId(projectDir, "grade"), - type: "grade", - source: "measured", - description: intent, - grading, - provenance: { provider: "color_grade.local", prompt: intent, measured: true }, - }; - return finalizeColorRecord(record, "measured"); - } - return colorMiss("grade", intent); - } - const frozen = freezeGeneratedLut(params, { projectDir, type: "grade" }); - const grading = mergeSmartAdjust({ intensity: 1, lut: frozen.lut }); - const record = { - id: frozen.id, - type: "grade", - path: frozen.localPath, - source: frozen.source, - description: intent, - grading, - provenance: { - provider: frozen.metadata.provider, - prompt: intent, - ...frozen.metadata.provenance, - }, - }; - return finalizeColorRecord(record, frozen.source, frozen.fullPath); -} - -async function resolveLut(intent, { projectDir }) { - if (args.for) { - throw new Error("--for is only supported with --type grade"); - } - const match = matchColorLook(intent); - if (match?.kind === "library") { - let frozen; - try { - frozen = await freezeLibraryLut(match, { - projectDir, - type: "lut", - localOnly: args["local-only"], - }); - } catch (err) { - if (isLibraryLutOfflineMiss(err)) return colorMiss("lut", intent); - throw err; - } - const record = { - id: frozen.id, - type: "lut", - path: frozen.localPath, - source: frozen.source, - description: frozen.description, - provenance: { - provider: frozen.metadata.provider, - prompt: intent, - ...frozen.metadata.provenance, - }, - }; - return finalizeColorRecord(record, frozen.source, frozen.fullPath); - } - - const params = paramsFromIntent(intent); - if (!params) return colorMiss("lut", intent); - const frozen = freezeGeneratedLut(params, { projectDir, type: "lut" }); - const record = { - id: frozen.id, - type: "lut", - path: frozen.localPath, - source: frozen.source, - description: intent, - provenance: { - provider: frozen.metadata.provider, - prompt: intent, - ...frozen.metadata.provenance, - }, - }; - return finalizeColorRecord(record, frozen.source, frozen.fullPath); -} - -async function resolveColor(type, intent, options) { - if (type === "grade") return resolveGrade(intent, options); - return resolveLut(intent, options); -} - -async function ingest(src) { - if (!type || !INGEST_TYPES.includes(type)) { - console.error(`error: --from requires --type (one of: ${INGEST_TYPES.join(", ")})`); - process.exit(2); - } - const isUrl = /^https?:\/\//i.test(src); - if (isUrl && !isDirectMediaUrl(src)) { - console.error( - `error: --from takes a direct public media URL or a local file; "${src}" is not a direct media link (no platform pages / yt-dlp)`, - ); - process.exit(2); - } - if (!isUrl && !existsSync(resolve(src))) { - console.error(`error: file not found: ${src}`); - process.exit(2); - } - // Refuse 0-byte input: an empty asset would register clean but fail at render - // (freezeUrl already rejects empty responses; this covers local files). - if (!isUrl && statSync(resolve(src)).size === 0) { - console.error(`error: refusing to ingest a 0-byte file: ${src}`); - process.exit(2); - } - const ext = extname(isUrl ? new URL(src).pathname : src) || defaultExt(type); - const { id, localPath, fullPath } = await withReservedFile( - projectDir, - type, - ext, - async (reservation) => { - if (isUrl) await freezeUrl(src, reservation.fullPath); - else freezeLocalFile(resolve(src), reservation.fullPath); - return reservation; - }, - ); - if (type === "lut" || type === "grade") { - try { - const check = validateCubeFile(fullPath); - if (!check.ok) throw new Error(check.error); - } catch (err) { - rmSync(fullPath, { force: true }); - exitError(`ingested LUT is invalid: ${err.message}`, 1); - } - } - const record = { - id, - type, - path: localPath, - source: "ingested", - description: basename(src.split("?")[0]), - provenance: { provider: "local", from: src }, - }; - appendRecord(projectDir, record); - regenerateIndex(projectDir); - try { - cachePut(fullPath, record); // surface ingested assets globally too (B3) - } catch { - // best-effort - } - await result(record, "ingested"); -} - -async function showCandidates() { - const projectDir = resolve(args.project); - const type = args.type; - if (!type || !listTypes().includes(type)) { - console.error(`error: --candidates requires --type (one of: ${listTypes().join(", ")})`); - process.exit(2); - } - const intent = args.intent || ""; - const { candidates, truncated, total, similar } = listCandidates({ - projectDir, - type, - intent, - cap: CANDIDATE_CAP, - }); - await track("media_use_candidates", { - type, - project_n: total.project, - global_n: total.global, - local_only: !!args["local-only"], - }); - if (args.json) { - console.log(JSON.stringify({ ok: true, candidates, truncated, total, similar })); - } else { - console.log(formatCandidates(candidates, { truncated, total })); - } -} - -// Best-effort latest stable CLI tag from the CDN (the install script's source of -// truth). null on any failure (offline, no curl) — treated as "unknown", never fatal. -function latestHeygenStable() { - const probe = runCommand("curl", [ - "-fsSL", - "--max-time", - "4", - "https://static.heygen.ai/cli/stable", - ]); - return probe.status === 0 ? firstSemver(commandText(probe)) : null; -} - -function heygenAuthCheck() { - // `heygen auth status` already emits JSON by default (only `--human` opts out - // to a table) — there is no `--json`/`--output` flag; passing one errors with - // "unknown flag". emailFromAuthStatus parses that default JSON. - // NOTE: JSON-by-default is a v0.3.0 behavior — this probe assumes it, which - // HEYGEN_MIN_VERSION >= 0.3.0 (+ the version gate above) guarantees. If that - // floor is ever lowered, auth detection on an older CLI would silently break. - const authProbe = runCommand("heygen", ["auth", "status"]); - // spawnSync sets .error/.signal on a timeout or spawn failure (status then - // null). A stalled auth endpoint (transient network/DNS) must not be reported - // as an authoritative "not authenticated" with a re-login fix. - const timedOut = authProbe.error?.code === "ETIMEDOUT" || authProbe.signal != null; - const email = authProbe.status === 0 ? emailFromAuthStatus(commandText(authProbe)) : null; - return { - name: "heygen authenticated", - ok: !!email, - detail: email - ? `heygen authenticated as ${email}` - : timedOut - ? "heygen auth status timed out — possible network issue, not proof of sign-out" - : "heygen not authenticated", - fix: email ? "" : timedOut ? "check network, then re-run --doctor" : HEYGEN_AUTH_COMMAND, - }; -} - -function runDoctor() { - const checks = []; - const bundledSfx = inspectBundledSfxAssets(); - checks.push({ - name: "bundled SFX assets", - ok: bundledSfx.ok, - detail: bundledSfx.detail, - fix: bundledSfx.fix, - }); - const heygenVersionProbe = runCommand("heygen", ["--version"]); - const heygenOnPath = heygenVersionProbe.status === 0; - const heygenVersionText = commandText(heygenVersionProbe); - const heygenVersion = firstSemver(heygenVersionText); - - checks.push({ - name: "heygen on PATH", - ok: heygenOnPath, - // Just "is the binary here" — the version row below owns the version string, - // so this row must not also render `heygen v0.3.0` (two byte-identical lines). - detail: heygenOnPath ? "heygen found on PATH" : "heygen not found", - fix: heygenOnPath ? "" : HEYGEN_INSTALL_COMMAND, - }); - - if (!heygenOnPath) { - checks.push({ - name: "heygen version", - ok: false, - detail: "heygen version unavailable", - fix: HEYGEN_INSTALL_COMMAND, - }); - checks.push({ - name: "heygen authenticated", - ok: false, - detail: "heygen auth status unavailable", - fix: HEYGEN_INSTALL_COMMAND, - }); - } else if (heygenVersion) { - const versionOk = !versionLessThan(heygenVersion, HEYGEN_MIN_VERSION); - // Keep it latest: even when the installed version clears the floor, nudge - // `heygen update` if a newer stable exists. Best-effort — silently skipped - // when the CDN is unreachable, so it never blocks the check. - const latest = versionOk ? latestHeygenStable() : null; - const behind = latest && versionLessThan(heygenVersion, latest); - checks.push({ - name: "heygen version", - ok: versionOk, - detail: versionOk - ? `heygen v${heygenVersion}${behind ? ` (latest v${latest} available)` : ""}` - : `heygen v${heygenVersion} (need >= v${HEYGEN_MIN_VERSION})`, - fix: versionOk ? (behind ? HEYGEN_UPDATE_COMMAND : "") : HEYGEN_UPDATE_COMMAND, - }); - - // Below the OAuth-capable floor the auth probe fails for the SAME root cause - // (an old CLI can't OAuth and doesn't emit JSON auth status), which would - // read as a confusing second "not authenticated" error. Skip it — one root - // cause, one fix. - checks.push( - versionOk - ? heygenAuthCheck() - : { - name: "heygen authenticated", - ok: false, - detail: "skipped — update heygen first", - fix: HEYGEN_UPDATE_COMMAND, - }, - ); - } else { - // Fail-open: heygen ran but printed no semver (dev/stripped build). We can't - // verify the version, so we don't block on it — but say so rather than a bare - // green check that implies a real version comparison happened. - checks.push({ - name: "heygen version", - ok: true, - detail: "heygen present; version unverifiable (no semver in --version output)", - fix: "", - }); - - checks.push(heygenAuthCheck()); - } - - const ffmpegProbe = runCommand("ffmpeg", ["-version"]); - checks.push({ - name: "ffmpeg on PATH", - ok: ffmpegProbe.status === 0, - detail: ffmpegProbe.status === 0 ? firstLine(ffmpegProbe.stdout) : "ffmpeg not found", - fix: ffmpegProbe.status === 0 ? "" : "brew install ffmpeg", - }); - - const ffprobeProbe = runCommand("ffprobe", ["-version"]); - checks.push({ - name: "ffprobe on PATH", - ok: ffprobeProbe.status === 0, - detail: ffprobeProbe.status === 0 ? firstLine(ffprobeProbe.stdout) : "ffprobe not found", - fix: ffprobeProbe.status === 0 ? "" : "brew install ffmpeg", - }); - - const nodeOk = !versionLessThan(process.versions.node, MIN_NODE_VERSION); - checks.push({ - name: "node version", - ok: nodeOk, - detail: `${process.version} (need >= v${MIN_NODE_VERSION})`, - fix: nodeOk ? "" : `upgrade Node to >= v${MIN_NODE_VERSION}`, - }); - - // ffmpeg AND ffprobe are both strictly required (see references/setup-providers.md); the exit code - // must reflect that so a script gating on `--doctor` doesn't pass with ffprobe - // missing and then break at the first probe call. - const ffmpeg = checks.find((check) => check.name === "ffmpeg on PATH"); - const ffprobe = checks.find((check) => check.name === "ffprobe on PATH"); - return { ok: bundledSfx.ok && !!ffmpeg?.ok && !!ffprobe?.ok, checks }; -} - -function printDoctor(checks) { - const heygenChecks = new Set(["heygen on PATH", "heygen version", "heygen authenticated"]); - for (const check of checks) { - const prefix = check.ok ? "✓" : "✗"; - const freePath = heygenChecks.has(check.name) - ? " — free-usage path: bgm/image/voice/avatar-video" - : ""; - const fix = check.ok || !check.fix ? "" : ` — fix: ${check.fix}`; - console.log(`${prefix} ${check.detail}${freePath}${fix}`); - } -} - -function printStats(report) { - console.log("media-use stats"); - console.log(`total resolves: ${report.total_resolves}`); - console.log(`misses: ${report.misses}`); - console.log( - `hit rate: ${report.hit_rate == null ? "n/a" : `${Math.round(report.hit_rate * 100)}%`}`, - ); - printMap("by type", report.by_type); - printMap("by source", report.by_source); - printMap("by provider", report.by_provider); - printMap("by via", report.by_via); - console.log(`global cache assets: ${report.global_cache_assets}`); - console.log(`global cache disk: ${report.global_cache_disk_bytes} bytes`); - console.log(`cross-project reuse: ${report.cross_project_reuse}`); - console.log("top missed intents:"); - const entries = Object.entries(report.top_missed_intents); - if (entries.length === 0) { - console.log(" none"); - return; - } - for (const [type, misses] of entries) { - console.log(` ${type}:`); - for (const miss of misses) console.log(` ${miss.count} ${miss.intent}`); - } -} - -function printMap(label, values) { - const entries = Object.entries(values); - console.log(`${label}:`); - if (entries.length === 0) { - console.log(" none"); - return; - } - for (const [key, value] of entries) console.log(` ${key}: ${value}`); -} - -function runCommand(bin, argv) { - return spawnSync(bin, argv, { - encoding: "utf8", - timeout: 15000, - }); -} - -function commandText(result) { - return [result.stdout, result.stderr].filter(Boolean).join("\n").trim(); -} - -function firstLine(text) { - return ( - String(text || "") - .trim() - .split(/\r?\n/)[0] || "" - ); -} - -function emailFromAuthStatus(text) { - // JSON only (auth status emits JSON by default). No prose regex fallback: a - // human-format body like "Session expired. Contact support@heygen.ai" would - // otherwise report the user as authenticated as support@heygen.ai. - const trimmed = String(text || "").trim(); - if (!trimmed.startsWith("{")) return null; - try { - const parsed = JSON.parse(trimmed); - return parsed?.data?.email || parsed?.email || null; - } catch { - return null; - } -} - -async function reuseGlobal(shaArg) { - const projectDir = resolve(args.project); - const type = args.type; - if (!type || !listTypes().includes(type)) { - console.error(`error: --reuse requires --type (one of: ${listTypes().join(", ")})`); - process.exit(2); - } - if (!shaArg || !shaArg.trim()) { - console.error("error: --reuse needs a content sha/prefix (from `resolve --candidates`)"); - process.exit(2); - } - const rec = findGlobalBySha(shaArg); - if (rec && rec.ambiguous) { - console.error( - `error: sha prefix "${shaArg}" is ambiguous (${rec.count} matches) — use more characters`, - ); - process.exit(2); - } - if (!rec) { - console.error(`error: no reusable global asset matches sha "${shaArg}"`); - process.exit(1); - } - // Type guard: don't import a bgm asset as an image (audio under images/). - // icon<->image are interchangeable; everything else must match --type. - if (!typesMatch(rec.type, type)) { - console.error(`error: sha "${shaArg}" is a ${rec.type} asset, not ${type}`); - process.exit(2); - } - const ext = extname(rec.cached_path || "") || defaultExt(type); - const imported = withReservedFileSync(projectDir, type, ext, ({ id, localPath }) => - localizeImportedRecord(importFromCache(rec, projectDir, id, localPath), localPath), - ); - if (!imported) { - console.error(`error: cache entry for "${shaArg}" is incomplete or missing on disk`); - process.exit(1); - } - // Distinguish an explicit agent reuse from an automatic normalize-exact hit. - imported.source = "reused-explicit"; - imported.provenance = { ...imported.provenance, reused_by: "agent" }; - appendRecord(projectDir, imported); - regenerateIndex(projectDir); - await result(imported, "reused-explicit"); -} - -async function result(record, source) { - // Non-PII usage event: which media type, how it resolved, which provider won. - // Never the intent text or paths. Awaited so a short-lived run flushes it. - await track("media_use_resolve", { - type: record.type, - source, - provider: record.provenance?.provider, - // How a library LUT resolved: "url" (CDN), "params-fallback" (CDN failed → - // parametric), or "params" (offline). Surfaces silent CDN→params downgrades - // in prod, which --doctor can't (it only answers "reachable now?"). - via: record.provenance?.via, - // Free (OAuth) vs. paid (API-key) heygen path — sparse: absent for every - // non-heygen provider (see heygenAuthMethodFor at construction time). On a - // cache/reuse hit this reports how the asset was ORIGINALLY fetched, not - // this resolve's own credential state — intentional: it's a conversion - // signal about the fetch that actually consumed a heygen credit, not - // about the (free, no-credential) act of copying a cached file. - auth_method: record.provenance?.authMethod, - // "local" / "network_free" / "network_paid", straight from the registry's own - // A/N/P declaration — so a dashboard can separate free lookups from calls that - // spend credit without hardcoding provider names. Sparse: absent when the - // record carries no provider (cache and reuse hits) or the name is unknown. - provider_tier: providerTierFor(record.provenance?.provider), - local_only: !!args["local-only"], - provider_override: !!args.provider, - }); - if (args.json) { - const grading = record.type === "grade" && record.grading ? record.grading : null; - console.log( - JSON.stringify({ - ok: true, - ...record, - ...(grading || {}), - ...(grading && { grading }), - _source: source, - }), - ); - } else { - const meta = formatMeta(record, source); - console.log(`resolved ${record.id} → ${record.path || "inline"} (${meta})`); - } -} - -function formatMeta(record, source) { - const parts = [record.type]; - if (record.grading?.preset) parts.push(`preset ${record.grading.preset}`); - if (record.grading?.lut) parts.push("lut"); - if (record.duration != null) parts.push(`${record.duration}s`); - if (record.width && record.height) parts.push(`${record.width}×${record.height}`); - if (record.transparent) parts.push("transparent"); - if (source === "reused" || source === "reused-explicit") parts.push("reused"); - if (source === "generated") parts.push("generated"); - return parts.join(", "); -} - -function extFromUrl(url) { - try { - return extname(new URL(url).pathname) || null; - } catch { - return null; - } -} - -function defaultExt(type) { - return DEFAULT_EXT[type] || ".bin"; -} - -run().catch((err) => { - if (args.json) { - console.log(JSON.stringify({ ok: false, error: err.message })); - } else { - console.error(`error: ${err.message}`); - } - process.exit(1); -}); +process.exit(result.status ?? 1);