Re-encode a video to a smaller resolution as a new asset

backend/app/enrichment/reencode.py

@@ -0,0 +1,195 @@
+"""The reencode job: a video's bytes in, a smaller copy of it out, as a new asset.
+
+For a high-resolution original that is too heavy to cut with in a 720p or 1080p Video
+Creator project. The original is never touched; the copy is an ordinary asset with the
+original as `parent_asset_id` (provenance only, so it never blocks a delete).
+
+One ffmpeg command. The only machinery around it is reading ffmpeg's own `-progress`
+output, which is not decoration: a 4K hour re-encodes for longer than the stale sweeper's
+`job_stale_minutes`, and a job that reports nothing for that long is ended as hung. The
+same reports are the cancellation checkpoint, so Cancel stops ffmpeg instead of
+leaving it to run to the end on a job nobody is waiting for.
+
+Runs on a worker thread, so it owns its own session.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+from typing import Callable, Optional
+
+from sqlmodel import Session
+
+from app.ingest import thumbnails
+from app.ingest.filetypes import TYPE_VIDEO
+from app.ingest.probe import probe
+from app.media_tools import ffmpeg_available
+from app.models.asset import Asset
+from app.services import assets as asset_service
+from app.storage import build_storage, capacity, new_key, thumb_key_for
+
+logger = logging.getLogger(__name__)
+
+Progress = Callable[[str, int, str], None]
+
+# The frame's short side, in pixels: "1080p" is 1920x1080 landscape and 1080x1920
+# portrait alike. The short side rather than the height because Video Creator renders
+# 9:16 and 1:1 as well as 16:9, and a phone's portrait 4K wants to become 1080x1920 for
+# a 1080p vertical project, not 608x1080.
+TARGETS = (2160, 1440, 1080, 720, 480, 360)
+
+# How often a progress tick may write to the database. ffmpeg reports twice a second,
+# and a row update per report is far more than the activity indicator can show.
+_REPORT_EVERY_SECONDS = 2.0
+
+
+class ReencodeError(Exception):
+    """Something the user can be told about."""
+
+
+def label_for(short_side: int) -> str:
+    return "4K" if short_side == 2160 else f"{short_side}p"
+
+
+def short_side_of(asset: Asset) -> Optional[int]:
+    if not asset.width or not asset.height:
+        return None
+    return min(asset.width, asset.height)
+
+
+def run(
+    session: Session, asset: Asset, payload: Optional[str], progress: Progress
+) -> tuple[str, str]:
+    """Re-encode `asset` to the payload's `short_side`. Returns (detail, created_asset_id)."""
+    try:
+        data = json.loads(payload or "{}")
+        short_side = int(data["short_side"])
+    except (ValueError, KeyError, TypeError) as exc:
+        raise ReencodeError("Missing or invalid target resolution") from exc
+    if short_side not in TARGETS:
+        raise ReencodeError("Unsupported target resolution")
+    if asset.asset_type != TYPE_VIDEO or not asset.storage_key:
+        raise ReencodeError("The source file is not available")
+    # Again, although the router already did: the job may have waited in the queue.
+    if not capacity.has_room():
+        raise ReencodeError(capacity.DISK_FULL_MESSAGE)
+
+    storage = build_storage()
+    progress("Encoding", 1, "")
+
+    with storage.materialise(asset.storage_key) as source_path:
+        with tempfile.TemporaryDirectory(prefix="gam-reencode-") as workdir:
+            target_path = Path(workdir) / "reencode.mp4"
+            encode(source_path, target_path, short_side, asset.duration_seconds, progress)
+
+            progress("Reading metadata", 92, "")
+            result = probe(target_path)
+            preview = thumbnails.generate(
+                target_path,
+                TYPE_VIDEO,
+                asset.original_name or "",
+                duration_seconds=result.duration_seconds,
+            )
+
+            progress("Saving", 95, "")
+            stored = storage.write_file(new_key(asset.user_id, ".mp4"), target_path)
+            thumb_key = None
+            if preview:
+                thumb_key = storage.write_bytes(thumb_key_for(stored.key), preview).key
+
+    label = label_for(short_side)
+    created = asset_service.create_reencoded_asset(
+        session,
+        source_asset=asset,
+        stored=stored,
+        thumb_key=thumb_key,
+        name=data.get("name") or f"{asset.name} ({label})",
+        probe_result=result,
+    )
+    return f"{label} copy", created.id
+
+
+def encode(
+    source: Path,
+    target: Path,
+    short_side: int,
+    duration_seconds: Optional[float],
+    progress: Progress,
+) -> None:
+    """Scale `source` so its short side is `short_side`, into an H.264/AAC MP4."""
+    if not ffmpeg_available():
+        raise ReencodeError("ffmpeg is not available on this server")
+
+    # Decided by ffmpeg from the frame it actually decodes, not from the probed
+    # width/height: a phone video stores landscape pixels plus a rotation flag, ffmpeg
+    # applies the rotation before the filter runs, and the probe can report either
+    # orientation. -2 keeps the aspect ratio and rounds to the even size H.264 needs.
+    #
+    # setsar=1 because that rounding changes the shape very slightly, and `scale` makes
+    # up for it with non-square pixels: a 1002x750 source became 480x360 pixels shown
+    # 481 wide. Video Creator, like most editors, assumes square pixels; a 0.2% squash
+    # nobody can see is the better trade.
+    n = short_side
+    scale = f"scale=w='if(gte(iw,ih),-2,{n})':h='if(gte(iw,ih),{n},-2)',setsar=1"
+    argv = [
+        "ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error",
+        "-i", str(source),
+        "-vf", scale,
+        # The same codecs and preset as a sub-video's re-encode. CRF 20 is visually
+        # indistinguishable from the source at the smaller size, which matters for a
+        # copy that will be cut and re-rendered again downstream.
+        "-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
+        "-pix_fmt", "yuv420p",
+        # Re-encoded rather than copied: an MP4 cannot hold every audio codec a source
+        # might carry (Vorbis from a WebM, PCM from a camera), and audio costs nothing
+        # next to the video.
+        "-c:a", "aac", "-b:a", "192k",
+        "-movflags", "+faststart",
+        "-progress", "pipe:1", "-nostats",
+        "-y", str(target),
+    ]
+    stderr_path = target.with_suffix(".log")
+    with stderr_path.open("w") as stderr_file:
+        try:
+            proc = subprocess.Popen(
+                argv, stdout=subprocess.PIPE, stderr=stderr_file, text=True
+            )
+        except OSError as exc:
+            raise ReencodeError(f"Could not run ffmpeg: {exc}") from exc
+
+        try:
+            assert proc.stdout is not None
+            last_report = 0.0
+            for line in proc.stdout:
+                key, _, value = line.strip().partition("=")
+                if key != "out_time_us" or time.monotonic() - last_report < _REPORT_EVERY_SECONDS:
+                    continue
+                last_report = time.monotonic()
+                done = _seconds(value)
+                percent = int(done / duration_seconds * 90) if duration_seconds and done else 1
+                # Raises JobCancelled once the job is stopped, which unwinds through
+                # the `finally` below and kills ffmpeg with it.
+                progress("Encoding", max(1, percent), "")
+            returncode = proc.wait()
+        finally:
+            if proc.poll() is None:
+                proc.kill()
+                proc.wait()
+
+    if returncode != 0 or not target.is_file() or target.stat().st_size == 0:
+        stderr = stderr_path.read_text(errors="replace").strip()
+        logger.warning("Re-encode of %s failed: %s", source.name, stderr[-400:])
+        raise ReencodeError("Could not re-encode this video")
+
+
+def _seconds(out_time_us: str) -> float:
+    # "N/A" until the first frame is out.
+    try:
+        return max(0.0, int(out_time_us) / 1_000_000)
+    except ValueError:
+        return 0.0

backend/app/ingest/filetypes.py

@@ -51,6 +51,9 @@
 # flips `source` to `SOURCE_SUBVIDEO` — a real, standalone file from here on.
 SOURCE_CLIP = "clip"
 SOURCE_SUBVIDEO = "sub_video"
+# A smaller copy of another video (`enrichment/reencode.py`). Its own file, with the
+# original as `parent_asset_id` for provenance only, exactly as a sub-video's.
+SOURCE_REENCODE = "reencode"
 
 # Pillow can decode these directly. The rest of the image set (HEIC, AVIF without a
 # plugin, SVG) needs something else, so a thumbnail is skipped rather than attempted.

backend/app/jobs/enrichment.py

@@ -34,6 +34,7 @@
 from app.enrichment.bulk import run as run_bulk
 from app.enrichment.describe import run as run_describe
 from app.enrichment.extract_subvideo import run as run_extract_subvideo
+from app.enrichment.reencode import ReencodeError, run as run_reencode
 from app.enrichment.extract_text import TextExtractionError
 from app.enrichment.extract_text import run as run_extract_text
 from app.enrichment.generate_all import run as run_generate_all
@@ -70,6 +71,7 @@
     KIND_DESCRIBE,
     KIND_EMBED,
     KIND_EXTRACT_SUBVIDEO,
+    KIND_REENCODE,
     KIND_EXTRACT_TEXT,
     GENERATION_JOB_KINDS,
     KIND_GENERATE,
@@ -318,6 +320,10 @@ def _run(job_id: str, q: JobQueue) -> None:
                 # one of them, so this survives it untouched.
                 if created_asset_id:
                     job.payload = json.dumps({"created_asset_id": created_asset_id})
+            elif job.kind == KIND_REENCODE:
+                detail, created_asset_id = run_reencode(session, asset, job.payload, progress)
+                # Merged, not replaced: the target size stays readable on the row.
+                job.payload = _with_created_asset(job.payload, created_asset_id)
             elif job.kind == KIND_SUMMARIZE:
                 detail = run_summarize(session, asset, progress)
             elif job.kind == KIND_AUTOTAG:
@@ -452,6 +458,11 @@ def _run(job_id: str, q: JobQueue) -> None:
             set_fields(session, job, status="error", stage="", error_message=message)
             logger.info("Enrichment job %s failed: %s", job_id, message)
 
+        except ReencodeError as exc:
+            message = str(exc)
+            set_fields(session, job, status="error", stage="", error_message=message)
+            logger.info("Enrichment job %s failed: %s", job_id, message)
+
         except SubvideoExtractionError as exc:
             message = str(exc)
             _mark_asset_failed(session, asset, job)

backend/app/jobs/registry.py

@@ -26,6 +26,7 @@
     KIND_GENERATE,
     KIND_IMPORT_TRACK,
     KIND_IMPORT_URL,
+    KIND_REENCODE,
     EnrichmentJob,
 )
 from app.schemas_jobs import ActivityJobRead
@@ -55,7 +56,7 @@ def __init__(
 # pointed at the source, and an import or a generation has no asset at all until it is
 # done.
 _CREATES_AN_ASSET = frozenset(
-    {KIND_EXTRACT_SUBVIDEO, KIND_IMPORT_URL, KIND_IMPORT_TRACK, KIND_GENERATE}
+    {KIND_EXTRACT_SUBVIDEO, KIND_REENCODE, KIND_IMPORT_URL, KIND_IMPORT_TRACK, KIND_GENERATE}
 )
 
 

backend/app/models/job.py

@@ -45,6 +45,10 @@
 # costs nothing, the same reason `KIND_EXTRACT_TEXT` sits apart from the LLM jobs
 # despite also being per-asset.
 KIND_EXTRACT_SUBVIDEO = "extract_subvideo"
+# Scaling a video down to a smaller frame as a new asset (`enrichment/reencode.py`) — a
+# 4K original made workable for a 1080p Video Creator project. Per-asset and outside
+# `ENRICHMENT_KINDS` for the reasons an extraction is: one ffmpeg run, no provider.
+KIND_REENCODE = "reencode"
 # M10. Proposing where an asset came from. Inside `ENRICHMENT_KINDS` unlike the two
 # above: it calls a provider and costs money, so it belongs in the bulk-enrichment
 # selection UI and its cost estimate.

backend/app/routers/assets.py

@@ -20,6 +20,7 @@
     KIND_ANALYZE_MUSIC_LIBRARY,
     KIND_EXTRACT_SUBVIDEO,
     KIND_HARVEST_ATTRIBUTION,
+    KIND_REENCODE,
 )
 from app.models.tag import Tag
 from app.schemas import DataResponse, ListResponse
@@ -455,8 +456,11 @@ def delete_asset(
     # `create_subvideo_asset`'s insert trips the parent foreign key — either way after
     # ffmpeg has already written a file nothing will ever reference.
     if any(
-        enrichment_jobs.active_job(session, row.id, KIND_EXTRACT_SUBVIDEO) is not None
+        enrichment_jobs.active_job(session, row.id, kind) is not None
         for row in (asset, *clips)
+        # A re-encode ends the same way as an extract: inserting a row whose parent
+        # is gone.
+        for kind in (KIND_EXTRACT_SUBVIDEO, KIND_REENCODE)
     ):
         raise HTTPException(
             status_code=status.HTTP_409_CONFLICT,

backend/app/routers/clips.py

@@ -1,4 +1,4 @@
-"""Clips (non-destructive) and sub-video extraction (destructive), over an asset."""
+"""Clips (non-destructive), and new files cut or re-encoded from an asset."""
 
 from __future__ import annotations
 
@@ -15,7 +15,8 @@
 from app.jobs import enrichment as enrichment_jobs
 from app.jobs.registry import KINDS
 from app.models.asset import Asset
-from app.models.job import KIND_EXTRACT_SUBVIDEO
+from app.enrichment.reencode import TARGETS, label_for, short_side_of
+from app.models.job import KIND_EXTRACT_SUBVIDEO, KIND_REENCODE
 from app.routers.assets import get_storage
 from app.schemas import DataResponse, ListResponse
 from app.schemas_assets import AssetRead
@@ -57,6 +58,14 @@ class ClipCreate(BaseModel):
     name: Optional[str] = Field(default=None, min_length=1, max_length=255)
 
 
+class ReencodeCreate(BaseModel):
+    # The frame's short side in pixels: 1080 is 1920x1080 or 1080x1920. Checked against
+    # `TARGETS` in the route rather than typed as a Literal, so a bad value gets the
+    # house error shape instead of a bare 422.
+    short_side: int
+    name: Optional[str] = Field(default=None, min_length=1, max_length=255)
+
+
 class ClipRangeUpdate(BaseModel):
     in_point: float = Field(ge=0)
     out_point: float = Field(gt=0)
@@ -230,3 +239,62 @@ def promote_clip(
         session, clip, KIND_EXTRACT_SUBVIDEO, payload=json.dumps({"mode": "promote"})
     )
     return DataResponse(data=KINDS["enrichment"].to_activity(job))
+
+
+# ─── re-encoding to a smaller frame ─────────────────────────────────────────────
+
+
+@router.post("/{asset_id}/reencode", response_model=DataResponse[ActivityJobRead], status_code=202)
+def start_reencode(
+    asset_id: str,
+    payload: ReencodeCreate,
+    user: CurrentUser,
+    session: Session = Depends(get_session),
+) -> DataResponse[ActivityJobRead]:
+    """Queue a smaller copy of a video as a new asset. The original is untouched."""
+    asset = _owned_asset(asset_id, user.id, session)
+    if asset.asset_type != TYPE_VIDEO or not asset.storage_key:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={
+                "code": "not_reencodable",
+                "message": "Only a video that owns a file can be re-encoded",
+            },
+        )
+    if payload.short_side not in TARGETS:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={
+                "code": "bad_request",
+                "message": "Choose one of " + ", ".join(label_for(t) for t in TARGETS),
+            },
+        )
+    # Down only. Scaling up adds bytes and no detail, and Video Creator already scales
+    # a small source up to its frame at render time for free. A video with no recorded
+    # size is let through: ffmpeg will still do what was asked.
+    current = short_side_of(asset)
+    if current is not None and payload.short_side >= current:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={
+                "code": "not_smaller",
+                "message": f"This video is already {current}p or smaller",
+            },
+        )
+    if enrichment_jobs.active_job(session, asset.id, KIND_REENCODE) is not None:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail={
+                "code": "already_running",
+                "message": "A re-encode is already running for this video",
+            },
+        )
+    capacity.require_room()
+
+    job = enrichment_jobs.submit(
+        session,
+        asset,
+        KIND_REENCODE,
+        payload=json.dumps({"short_side": payload.short_side, "name": payload.name}),
+    )
+    return DataResponse(data=KINDS["enrichment"].to_activity(job))