feat(plex): transcode playback mode — VAAPI/libx264 re-encode (E6 S1b)

Non-h264 items (playable="transcode": HEVC/VP9/AV1/MPEG-2/XviD…) now play by re-encoding the video to H.264 inside the existing HLS session, instead of the old HTTP 501. VAAPI hardware encode (h264_vaapi, full GPU decode→scale→encode pipeline) where the render node is present and the source is HW-decodable; libx264 software fallback otherwise (yuv420p 8-bit, -threads reserve-one-core). The re-encode mints a keyframe at every segment boundary (-force_key_frames) so the HLS muxer still cuts independent segments and seeking works; everything else — -copyts, the HLS tail, multi-audio renditions, the AAC audio path — is the same machinery as remux, so hls.js/subtitles/seeking are unchanged. The session route returns transcode/hw so the player can surface the active mode. pytest pins both command branches (software + VAAPI shape) and the HW-selection gate (render node present AND source HW-decodable). Browser-UAT'd on the XviD fixture (Winning London): 501→plays, real 528×384 H.264 frames, 0 dropped, seek lands clean; stats overlay Mode row reads "transcode".
2026-08-30 01:06:08 +02:00
parent f162788135
commit 0658f9550b
3 changed files with 217 additions and 21 deletions
@@ -3,7 +3,10 @@
 Playback is from the LOCAL physical file. Browser-compatible files (playable="direct") are served
 raw with HTTP range requests. Everything else that is h264 video (playable="remux") is remuxed on
 the fly to HLS with **video stream-copy** (cheap, I/O-bound — no video re-encode) and audio to AAC
-when needed. HEVC/VP9 (playable="transcode") needs a full re-encode and is deferred to P3.
+when needed. Non-h264 video (playable="transcode": HEVC/VP9/AV1/MPEG-2/XviD…) needs a full re-encode
+to H.264: on the prod host that runs on the Intel iGPU via VAAPI (`h264_vaapi`, near-zero CPU), with
+software libx264 as the automatic fallback where the render node is absent or the source codec isn't
+hardware-decodable. Same HLS session/seek machinery either way — only the video encoder differs.

 Seek model (Jellyfin-style): one ffmpeg session per item, started at a given offset via `-ss`
 (fast keyframe seek). A seek beyond the generated region restarts the session at the new offset,
@@ -27,6 +30,7 @@ from sqlalchemy.orm import Session as DbSession

 from app import sysconfig
 from app.config import settings
+from app.downloads.edit import encode_thread_cap
 from app.models import PlexItem
 from app.plex import paths

@@ -83,11 +87,20 @@ _sessions: dict[str, "HlsSession"] = {}

 class HlsSession:
    def __init__(
-        self, key: str, directory: Path, proc: subprocess.Popen, start_s: float, entry: str
+        self,
+        key: str,
+        directory: Path,
+        proc: subprocess.Popen,
+        start_s: float,
+        entry: str,
+        transcode: bool = False,
+        hw: bool = False,
    ):
        self.key = key
        self.dir = directory
        self.proc = proc
+        self.transcode = transcode  # video re-encoded (non-h264 source), not stream-copied
+        self.hw = hw  # transcode ran on the VAAPI hardware encoder (else libx264 software)
        self.start_s = start_s  # the REQUESTED seek offset
        # The REAL media start: with `-ss X -c:v copy` ffmpeg can only cut at the nearest earlier
        # keyframe K<=X, and hls.js zero-bases the timeline to K (so video.currentTime=0 is K, not X).
@@ -127,6 +140,80 @@ _HLS_TAIL = [
 ]


+# --- Transcode (playable="transcode") video re-encode -----------------------------------------
+# The remux path stream-copies the video (`-c:v copy`); a non-h264 source can't be copied and must be
+# re-encoded to H.264. Two encoders, chosen per-session by `_want_hw`:
+#   * VAAPI hardware (`h264_vaapi`) — the prod default once the iGPU is passed in. A full GPU pipeline
+#     (decode + scale + encode all on-device) offloads the CPU almost entirely, so several concurrent
+#     1080p transcodes are fine and the API stays responsive.
+#   * libx264 software — the automatic fallback where `/dev/dri` is absent (dev box, or prod before the
+#     iGPU enablement) or the source codec isn't hardware-decodable (AV1/XviD on Gen 9.5). CPU-bound, so
+#     it reserves a core via `-threads` (see encode_thread_cap).
+# Everything else — the `-copyts`/`_HLS_TAIL`/segment/seek-restart model and the AAC audio path — is
+# unchanged, so hls.js, subtitles, multi-audio and seeking keep working exactly as for remux.
+_VAAPI_DEVICE = "/dev/dri/renderD128"
+# Codecs the prod iGPU (UHD 630 / Quick Sync Gen 9.5) can hardware-DECODE — the precondition for the
+# full-GPU VAAPI pipeline. A source outside this set (AV1, MPEG-4 ASP/XviD…) falls back to software even
+# when the render node is present, because the GPU can't decode it to feed its own encoder.
+_HW_DECODABLE = {"h264", "hevc", "h265", "vp9", "vc1", "mpeg2video"}
+
+
+def _hwaccel_in_opts(hw: bool) -> list[str]:
+    """VAAPI decode options, placed before `-i` so the source is decoded (and kept) on the GPU.
+    `-hwaccel_output_format vaapi` keeps frames as GPU surfaces all the way to `h264_vaapi` — no
+    round-trip to system memory. Empty for the software path."""
+    if not hw:
+        return []
+    return ["-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", "-vaapi_device", _VAAPI_DEVICE]
+
+
+def _venc_opts(transcode: bool, hw: bool) -> list[str]:
+    """The video-encoder args that replace `-c:v copy` for a transcode session (empty-of-meaning:
+    returns the copy args when not transcoding). A re-encode must MINT its own keyframes at the segment
+    boundaries — the copy path leaned on the source's keyframes, but a re-encoded stream has none the
+    HLS muxer can cut on unless we force them (`-force_key_frames` every `_SEG_SECONDS`), else segments
+    aren't independent and seeking breaks."""
+    if not transcode:
+        return ["-c:v", "copy"]
+    force_kf = ["-force_key_frames", f"expr:gte(t,n_forced*{_SEG_SECONDS})"]
+    if hw:
+        # scale_vaapi with format=nv12 downconverts a 10-bit (p010) HEVC source to 8-bit nv12 so
+        # h264_vaapi (8-bit High) can encode it; for an 8-bit source it's a cheap format pass-through.
+        return ["-vf", "scale_vaapi=format=nv12", "-c:v", "h264_vaapi", *force_kf]
+    # Software fallback: same browser-safe recipe as downloads/edit.py (libx264 veryfast/crf20).
+    # yuv420p forces 8-bit High (not High10) so any browser decodes it; -threads reserves a core so the
+    # CPU-bound encode can't starve request handling / playback on the shared box.
+    return [
+        "-c:v",
+        "libx264",
+        "-preset",
+        "veryfast",
+        "-crf",
+        "20",
+        "-pix_fmt",
+        "yuv420p",
+        "-threads",
+        str(encode_thread_cap()),
+        *force_kf,
+    ]
+
+
+def _render_node_present() -> bool:
+    """Whether the VAAPI render node is available — i.e. `/dev/dri` was passed into the container.
+    Present on prod after the iGPU enablement; absent on the CPU-only dev box and in the localdev
+    Docker. Its own function so the selection is monkeypatchable in tests and one place for S1c's knob."""
+    return Path(_VAAPI_DEVICE).exists()
+
+
+def _want_hw(item: PlexItem, transcode: bool) -> bool:
+    """Whether to use the VAAPI hardware encoder for this transcode session. True iff we're
+    transcoding AND the render node is present AND the iGPU can hardware-decode the source (else the
+    full-GPU pipeline can't feed itself → software libx264). (S1c layers an admin on/off/auto override.)"""
+    if not transcode or not _render_node_present():
+        return False
+    return (item.codec_video or "").lower() in _HW_DECODABLE
+
+
 def _probe_audio_langs(src: Path) -> list[str]:
    """One entry per audio stream (its language tag, or "" if untagged). len() = audio-track count.
    Used to build the multi-rendition var_stream_map. Empty on probe failure → single-audio path."""
@@ -160,8 +247,13 @@ def _ffmpeg_cmd(
    audio_ord: int | None,
    aoff: float = 0.0,
    audio_langs: list[str] | None = None,
+    transcode: bool = False,
+    hw: bool = False,
 ) -> list[str]:
-    args = ["ffmpeg", "-nostdin", "-loglevel", "error"]
+    # For a transcode session the video is re-encoded (VAAPI or libx264) instead of stream-copied; the
+    # hwaccel decode options go before -i, the encoder args replace `-c:v copy` below. Everything else is
+    # identical to remux.
+    args = ["ffmpeg", "-nostdin", "-loglevel", "error", *_hwaccel_in_opts(hw)]
    # Keep ORIGINAL timestamps on the output so the first segment carries the true absolute PTS of the
    # keyframe ffmpeg actually seeked to. hls.js still zero-bases playback to that PTS; we read it back
    # (see _probe_first_pts) to learn the real content offset. Without -copyts the segment PTS is
@@ -182,7 +274,7 @@ def _ffmpeg_cmd(
        args += ["-map", "0:v:0"]
        for i in range(len(audio_langs)):
            args += ["-map", f"{ai}:a:{i}?"]
-        args += ["-sn", "-c:v", "copy", "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
+        args += ["-sn", *_venc_opts(transcode, hw), "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
        var = ["v:0,agroup:aud"]
        for i, lang in enumerate(audio_langs):
            entry = f"a:{i},agroup:aud,name:a{i}"
@@ -203,7 +295,7 @@ def _ffmpeg_cmd(
    else:
        ao = audio_ord if audio_ord is not None else 0
        args += ["-map", "0:v:0", "-map", f"{ai}:a:{ao}?", "-sn"]
-        args += ["-c:v", "copy", "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
+        args += [*_venc_opts(transcode, hw), "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
        args += _HLS_TAIL + [
            "-hls_segment_filename",
            str(out_dir / "seg_%d.ts"),
@@ -275,16 +367,20 @@ def start_session(
    audio_ord: int | None = None,
    aoff: float = 0.0,
    multi: bool = False,
+    transcode: bool = False,
 ) -> HlsSession | None:
-    """(Re)start the HLS remux for an item at the given offset. `multi` (item has >1 audio track) maps
-    every audio track as an HLS rendition so the client switches audio without a restart; otherwise a
-    single audio track (`audio_ord`) is muxed. Subtitles are separate WebVTT tracks (not muxed here).
-    Returns None if the local file can't be read."""
+    """(Re)start the HLS session for an item at the given offset. `transcode` (item is non-h264,
+    playable="transcode") re-encodes the video to H.264 (VAAPI or libx264) instead of stream-copying it;
+    otherwise the video is copied (remux). `multi` (item has >1 audio track) maps every audio track as an
+    HLS rendition so the client switches audio without a restart; otherwise a single audio track
+    (`audio_ord`) is muxed. Subtitles are separate WebVTT tracks (not muxed here). Returns None if the
+    local file can't be read."""
    src = paths.local_media_path(db, item.file_path)
    if src is None:
        return None
    key = item.rating_key
    start_s = max(0.0, float(start_s))
+    hw = _want_hw(item, transcode)
    audio_langs = _probe_audio_langs(src) if multi else []
    is_multi = len(audio_langs) > 1  # only worth a master playlist when there really are ≥2 tracks
    with _lock:
@@ -297,12 +393,27 @@ def start_session(
        directory.mkdir(parents=True, exist_ok=True)
        proc = subprocess.Popen(
            _ffmpeg_cmd(
-                src, start_s, directory, audio_ord, aoff, audio_langs if is_multi else None
+                src,
+                start_s,
+                directory,
+                audio_ord,
+                aoff,
+                audio_langs if is_multi else None,
+                transcode=transcode,
+                hw=hw,
            ),
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
        )
-        s = HlsSession(key, directory, proc, start_s, "master.m3u8" if is_multi else "index.m3u8")
+        s = HlsSession(
+            key,
+            directory,
+            proc,
+            start_s,
+            "master.m3u8" if is_multi else "index.m3u8",
+            transcode=transcode,
+            hw=hw,
+        )
        _sessions[key] = s
        # Admin-configurable concurrency cap (Configuration → Plex → Max concurrent transcodes);
        # at least 1 so playback can't be capped to zero.
@@ -329,12 +440,14 @@ def start_session(
            if not wait_for(_seg(i), timeout=_WARMUP_SEG_TIMEOUT_S):
                break
    log.info(
-        "plex hls session start key=%s req=%.1f real=%.3f multi=%s audio=%s",
+        "plex hls session start key=%s req=%.1f real=%.3f multi=%s audio=%s transcode=%s hw=%s",
        key,
        start_s,
        s.media_start_s,
        is_multi,
        audio_ord,
+        transcode,
+        hw,
    )
    return s

@@ -532,15 +532,13 @@ def stream_session(
    db: Session = Depends(get_db),
 ) -> dict:
    """Prepare playback from `start` seconds. Direct-playable files are served raw; remux files
-    (re)start an HLS session at the offset (seek-restart); transcode (HEVC/VP9) is P3. Selecting a
-    non-default audio track (`audio` = stream ordinal) forces the HLS path so the chosen audio can be
-    muxed in. Subtitles are independent WebVTT tracks (GET /subtitle) — no session restart."""
+    (re)start an HLS session at the offset (seek-restart, video stream-copied); transcode files (non-h264:
+    HEVC/VP9/AV1/…) take the same HLS path but with the video re-encoded to H.264 (VAAPI or libx264).
+    Selecting a non-default audio track (`audio` = stream ordinal) forces the HLS path so the chosen audio
+    can be muxed in. Subtitles are independent WebVTT tracks (GET /subtitle) — no session restart."""
    plex_stream.reap_idle()  # opportunistic cleanup of idle sessions on each new playback
    it = item_or_404(db, rating_key)
-    if it.playable == "transcode":
-        raise HTTPException(
-            status_code=501, detail="This format needs transcoding (a later phase)."
-        )
+    transcode = it.playable == "transcode"
    # `multi` (item has ≥2 audio tracks) forces the HLS path so ALL audio tracks ship as renditions and
    # the client can switch audio without a restart — even for otherwise direct-playable files.
    if it.playable == "direct" and audio is None and not aoff and not multi:
@@ -551,15 +549,19 @@ def stream_session(
        }
    if plex_paths.local_media_path(db, it.file_path) is None:
        raise HTTPException(status_code=404, detail="Media file not found on disk")
-    s = plex_stream.start_session(db, it, start, audio, aoff, multi)
+    s = plex_stream.start_session(db, it, start, audio, aoff, multi, transcode=transcode)
    if s is None:
        raise HTTPException(status_code=404, detail="Media file not found on disk")
    # Report the REAL media start (the keyframe ffmpeg landed on, which hls.js zero-bases to), not the
    # requested offset — the client's absolute clock and the subtitle cue-shift both key off this.
+    # `transcode`/`hw` let the player show an honest cold-start ("Preparing stream…") and surface the
+    # active mode + encoder in the stats overlay; both segment streams are otherwise identical.
    return {
        "mode": "hls",
        "start_s": s.media_start_s,
        "url": f"/api/plex/stream/{it.rating_key}/hls/{s.entry}",
+        "transcode": s.transcode,
+        "hw": s.hw,
    }


@@ -8,6 +8,7 @@ Since the sweep stopped being awaited at startup it also runs CONCURRENTLY with
 second half of this file pins what keeps it off a LIVE session's segments."""

 import os
+from pathlib import Path

 import pytest

@@ -164,10 +165,90 @@ class _FakeProc:


 class _FakeItem:
-    def __init__(self, rating_key="12345", file_path="/media/movie.mkv"):
+    def __init__(self, rating_key="12345", file_path="/media/movie.mkv", codec_video="h264"):
        self.rating_key = rating_key
        self.file_path = file_path
        self.playable = "remux"
+        self.codec_video = codec_video
+
+
+def _venc(cmd):
+    """The video codec ffmpeg was told to use (the value after the first `-c:v`)."""
+    return cmd[cmd.index("-c:v") + 1]
+
+
+class TestTranscodeCommand:
+    """The transcode video re-encode (playable='transcode'): the SAME HLS command as remux with the
+    video encoder swapped in for `-c:v copy`. The HW path can only be exercised on prod (VAAPI needs
+    Linux + Intel /dev/dri), so its SHAPE is pinned here; the software path is the one UAT'd locally."""
+
+    SRC = Path("/media/movie.mkv")
+    OUT = Path("/out")
+
+    def test_remux_still_stream_copies_the_video(self):
+        cmd = stream._ffmpeg_cmd(self.SRC, 0.0, self.OUT, None)
+        assert _venc(cmd) == "copy"
+        assert "libx264" not in cmd and "h264_vaapi" not in cmd
+        assert "-hwaccel" not in cmd  # no GPU decode on the copy path
+        assert "-force_key_frames" not in cmd  # copy leans on the source's own keyframes
+
+    def test_software_transcode_uses_libx264_and_mints_keyframes(self):
+        cmd = stream._ffmpeg_cmd(self.SRC, 0.0, self.OUT, None, transcode=True, hw=False)
+        assert _venc(cmd) == "libx264"
+        assert "copy" not in cmd  # the video is no longer stream-copied
+        # 8-bit High (yuv420p), not High10, so any browser decodes a 10-bit HEVC source.
+        assert cmd[cmd.index("-pix_fmt") + 1] == "yuv420p"
+        assert "-threads" in cmd  # a core reserved for request handling / playback
+        assert "-hwaccel" not in cmd  # software path: CPU decode, no render node needed
+        # A forced keyframe at every segment boundary so the HLS muxer cuts independent segments.
+        assert (
+            cmd[cmd.index("-force_key_frames") + 1] == f"expr:gte(t,n_forced*{stream._SEG_SECONDS})"
+        )
+
+    def test_hardware_transcode_uses_the_full_vaapi_pipeline(self):
+        cmd = stream._ffmpeg_cmd(self.SRC, 0.0, self.OUT, None, transcode=True, hw=True)
+        assert _venc(cmd) == "h264_vaapi"
+        assert "libx264" not in cmd
+        # GPU decode options must precede the input so the source is decoded on-device.
+        assert cmd[cmd.index("-hwaccel") + 1] == "vaapi"
+        assert "-hwaccel_output_format" in cmd
+        assert cmd[cmd.index("-vaapi_device") + 1] == stream._VAAPI_DEVICE
+        assert cmd.index("-hwaccel") < cmd.index("-i")
+        # scale_vaapi=format=nv12 downconverts a 10-bit source to 8-bit nv12 for h264_vaapi.
+        assert "scale_vaapi=format=nv12" in cmd
+        assert (
+            cmd[cmd.index("-force_key_frames") + 1] == f"expr:gte(t,n_forced*{stream._SEG_SECONDS})"
+        )
+
+    def test_multi_audio_transcode_reencodes_the_video_variant(self):
+        cmd = stream._ffmpeg_cmd(
+            self.SRC, 0.0, self.OUT, None, audio_langs=["eng", "hun"], transcode=True, hw=False
+        )
+        assert _venc(cmd) == "libx264"  # video re-encoded…
+        assert "-var_stream_map" in cmd  # …but still multi-rendition (client-side audio switch)
+        assert "copy" not in cmd
+
+
+class TestWantHw:
+    """VAAPI is used only where the render node was passed in AND the source is HW-decodable."""
+
+    def test_no_hw_when_not_transcoding(self, monkeypatch):
+        monkeypatch.setattr(stream, "_render_node_present", lambda: True)
+        assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=False) is False
+
+    def test_no_hw_without_a_render_node(self, monkeypatch):
+        monkeypatch.setattr(stream, "_render_node_present", lambda: False)
+        assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=True) is False
+
+    def test_hw_for_a_decodable_source_with_the_render_node(self, monkeypatch):
+        monkeypatch.setattr(stream, "_render_node_present", lambda: True)
+        assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=True) is True
+
+    def test_software_for_a_non_decodable_source_even_with_the_render_node(self, monkeypatch):
+        # AV1 / MPEG-4 ASP (XviD) aren't decodable on Gen 9.5 → the GPU can't feed its own encoder.
+        monkeypatch.setattr(stream, "_render_node_present", lambda: True)
+        assert stream._want_hw(_FakeItem(codec_video="av1"), transcode=True) is False
+        assert stream._want_hw(_FakeItem(codec_video="mpeg4"), transcode=True) is False


 class TestColdStartWarmup: