feat(plex): transcode playback mode — VAAPI/libx264 re-encode (E6 S1b)

Non-h264 items (playable="transcode": HEVC/VP9/AV1/MPEG-2/XviD…) now play by
re-encoding the video to H.264 inside the existing HLS session, instead of the
old HTTP 501. VAAPI hardware encode (h264_vaapi, full GPU decode→scale→encode
pipeline) where the render node is present and the source is HW-decodable;
libx264 software fallback otherwise (yuv420p 8-bit, -threads reserve-one-core).

The re-encode mints a keyframe at every segment boundary (-force_key_frames) so
the HLS muxer still cuts independent segments and seeking works; everything else
— -copyts, the HLS tail, multi-audio renditions, the AAC audio path — is the
same machinery as remux, so hls.js/subtitles/seeking are unchanged. The session
route returns transcode/hw so the player can surface the active mode.

pytest pins both command branches (software + VAAPI shape) and the HW-selection
gate (render node present AND source HW-decodable). Browser-UAT'd on the XviD
fixture (Winning London): 501→plays, real 528×384 H.264 frames, 0 dropped, seek
lands clean; stats overlay Mode row reads "transcode".

backend/tests/test_plex_stream.py

@@ -8,6 +8,7 @@ Since the sweep stopped being awaited at startup it also runs CONCURRENTLY with
 second half of this file pins what keeps it off a LIVE session's segments."""
 
 import os
+from pathlib import Path
 
 import pytest
 
@@ -164,10 +165,90 @@ class _FakeProc:
 
 
 class _FakeItem:
-    def __init__(self, rating_key="12345", file_path="/media/movie.mkv"):
+    def __init__(self, rating_key="12345", file_path="/media/movie.mkv", codec_video="h264"):
         self.rating_key = rating_key
         self.file_path = file_path
         self.playable = "remux"
+        self.codec_video = codec_video
+
+
+def _venc(cmd):
+    """The video codec ffmpeg was told to use (the value after the first `-c:v`)."""
+    return cmd[cmd.index("-c:v") + 1]
+
+
+class TestTranscodeCommand:
+    """The transcode video re-encode (playable='transcode'): the SAME HLS command as remux with the
+    video encoder swapped in for `-c:v copy`. The HW path can only be exercised on prod (VAAPI needs
+    Linux + Intel /dev/dri), so its SHAPE is pinned here; the software path is the one UAT'd locally."""
+
+    SRC = Path("/media/movie.mkv")
+    OUT = Path("/out")
+
+    def test_remux_still_stream_copies_the_video(self):
+        cmd = stream._ffmpeg_cmd(self.SRC, 0.0, self.OUT, None)
+        assert _venc(cmd) == "copy"
+        assert "libx264" not in cmd and "h264_vaapi" not in cmd
+        assert "-hwaccel" not in cmd  # no GPU decode on the copy path
+        assert "-force_key_frames" not in cmd  # copy leans on the source's own keyframes
+
+    def test_software_transcode_uses_libx264_and_mints_keyframes(self):
+        cmd = stream._ffmpeg_cmd(self.SRC, 0.0, self.OUT, None, transcode=True, hw=False)
+        assert _venc(cmd) == "libx264"
+        assert "copy" not in cmd  # the video is no longer stream-copied
+        # 8-bit High (yuv420p), not High10, so any browser decodes a 10-bit HEVC source.
+        assert cmd[cmd.index("-pix_fmt") + 1] == "yuv420p"
+        assert "-threads" in cmd  # a core reserved for request handling / playback
+        assert "-hwaccel" not in cmd  # software path: CPU decode, no render node needed
+        # A forced keyframe at every segment boundary so the HLS muxer cuts independent segments.
+        assert (
+            cmd[cmd.index("-force_key_frames") + 1] == f"expr:gte(t,n_forced*{stream._SEG_SECONDS})"
+        )
+
+    def test_hardware_transcode_uses_the_full_vaapi_pipeline(self):
+        cmd = stream._ffmpeg_cmd(self.SRC, 0.0, self.OUT, None, transcode=True, hw=True)
+        assert _venc(cmd) == "h264_vaapi"
+        assert "libx264" not in cmd
+        # GPU decode options must precede the input so the source is decoded on-device.
+        assert cmd[cmd.index("-hwaccel") + 1] == "vaapi"
+        assert "-hwaccel_output_format" in cmd
+        assert cmd[cmd.index("-vaapi_device") + 1] == stream._VAAPI_DEVICE
+        assert cmd.index("-hwaccel") < cmd.index("-i")
+        # scale_vaapi=format=nv12 downconverts a 10-bit source to 8-bit nv12 for h264_vaapi.
+        assert "scale_vaapi=format=nv12" in cmd
+        assert (
+            cmd[cmd.index("-force_key_frames") + 1] == f"expr:gte(t,n_forced*{stream._SEG_SECONDS})"
+        )
+
+    def test_multi_audio_transcode_reencodes_the_video_variant(self):
+        cmd = stream._ffmpeg_cmd(
+            self.SRC, 0.0, self.OUT, None, audio_langs=["eng", "hun"], transcode=True, hw=False
+        )
+        assert _venc(cmd) == "libx264"  # video re-encoded…
+        assert "-var_stream_map" in cmd  # …but still multi-rendition (client-side audio switch)
+        assert "copy" not in cmd
+
+
+class TestWantHw:
+    """VAAPI is used only where the render node was passed in AND the source is HW-decodable."""
+
+    def test_no_hw_when_not_transcoding(self, monkeypatch):
+        monkeypatch.setattr(stream, "_render_node_present", lambda: True)
+        assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=False) is False
+
+    def test_no_hw_without_a_render_node(self, monkeypatch):
+        monkeypatch.setattr(stream, "_render_node_present", lambda: False)
+        assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=True) is False
+
+    def test_hw_for_a_decodable_source_with_the_render_node(self, monkeypatch):
+        monkeypatch.setattr(stream, "_render_node_present", lambda: True)
+        assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=True) is True
+
+    def test_software_for_a_non_decodable_source_even_with_the_render_node(self, monkeypatch):
+        # AV1 / MPEG-4 ASP (XviD) aren't decodable on Gen 9.5 → the GPU can't feed its own encoder.
+        monkeypatch.setattr(stream, "_render_node_present", lambda: True)
+        assert stream._want_hw(_FakeItem(codec_video="av1"), transcode=True) is False
+        assert stream._want_hw(_FakeItem(codec_video="mpeg4"), transcode=True) is False
 
 
 class TestColdStartWarmup: