From ce96032edbb6d23b829e86fa9491219677845581 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 30 Aug 2026 01:22:30 +0200 Subject: [PATCH] feat(plex): transcode resource discipline + admin knobs (E6 S1c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The software-fallback re-encode now runs deprioritized and admin-configurable. The SW transcode ffmpeg child is niced (preexec_fn → os.nice, applied per-child because the Popen lives in the long-lived API process, which must NOT itself be niced — unlike the download worker's whole-process _deprioritize) so a CPU-bound re-encode can't starve request handling or playback; the VAAPI path needs none of it. New sysconfig knobs (plex group, rendered on the admin Configuration page, i18n HU/EN): plex_transcode_hw (auto/on/off, layered onto _want_hw), plex_transcode_ preset, plex_transcode_crf, plex_transcode_max_height (software-only downscale-to cap; the GPU path encodes at source resolution). plex_max_transcodes stays the concurrency cap (comment corrected — the fallback is shipped, not "later"). pytest extends the command-builder + _want_hw coverage (preset/crf/max_height plumbing, the on/off/auto policy). Browser-UAT'd: max_height=240 drops the live resolution to 330x240; the SW ffmpeg child runs at nice 10 (ppid=api process). --- backend/app/config.py | 17 +++- backend/app/plex/stream.py | 98 +++++++++++++++++++----- backend/app/sysconfig.py | 8 ++ backend/tests/test_plex_stream.py | 66 ++++++++++++++-- frontend/src/i18n/locales/en/config.json | 18 ++++- frontend/src/i18n/locales/hu/config.json | 18 ++++- 6 files changed, 193 insertions(+), 32 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index 0af52517..bd916a90 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -258,9 +258,22 @@ class Settings(BaseSettings): # that catches what the incremental history feed can't — notably an un-watch done on the Plex # side — so it runs far less often (daily) than the incremental pass. plex_watch_reconcile_interval_min: int = 1440 - # Max concurrent transcodes for the P3 fallback. Low by default — CPU-only transcode is - # expensive; direct-serve (browser-compatible files) has no such limit. + # Max concurrent transcode sessions (playable="transcode"). Low by default — the software-fallback + # re-encode is CPU-heavy (the VAAPI hardware path is far cheaper and could sustain more); direct-serve + # and remux (stream-copy) are not capped by this. plex_max_transcodes: int = 4 + # --- Transcode (playable="transcode") re-encode tuning (E6 S1) --- + # Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the + # source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render + # node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_want_hw`. + plex_transcode_hw: str = "auto" + # Software-fallback libx264 tuning (IGNORED on the VAAPI path). `preset` trades CPU for size; `crf` + # trades quality for size (lower = better + bigger). `max_height` caps the SOFTWARE transcode's output + # height (0 = keep source) so a CPU re-encode of a tall source stays affordable — the GPU path always + # encodes at source resolution. + plex_transcode_preset: str = "veryfast" + plex_transcode_crf: int = 20 + plex_transcode_max_height: int = 1080 # Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls` # dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point # this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index 5f584cfa..ab41436d 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -19,6 +19,7 @@ CPU-only prod host; only P3's full transcode is CPU-heavy. """ import logging +import os import re import shutil import subprocess @@ -167,35 +168,49 @@ def _hwaccel_in_opts(hw: bool) -> list[str]: return ["-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", "-vaapi_device", _VAAPI_DEVICE] -def _venc_opts(transcode: bool, hw: bool) -> list[str]: - """The video-encoder args that replace `-c:v copy` for a transcode session (empty-of-meaning: - returns the copy args when not transcoding). A re-encode must MINT its own keyframes at the segment - boundaries — the copy path leaned on the source's keyframes, but a re-encoded stream has none the - HLS muxer can cut on unless we force them (`-force_key_frames` every `_SEG_SECONDS`), else segments - aren't independent and seeking breaks.""" +def _venc_opts( + transcode: bool, + hw: bool, + preset: str = "veryfast", + crf: int = 20, + max_height: int = 0, +) -> list[str]: + """The video-encoder args that replace `-c:v copy` for a transcode session (returns the copy args + when not transcoding). A re-encode must MINT its own keyframes at the segment boundaries — the copy + path leaned on the source's keyframes, but a re-encoded stream has none the HLS muxer can cut on + unless we force them (`-force_key_frames` every `_SEG_SECONDS`), else segments aren't independent and + seeking breaks. `preset`/`crf`/`max_height` (software only) come from admin sysconfig.""" if not transcode: return ["-c:v", "copy"] force_kf = ["-force_key_frames", f"expr:gte(t,n_forced*{_SEG_SECONDS})"] if hw: # scale_vaapi with format=nv12 downconverts a 10-bit (p010) HEVC source to 8-bit nv12 so # h264_vaapi (8-bit High) can encode it; for an 8-bit source it's a cheap format pass-through. + # The GPU encode is cheap, so it runs at source resolution (max_height is a software-only knob). return ["-vf", "scale_vaapi=format=nv12", "-c:v", "h264_vaapi", *force_kf] - # Software fallback: same browser-safe recipe as downloads/edit.py (libx264 veryfast/crf20). - # yuv420p forces 8-bit High (not High10) so any browser decodes it; -threads reserves a core so the - # CPU-bound encode can't starve request handling / playback on the shared box. - return [ + # Software fallback: same browser-safe recipe as downloads/edit.py (libx264). yuv420p forces 8-bit + # High (not High10) so any browser decodes it; -threads reserves a core so the CPU-bound encode can't + # starve request handling / playback on the shared box. + args: list[str] = [] + if max_height and max_height > 0: + # Downscale-ONLY to the height cap (min(ih,H)), keeping aspect with an even width (-2), so a tall + # source doesn't make the software re-encode ruinously expensive. The comma in min() is escaped + # (\,) so ffmpeg's filter parser reads one filter, not two. Never upscales a smaller source. + args += ["-vf", f"scale=-2:min(ih\\,{max_height})"] + args += [ "-c:v", "libx264", "-preset", - "veryfast", + preset, "-crf", - "20", + str(crf), "-pix_fmt", "yuv420p", "-threads", str(encode_thread_cap()), *force_kf, ] + return args def _render_node_present() -> bool: @@ -205,15 +220,39 @@ def _render_node_present() -> bool: return Path(_VAAPI_DEVICE).exists() -def _want_hw(item: PlexItem, transcode: bool) -> bool: - """Whether to use the VAAPI hardware encoder for this transcode session. True iff we're - transcoding AND the render node is present AND the iGPU can hardware-decode the source (else the - full-GPU pipeline can't feed itself → software libx264). (S1c layers an admin on/off/auto override.)""" - if not transcode or not _render_node_present(): +def _want_hw(db: DbSession, item: PlexItem, transcode: bool) -> bool: + """Whether to use the VAAPI hardware encoder for this transcode session, honouring the admin + `plex_transcode_hw` policy: "off" → always software; "on" → VAAPI wherever the render node exists + (trusting the admin, skipping the source-decodability gate); "auto" (default) → VAAPI only when the + render node is present AND the iGPU can hardware-decode the source (else the full-GPU pipeline can't + feed itself → software libx264).""" + if not transcode: return False + mode = (sysconfig.get_str(db, "plex_transcode_hw") or "auto").lower() + if mode == "off" or not _render_node_present(): + return False + if mode == "on": + return True return (item.codec_video or "").lower() in _HW_DECODABLE +# CPU-nice increment for the SOFTWARE transcode child. The Plex ffmpeg is a Popen in the long-lived API +# process (which must NOT itself be niced), so — unlike the download worker's _deprioritize (run once in +# its own main()) — we lower priority per-child via preexec_fn in the fork. The GPU path needs none of +# this. Reuses the download child's nice default so all co-located re-encoders share one handicap. +_SW_TRANSCODE_NICE = settings.download_child_nice + + +def _sw_child_preexec() -> None: + """Runs in the forked child (POSIX) before exec: renice this software-transcode ffmpeg down so it + can't starve the co-located API / Postgres / playback. Best-effort — a failed renice must not fail + playback.""" + try: + os.nice(_SW_TRANSCODE_NICE) + except OSError: + pass + + def _probe_audio_langs(src: Path) -> list[str]: """One entry per audio stream (its language tag, or "" if untagged). len() = audio-track count. Used to build the multi-rendition var_stream_map. Empty on probe failure → single-audio path.""" @@ -249,10 +288,14 @@ def _ffmpeg_cmd( audio_langs: list[str] | None = None, transcode: bool = False, hw: bool = False, + preset: str = "veryfast", + crf: int = 20, + max_height: int = 0, ) -> list[str]: # For a transcode session the video is re-encoded (VAAPI or libx264) instead of stream-copied; the # hwaccel decode options go before -i, the encoder args replace `-c:v copy` below. Everything else is # identical to remux. + venc = _venc_opts(transcode, hw, preset, crf, max_height) args = ["ffmpeg", "-nostdin", "-loglevel", "error", *_hwaccel_in_opts(hw)] # Keep ORIGINAL timestamps on the output so the first segment carries the true absolute PTS of the # keyframe ffmpeg actually seeked to. hls.js still zero-bases playback to that PTS; we read it back @@ -274,7 +317,7 @@ def _ffmpeg_cmd( args += ["-map", "0:v:0"] for i in range(len(audio_langs)): args += ["-map", f"{ai}:a:{i}?"] - args += ["-sn", *_venc_opts(transcode, hw), "-c:a", "aac", "-ac", "2", "-b:a", "192k"] + args += ["-sn", *venc, "-c:a", "aac", "-ac", "2", "-b:a", "192k"] var = ["v:0,agroup:aud"] for i, lang in enumerate(audio_langs): entry = f"a:{i},agroup:aud,name:a{i}" @@ -295,7 +338,7 @@ def _ffmpeg_cmd( else: ao = audio_ord if audio_ord is not None else 0 args += ["-map", "0:v:0", "-map", f"{ai}:a:{ao}?", "-sn"] - args += [*_venc_opts(transcode, hw), "-c:a", "aac", "-ac", "2", "-b:a", "192k"] + args += [*venc, "-c:a", "aac", "-ac", "2", "-b:a", "192k"] args += _HLS_TAIL + [ "-hls_segment_filename", str(out_dir / "seg_%d.ts"), @@ -380,9 +423,20 @@ def start_session( return None key = item.rating_key start_s = max(0.0, float(start_s)) - hw = _want_hw(item, transcode) + hw = _want_hw(db, item, transcode) + # Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and remux + # ignore preset/crf/max_height, and remux must not touch the DB for them). + preset, crf, max_height = "veryfast", 20, 0 + if transcode and not hw: + preset = sysconfig.get_str(db, "plex_transcode_preset") or "veryfast" + crf = sysconfig.get_int(db, "plex_transcode_crf") + max_height = sysconfig.get_int(db, "plex_transcode_max_height") audio_langs = _probe_audio_langs(src) if multi else [] is_multi = len(audio_langs) > 1 # only worth a master playlist when there really are ≥2 tracks + # Nice the SOFTWARE transcode child (not the HW path, and not remux) so a CPU-bound re-encode can't + # starve the co-located API/playback. preexec_fn runs in the forked child (POSIX); on a platform + # without fork (the API always runs in a Linux container in prod) it's simply omitted. + preexec = _sw_child_preexec if (transcode and not hw and os.name == "posix") else None with _lock: old = _sessions.pop(key, None) if old is not None: @@ -401,9 +455,13 @@ def start_session( audio_langs if is_multi else None, transcode=transcode, hw=hw, + preset=preset, + crf=crf, + max_height=max_height, ), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + preexec_fn=preexec, ) s = HlsSession( key, diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 05299179..1fa7802f 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -159,6 +159,14 @@ SPECS: tuple[ConfigSpec, ...] = ( ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"), # (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.) ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16), + # Transcode re-encode tuning (E6 S1). hw = "auto"|"on"|"off" (free-text, like search_source); the + # libx264 knobs apply only to the software fallback. crf 0–51 (x264 range); max_height 0 = source. + ConfigSpec("plex_transcode_hw", "str", "plex", "plex_transcode_hw"), + ConfigSpec("plex_transcode_preset", "str", "plex", "plex_transcode_preset"), + ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51), + ConfigSpec( + "plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320 + ), # NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job # (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS. # --- TMDB enrichment (E5 S2; optional) — the API key is the only secret --- diff --git a/backend/tests/test_plex_stream.py b/backend/tests/test_plex_stream.py index 3b0ba443..592bad9b 100644 --- a/backend/tests/test_plex_stream.py +++ b/backend/tests/test_plex_stream.py @@ -228,27 +228,77 @@ class TestTranscodeCommand: assert "-var_stream_map" in cmd # …but still multi-rendition (client-side audio switch) assert "copy" not in cmd + def test_software_preset_and_crf_come_from_the_knobs(self): + cmd = stream._ffmpeg_cmd( + self.SRC, 0.0, self.OUT, None, transcode=True, hw=False, preset="slow", crf=18 + ) + assert cmd[cmd.index("-preset") + 1] == "slow" + assert cmd[cmd.index("-crf") + 1] == "18" + + def test_software_max_height_caps_with_a_downscale_only_filter(self): + cmd = stream._ffmpeg_cmd( + self.SRC, 0.0, self.OUT, None, transcode=True, hw=False, max_height=720 + ) + # Downscale-only, aspect-preserving, even width; the comma in min() escaped so it's ONE filter. + assert cmd[cmd.index("-vf") + 1] == "scale=-2:min(ih\\,720)" + + def test_software_max_height_zero_means_no_scale_filter(self): + cmd = stream._ffmpeg_cmd( + self.SRC, 0.0, self.OUT, None, transcode=True, hw=False, max_height=0 + ) + assert "-vf" not in cmd # source resolution kept + + def test_hardware_ignores_max_height(self): + # The GPU encode is cheap, so it runs at source resolution; max_height is a software-only knob. + cmd = stream._ffmpeg_cmd( + self.SRC, 0.0, self.OUT, None, transcode=True, hw=True, max_height=720 + ) + assert cmd[cmd.index("-vf") + 1] == "scale_vaapi=format=nv12" # not a software scale + class TestWantHw: - """VAAPI is used only where the render node was passed in AND the source is HW-decodable.""" + """VAAPI is used only where the render node was passed in AND (in auto) the source is HW-decodable, + with an admin `plex_transcode_hw` policy (auto/on/off) on top.""" + + @staticmethod + def _policy(monkeypatch, mode): + monkeypatch.setattr(stream.sysconfig, "get_str", lambda db, k: mode) def test_no_hw_when_not_transcoding(self, monkeypatch): + self._policy(monkeypatch, "auto") monkeypatch.setattr(stream, "_render_node_present", lambda: True) - assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=False) is False + assert stream._want_hw(None, _FakeItem(codec_video="hevc"), transcode=False) is False def test_no_hw_without_a_render_node(self, monkeypatch): + self._policy(monkeypatch, "auto") monkeypatch.setattr(stream, "_render_node_present", lambda: False) - assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=True) is False + assert stream._want_hw(None, _FakeItem(codec_video="hevc"), transcode=True) is False - def test_hw_for_a_decodable_source_with_the_render_node(self, monkeypatch): + def test_auto_uses_hw_for_a_decodable_source_with_the_render_node(self, monkeypatch): + self._policy(monkeypatch, "auto") monkeypatch.setattr(stream, "_render_node_present", lambda: True) - assert stream._want_hw(_FakeItem(codec_video="hevc"), transcode=True) is True + assert stream._want_hw(None, _FakeItem(codec_video="hevc"), transcode=True) is True - def test_software_for_a_non_decodable_source_even_with_the_render_node(self, monkeypatch): + def test_auto_falls_back_to_software_for_a_non_decodable_source(self, monkeypatch): # AV1 / MPEG-4 ASP (XviD) aren't decodable on Gen 9.5 → the GPU can't feed its own encoder. + self._policy(monkeypatch, "auto") monkeypatch.setattr(stream, "_render_node_present", lambda: True) - assert stream._want_hw(_FakeItem(codec_video="av1"), transcode=True) is False - assert stream._want_hw(_FakeItem(codec_video="mpeg4"), transcode=True) is False + assert stream._want_hw(None, _FakeItem(codec_video="av1"), transcode=True) is False + assert stream._want_hw(None, _FakeItem(codec_video="mpeg4"), transcode=True) is False + + def test_off_forces_software_even_for_a_decodable_source(self, monkeypatch): + self._policy(monkeypatch, "off") + monkeypatch.setattr(stream, "_render_node_present", lambda: True) + assert stream._want_hw(None, _FakeItem(codec_video="hevc"), transcode=True) is False + + def test_on_forces_hw_even_for_a_non_decodable_source(self, monkeypatch): + # The admin override trusts the GPU to decode it; skips the decodability gate (render node still + # required — HW is impossible without it). + self._policy(monkeypatch, "on") + monkeypatch.setattr(stream, "_render_node_present", lambda: True) + assert stream._want_hw(None, _FakeItem(codec_video="av1"), transcode=True) is True + monkeypatch.setattr(stream, "_render_node_present", lambda: False) + assert stream._want_hw(None, _FakeItem(codec_video="hevc"), transcode=True) is False class TestColdStartWarmup: diff --git a/frontend/src/i18n/locales/en/config.json b/frontend/src/i18n/locales/en/config.json index 25ade895..179fd5a6 100644 --- a/frontend/src/i18n/locales/en/config.json +++ b/frontend/src/i18n/locales/en/config.json @@ -166,7 +166,23 @@ }, "plex_max_transcodes": { "label": "Max concurrent transcodes", - "hint": "Cap on simultaneous CPU transcodes for incompatible files (a later fallback). Direct-playable files aren't limited." + "hint": "Cap on simultaneous transcode sessions for non-H.264 files. Direct-play and remux (stream-copy) files aren't limited." + }, + "plex_transcode_hw": { + "label": "Hardware transcode", + "hint": "GPU (VAAPI) encoding policy: auto (GPU when available and the source is decodable, else software), on (force GPU), off (always software)." + }, + "plex_transcode_preset": { + "label": "Software preset", + "hint": "libx264 preset for the software fallback (e.g. veryfast, faster, medium). Slower presets shrink the stream but cost more CPU. Ignored on the GPU path." + }, + "plex_transcode_crf": { + "label": "Software quality (CRF)", + "hint": "libx264 CRF for the software fallback (0–51; lower = better quality, larger). 20 is a good default. Ignored on the GPU path." + }, + "plex_transcode_max_height": { + "label": "Software max height", + "hint": "Cap the software transcode's output height in pixels (e.g. 1080; 0 = keep source) so a tall source stays affordable to re-encode. The GPU path keeps source resolution." } }, "save": "Save", diff --git a/frontend/src/i18n/locales/hu/config.json b/frontend/src/i18n/locales/hu/config.json index a9ad80e9..64fa5b3d 100644 --- a/frontend/src/i18n/locales/hu/config.json +++ b/frontend/src/i18n/locales/hu/config.json @@ -166,7 +166,23 @@ }, "plex_max_transcodes": { "label": "Max párhuzamos transcode", - "hint": "A nem kompatibilis fájlok egyidejű CPU-transcode-jainak felső korlátja (későbbi tartalék). A direktben játszható fájlokra nincs korlát." + "hint": "A nem-H.264 fájlok egyidejű transcode-munkameneteinek felső korlátja. A direktben játszható és a remux (stream-copy) fájlokra nincs korlát." + }, + "plex_transcode_hw": { + "label": "Hardveres transcode", + "hint": "GPU (VAAPI) kódolási házirend: auto (GPU, ha elérhető és a forrás dekódolható, különben szoftver), on (GPU kényszerítése), off (mindig szoftver)." + }, + "plex_transcode_preset": { + "label": "Szoftveres preset", + "hint": "libx264 preset a szoftveres tartalékhoz (pl. veryfast, faster, medium). A lassabb preset kisebb streamet ad, de több CPU-t eszik. A GPU-úton figyelmen kívül marad." + }, + "plex_transcode_crf": { + "label": "Szoftveres minőség (CRF)", + "hint": "libx264 CRF a szoftveres tartalékhoz (0–51; alacsonyabb = jobb minőség, nagyobb méret). A 20 jó alapérték. A GPU-úton figyelmen kívül marad." + }, + "plex_transcode_max_height": { + "label": "Szoftveres max. magasság", + "hint": "A szoftveres transcode kimeneti magasságának korlátja képpontban (pl. 1080; 0 = forrás megtartása), hogy egy magas forrás újrakódolása megfizethető maradjon. A GPU-út a forrásfelbontást tartja." } }, "save": "Mentés",