peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(plex): transcode resource discipline + admin knobs (E6 S1c)

Browse Source 
The software-fallback re-encode now runs deprioritized and admin-configurable.
The SW transcode ffmpeg child is niced (preexec_fn → os.nice, applied per-child
because the Popen lives in the long-lived API process, which must NOT itself be
niced — unlike the download worker's whole-process _deprioritize) so a CPU-bound
re-encode can't starve request handling or playback; the VAAPI path needs none of
it.

New sysconfig knobs (plex group, rendered on the admin Configuration page, i18n
HU/EN): plex_transcode_hw (auto/on/off, layered onto _want_hw), plex_transcode_
preset, plex_transcode_crf, plex_transcode_max_height (software-only downscale-to
cap; the GPU path encodes at source resolution). plex_max_transcodes stays the
concurrency cap (comment corrected — the fallback is shipped, not "later").

pytest extends the command-builder + _want_hw coverage (preset/crf/max_height
plumbing, the on/off/auto policy). Browser-UAT'd: max_height=240 drops the live
resolution to 330x240; the SW ffmpeg child runs at nice 10 (ppid=api process).
This commit is contained in:
peter
2026-08-30 01:22:30 +02:00
parent 0658f9550b
commit ce96032edb
6 changed files with 193 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/plex/stream.py
+78 -20
View File
@@ -19,6 +19,7 @@ CPU-only prod host; only P3's full transcode is CPU-heavy.
"""
import logging
import os
import re
import shutil
import subprocess
@@ -167,35 +168,49 @@ def _hwaccel_in_opts(hw: bool) -> list[str]:
return ["-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", "-vaapi_device", _VAAPI_DEVICE]
def _venc_opts(transcode: bool, hw: bool) -> list[str]:
"""The video-encoder args that replace `-c:v copy` for a transcode session (empty-of-meaning:
returns the copy args when not transcoding). A re-encode must MINT its own keyframes at the segment
boundaries — the copy path leaned on the source's keyframes, but a re-encoded stream has none the
HLS muxer can cut on unless we force them (`-force_key_frames` every `_SEG_SECONDS`), else segments
aren't independent and seeking breaks."""
def _venc_opts(
transcode: bool,
hw: bool,
preset: str = "veryfast",
crf: int = 20,
max_height: int = 0,
) -> list[str]:
"""The video-encoder args that replace `-c:v copy` for a transcode session (returns the copy args
when not transcoding). A re-encode must MINT its own keyframes at the segment boundaries — the copy
path leaned on the source's keyframes, but a re-encoded stream has none the HLS muxer can cut on
unless we force them (`-force_key_frames` every `_SEG_SECONDS`), else segments aren't independent and
seeking breaks. `preset`/`crf`/`max_height` (software only) come from admin sysconfig."""
if not transcode:
return ["-c:v", "copy"]
force_kf = ["-force_key_frames", f"expr:gte(t,n_forced*{_SEG_SECONDS})"]
if hw:
# scale_vaapi with format=nv12 downconverts a 10-bit (p010) HEVC source to 8-bit nv12 so
# h264_vaapi (8-bit High) can encode it; for an 8-bit source it's a cheap format pass-through.
# The GPU encode is cheap, so it runs at source resolution (max_height is a software-only knob).
return ["-vf", "scale_vaapi=format=nv12", "-c:v", "h264_vaapi", *force_kf]
# Software fallback: same browser-safe recipe as downloads/edit.py (libx264 veryfast/crf20).
# yuv420p forces 8-bit High (not High10) so any browser decodes it; -threads reserves a core so the
# CPU-bound encode can't starve request handling / playback on the shared box.
return [
# Software fallback: same browser-safe recipe as downloads/edit.py (libx264). yuv420p forces 8-bit
# High (not High10) so any browser decodes it; -threads reserves a core so the CPU-bound encode can't
# starve request handling / playback on the shared box.
args: list[str] = []
if max_height and max_height > 0:
# Downscale-ONLY to the height cap (min(ih,H)), keeping aspect with an even width (-2), so a tall
# source doesn't make the software re-encode ruinously expensive. The comma in min() is escaped
# (\,) so ffmpeg's filter parser reads one filter, not two. Never upscales a smaller source.
args += ["-vf", f"scale=-2:min(ih\\,{max_height})"]
args += [
"-c:v",
"libx264",
"-preset",
"veryfast",
preset,
"-crf",
"20",
str(crf),
"-pix_fmt",
"yuv420p",
"-threads",
str(encode_thread_cap()),
*force_kf,
]
return args
def _render_node_present() -> bool:
@@ -205,15 +220,39 @@ def _render_node_present() -> bool:
return Path(_VAAPI_DEVICE).exists()
def _want_hw(item: PlexItem, transcode: bool) -> bool:
"""Whether to use the VAAPI hardware encoder for this transcode session. True iff we're
transcoding AND the render node is present AND the iGPU can hardware-decode the source (else the
full-GPU pipeline can't feed itself → software libx264). (S1c layers an admin on/off/auto override.)"""
if not transcode or not _render_node_present():
def _want_hw(db: DbSession, item: PlexItem, transcode: bool) -> bool:
"""Whether to use the VAAPI hardware encoder for this transcode session, honouring the admin
`plex_transcode_hw` policy: "off" → always software; "on" → VAAPI wherever the render node exists
(trusting the admin, skipping the source-decodability gate); "auto" (default) → VAAPI only when the
render node is present AND the iGPU can hardware-decode the source (else the full-GPU pipeline can't
feed itself → software libx264)."""
if not transcode:
return False
mode = (sysconfig.get_str(db, "plex_transcode_hw") or "auto").lower()
if mode == "off" or not _render_node_present():
return False
if mode == "on":
return True
return (item.codec_video or "").lower() in _HW_DECODABLE
# CPU-nice increment for the SOFTWARE transcode child. The Plex ffmpeg is a Popen in the long-lived API
# process (which must NOT itself be niced), so — unlike the download worker's _deprioritize (run once in
# its own main()) — we lower priority per-child via preexec_fn in the fork. The GPU path needs none of
# this. Reuses the download child's nice default so all co-located re-encoders share one handicap.
_SW_TRANSCODE_NICE = settings.download_child_nice
def _sw_child_preexec() -> None:
"""Runs in the forked child (POSIX) before exec: renice this software-transcode ffmpeg down so it
can't starve the co-located API / Postgres / playback. Best-effort — a failed renice must not fail
playback."""
try:
os.nice(_SW_TRANSCODE_NICE)
except OSError:
pass
def _probe_audio_langs(src: Path) -> list[str]:
"""One entry per audio stream (its language tag, or "" if untagged). len() = audio-track count.
Used to build the multi-rendition var_stream_map. Empty on probe failure → single-audio path."""
@@ -249,10 +288,14 @@ def _ffmpeg_cmd(
audio_langs: list[str] | None = None,
transcode: bool = False,
hw: bool = False,
preset: str = "veryfast",
crf: int = 20,
max_height: int = 0,
) -> list[str]:
# For a transcode session the video is re-encoded (VAAPI or libx264) instead of stream-copied; the
# hwaccel decode options go before -i, the encoder args replace `-c:v copy` below. Everything else is
# identical to remux.
venc = _venc_opts(transcode, hw, preset, crf, max_height)
args = ["ffmpeg", "-nostdin", "-loglevel", "error", *_hwaccel_in_opts(hw)]
# Keep ORIGINAL timestamps on the output so the first segment carries the true absolute PTS of the
# keyframe ffmpeg actually seeked to. hls.js still zero-bases playback to that PTS; we read it back
@@ -274,7 +317,7 @@ def _ffmpeg_cmd(
args += ["-map", "0:v:0"]
for i in range(len(audio_langs)):
args += ["-map", f"{ai}:a:{i}?"]
args += ["-sn", *_venc_opts(transcode, hw), "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
args += ["-sn", *venc, "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
var = ["v:0,agroup:aud"]
for i, lang in enumerate(audio_langs):
entry = f"a:{i},agroup:aud,name:a{i}"
@@ -295,7 +338,7 @@ def _ffmpeg_cmd(
else:
ao = audio_ord if audio_ord is not None else 0
args += ["-map", "0:v:0", "-map", f"{ai}:a:{ao}?", "-sn"]
args += [*_venc_opts(transcode, hw), "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
args += [*venc, "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
args += _HLS_TAIL + [
"-hls_segment_filename",
str(out_dir / "seg_%d.ts"),
@@ -380,9 +423,20 @@ def start_session(
return None
key = item.rating_key
start_s = max(0.0, float(start_s))
hw = _want_hw(item, transcode)
hw = _want_hw(db, item, transcode)
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and remux
# ignore preset/crf/max_height, and remux must not touch the DB for them).
preset, crf, max_height = "veryfast", 20, 0
if transcode and not hw:
preset = sysconfig.get_str(db, "plex_transcode_preset") or "veryfast"
crf = sysconfig.get_int(db, "plex_transcode_crf")
max_height = sysconfig.get_int(db, "plex_transcode_max_height")
audio_langs = _probe_audio_langs(src) if multi else []
is_multi = len(audio_langs) > 1 # only worth a master playlist when there really are ≥2 tracks
# Nice the SOFTWARE transcode child (not the HW path, and not remux) so a CPU-bound re-encode can't
# starve the co-located API/playback. preexec_fn runs in the forked child (POSIX); on a platform
# without fork (the API always runs in a Linux container in prod) it's simply omitted.
preexec = _sw_child_preexec if (transcode and not hw and os.name == "posix") else None
with _lock:
old = _sessions.pop(key, None)
if old is not None:
@@ -401,9 +455,13 @@ def start_session(
audio_langs if is_multi else None,
transcode=transcode,
hw=hw,
preset=preset,
crf=crf,
max_height=max_height,
),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
preexec_fn=preexec,
)
s = HlsSession(
key,