peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(plex): the subtitle extractor's seams — doubled cues, a truncated track cached as whole, a poll that respawned ffmpeg

Browse Source 
Deep-review of the bug-hunt round (R1). The background extractor from ed4ab4e1 held, but its seams
did not:

- Window + full pass doubled every cue in the first hour. merge_vtt de-duplicated on the raw timing
  text, and the two passes spell a timestamp differently (shift_vtt writes HH:MM:SS.mmm, ffmpeg's
  webvtt muxer drops the hour below 1 h; shift_vtt(-0.0) never normalised the full pass). Timing is
  now parsed and re-emitted canonically, keyed with the cue text so two cues sharing a time survive.
- A full pass that exited non-zero was published as complete: "ready" forever, subtitles ending
  where ffmpeg died, and no eviction to ever retry. It now covers only up to its last cue, is never
  complete, and records the failure memo.
- The window's dedup identity held the live playhead, so every 1 s poll queued another read of the
  file. It is keyed on the track now, and a queued job re-checks coverage before reading anything.
- A sidecar track waited behind an unrelated embedded full pass (the full job is per item). Its
  window job now fetches it whole — one small request.
- The <track> fetch ran the scheduler at position 0: an unrequested window over the head of the file,
  and a refusal of a window that was on disk. It is a pure read of the cache now.
- publish() is serialised (a lost update became permanent once complete was set); the pool is shut
  down and running ffmpeg children killed at app shutdown (they held interpreter exit for minutes);
  ?at=inf is a 422, not a 500; and both subtitle routes use current_user instead of the cookie-only
  poster gate, which let a suspended or epoch-revoked session keep scheduling reads (pre-existing).

_extract_and_publish finally has tests of its own — every earlier test stubbed it out. 19 mutations,
each red on the assertion that names it.
This commit is contained in:
peter
2026-09-11 00:52:59 +02:00
parent f76a2afbf1
commit cf7ec5e9ec
6 changed files with 508 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/plex/subtitles.py
+156 -37
View File
@@ -245,35 +245,77 @@ def covers(ranges: list[list[float]], at: float) -> bool:
def merge_vtt(*sources: str) -> str:
"""Union of the cue blocks of several absolute-time VTTs, ordered by start time.
Two extraction passes over the same file emit identical cues for the region they share, so
de-duplication keys off the TIMING line — the one part of a cue that identifies it uniquely and
that neither pass can renumber (ffmpeg's webvtt muxer writes no cue identifiers)."""
Two extraction passes over the same file emit identical cues for the region they share, so they
are de-duplicated — but NOT on the raw timing text. The two passes do not spell a timestamp the
same way: a window comes back through `shift_vtt`, which always writes `HH:MM:SS.mmm`, while a
full pass is ffmpeg's own output, and its webvtt muxer drops the hour below one hour
(`10:00.123`). Keyed on the text, every cue of a window in the first hour survived twice and the
viewer saw each line stacked on itself. So each timing line is parsed and re-emitted in one
canonical form, and the key is that canonical timing PLUS the cue text: two different cues that
happen to share a time (two speakers, a sign over dialogue) are both kept."""
seen: set[str] = set()
cues: list[tuple[float, str]] = []
for src in sources:
for block in re.split(r"\n[ \t]*\n", src or ""):
b = block.strip()
if not b or "-->" not in b:
canon = _canonical_cue(block.strip())
if canon is None:
continue # header / NOTE / STYLE — the merged file gets its own single header
timing = next(ln for ln in b.split("\n") if "-->" in ln)
if timing in seen:
start, key, text = canon
if key in seen:
continue
seen.add(timing)
try:
start = _vtt_ts_to_s(timing.partition("-->")[0].strip())
except (ValueError, IndexError):
continue
cues.append((start, b))
seen.add(key)
cues.append((start, text))
cues.sort(key=lambda c: c[0])
return "WEBVTT\n\n" + "\n\n".join(b for _, b in cues) + "\n"
def _canonical_cue(block: str) -> tuple[float, str, str] | None:
"""(start seconds, identity, the block with its timing line canonicalised) — or None for a block
that is not a cue. The identity leaves out anything before the timing line (an SRT-style cue
number), which is a label, not content."""
lines = block.split("\n")
for i, ln in enumerate(lines):
if "-->" not in ln:
continue
left, _, right = ln.partition("-->")
rparts = right.strip().split(None, 1)
settings = (" " + rparts[1]) if len(rparts) > 1 else ""
try:
st = _vtt_ts_to_s(left.strip())
en = _vtt_ts_to_s(rparts[0])
except (ValueError, IndexError):
return None
lines[i] = f"{_vtt_s_to_ts(st)} --> {_vtt_s_to_ts(en)}{settings}"
return st, "\n".join(lines[i:]), "\n".join(lines)
return None
def _last_cue_end(vtt: str) -> float:
"""Where the last cue of a VTT ends — how far a pass that died early provably got."""
end = 0.0
for ln in vtt.split("\n"):
if "-->" not in ln:
continue
try:
end = max(end, _vtt_ts_to_s(ln.partition("-->")[2].strip().split(None, 1)[0]))
except (ValueError, IndexError):
continue
return end
def publish(key: str, vtt: str, covered: tuple[float, float], complete: bool) -> None:
"""Merge a freshly extracted region into the cached track and widen its covered ranges."""
old = cache_read(key)
old_ranges, was_complete = read_meta(key)
cache_write(key, merge_vtt(old or "", vtt))
_write_meta(key, merge_ranges(old_ranges + [list(covered)]), complete or was_complete)
"""Merge a freshly extracted region into the cached track and widen its covered ranges.
A read-merge-write, so it runs under `_PUBLISH_LOCK`: two jobs publish into the same keys (a
window and the full pass, and every embedded track of the file rides each of them), and without
the lock the later writer drops whatever the earlier one merged — permanently, once `complete` is
set. It is also the only writer of the cache files, which is what keeps their fixed `.tmp`
staging names safe."""
with _PUBLISH_LOCK:
old = cache_read(key)
old_ranges, was_complete = read_meta(key)
cache_write(key, merge_vtt(old or "", vtt))
_write_meta(key, merge_ranges(old_ranges + [list(covered)]), complete or was_complete)
# --- background extraction ----------------------------------------------------------------------
@@ -298,6 +340,9 @@ _POOL = ThreadPoolExecutor(max_workers=3, thread_name_prefix="plex-subs")
_LOCK = threading.Lock()
_INFLIGHT: set[tuple] = set()
_FAILURES: dict[str, tuple[str, float]] = {}
_PUBLISH_LOCK = threading.Lock()
# The ffmpeg children currently running, so a shutdown can kill them (see `shutdown`).
_PROCS: set[subprocess.Popen] = set()
class SubtitleUnavailable(Exception):
@@ -328,6 +373,20 @@ def reset_state() -> None:
_FAILURES.clear()
def shutdown() -> None:
"""Stop extracting, for the app's shutdown. The pool's workers are non-daemon threads and Python
joins them at interpreter exit — so a worker still inside a full pass (minutes on a big file, up
to EXTRACT_TIMEOUT_S) held the process open after uvicorn had stopped, until the container runtime
SIGKILLed it. Queued jobs are dropped and running ffmpeg children are killed, which returns their
workers at once. Nothing that matters is lost: a publish is an atomic rename, and an interrupted
track is simply extracted again on the next poll."""
_POOL.shutdown(wait=False, cancel_futures=True)
with _LOCK:
procs = list(_PROCS)
for p in procs:
p.kill()
def _fail(key: str, code: str) -> None:
with _LOCK:
_FAILURES[key] = (code, time.monotonic())
@@ -362,25 +421,57 @@ def ensure(rating_key: str, sub_ord: int, at: float = 0.0) -> dict:
return {"state": "partial" if covered else "preparing", "code": None, "gen": generation(key)}
def cached_track(rating_key: str, sub_ord: int) -> tuple[str | None, str | None]:
"""What the <track> fetch can serve RIGHT NOW, scheduling nothing: the cached VTT (whatever it
covers), else the recent failure that explains why there is none.
Scheduling belongs to `ensure` — the status poll, which knows where the viewer is. The fetch does
not: when it scheduled too, it judged readiness at position 0, so a viewer who resumed an hour in
got an unrequested window over the head of the file (plus a remount when it landed), and with a
failure memo alive it was refused a window that WAS on disk while the poll called it showable."""
key = cache_key(rating_key, sub_ord)
hit = cache_read(key)
return (hit, None) if hit is not None else (None, _recent_failure(key))
def _schedule(rating_key: str, sub_ord: int, at: float, want_window: bool) -> None:
"""Kick off the jobs this track still needs: the window that makes it watchable now, and the one
full pass that makes it complete. Each is deduplicated by identity, so a 1 s status poll (or a
second viewer) joins the running job instead of starting another ffmpeg."""
second viewer) joins the running job instead of starting another ffmpeg.
A window's identity is its TRACK, never its position. The poll sends the live playhead, so a
position-keyed window minted a new identity every second and queued one more read of the file per
poll until the first window landed. One window per track is in flight at a time; a seek past it
is picked up by the next poll after it publishes. The full pass is per ITEM, because one read
fans out to every embedded text track of the file."""
jobs: list[tuple[tuple, float | None]] = []
if want_window:
w_start = max(0.0, at - WINDOW_PRE_S)
jobs.append((("win", rating_key, round(w_start)), w_start))
jobs.append((("win", rating_key, sub_ord), max(0.0, at - WINDOW_PRE_S)))
jobs.append((("full", rating_key), None))
for ident, window_start in jobs:
with _LOCK:
if ident in _INFLIGHT:
continue
_INFLIGHT.add(ident)
_POOL.submit(_run, ident, rating_key, sub_ord, window_start)
_POOL.submit(_run, ident, rating_key, sub_ord, window_start, at)
def _run(ident: tuple, rating_key: str, sub_ord: int, window_start: float | None) -> None:
def _already_done(key: str, window_start: float | None, at: float) -> bool:
"""A queued job looks again before it reads anything. While it waited, another job may have
covered it: every embedded track rides each pass over its file, so a window for one language also
publishes the others."""
if not _vtt_path(key).exists():
return False
ranges, complete = read_meta(key)
return complete or (window_start is not None and covers(ranges, at))
def _run(
ident: tuple, rating_key: str, sub_ord: int, window_start: float | None, at: float
) -> None:
try:
if _already_done(cache_key(rating_key, sub_ord), window_start, at):
return
_extract_and_publish(rating_key, sub_ord, window_start)
except SubtitleUnavailable as e:
_fail(cache_key(rating_key, sub_ord), e.code)
@@ -422,9 +513,11 @@ def _extract_and_publish(rating_key: str, sub_ord: int, window_start: float | No
want = subs[sub_ord]
if _is_bitmap(want):
raise SubtitleUnavailable("bitmap")
if want.get("key"): # external sidecar → one fetch, always the complete track
if window_start is not None:
return # nothing to window — the full job serves it in a single fetch
if want.get("key"):
# An external sidecar is one small fetch from Plex, never a read of the media file, so a
# window job fetches the whole thing too. It must not be left to the full job: that
# identity is per ITEM (the embedded fan-out shares it), so a sidecar picked while an
# embedded track's full pass was running sat behind that pass for minutes.
vtt = _sidecar_vtt(db, want)
publish(cache_key(rating_key, sub_ord), vtt, (0.0, _FULL_END_S), True)
return
@@ -443,13 +536,22 @@ def _extract_and_publish(rating_key: str, sub_ord: int, window_start: float | No
start = window_start or 0.0
length = (WINDOW_LEN_S + WINDOW_PRE_S) if window_start is not None else None
end = _FULL_END_S if length is None else start + length
for emb_ord, vtt in _ffmpeg_extract(str(src), sorted(targets), window_start, length).items():
texts, clean = _ffmpeg_extract(str(src), sorted(targets), window_start, length)
for emb_ord, vtt in texts.items():
key = cache_key(rating_key, targets[emb_ord])
# A pass that exited non-zero holds only what ffmpeg reached before it died. Claim no more
# than its last cue, never call it complete, and remember the failure: published as complete,
# the track read "ready" for good while its subtitles stopped wherever ffmpeg did (the cache
# has no eviction, so nothing would ever have extracted it again).
covered_end = end if clean else start + _last_cue_end(vtt)
publish(
cache_key(rating_key, targets[emb_ord]),
key,
shift_vtt(vtt, -start), # zero-based window clock → absolute cue times
(start, end),
length is None,
(start, covered_end),
clean and length is None,
)
if not clean:
_fail(key, "extract_failed")
def _sidecar_vtt(db: Session, stream: dict) -> str:
@@ -470,14 +572,16 @@ def _sidecar_vtt(db: Session, stream: dict) -> str:
def _ffmpeg_extract(
src: str, emb_ords: list[int], start: float | None, length: float | None
) -> dict[int, str]:
) -> tuple[dict[int, str], bool]:
"""Extract several embedded subtitle streams to WebVTT in ONE ffmpeg run (= one read of the file).
Returns the texts and whether ffmpeg exited cleanly — a non-zero exit can still leave partial
output, which is worth publishing but must never be taken for the whole track.
`start`/`length` bound the read to a window. Without `-copyts` the cues come out on a zero-based
clock (the caller shifts them back) — which is exactly why `-copyts` is absent: with it ffmpeg
ignores `-t` and reads the whole container anyway (measured)."""
if not emb_ords:
return {}
return {}, True
with tempfile.TemporaryDirectory(prefix="plexsub-") as tmpdir:
cmd = ["ffmpeg", "-nostdin", "-loglevel", "error"]
if start:
@@ -489,7 +593,22 @@ def _ffmpeg_extract(
if length is not None:
cmd += ["-t", f"{length:.3f}"]
cmd += ["-f", "webvtt", "-y", str(out)]
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S)
# Popen rather than run(): the child is registered so `shutdown` can kill it.
proc = subprocess.Popen(
cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
)
with _LOCK:
_PROCS.add(proc)
try:
try:
_, err = proc.communicate(timeout=EXTRACT_TIMEOUT_S)
except subprocess.TimeoutExpired:
proc.kill()
proc.communicate()
raise
finally:
with _LOCK:
_PROCS.discard(proc)
got = {
o: p.read_text(encoding="utf-8", errors="replace")
for o, p in outs.items()
@@ -497,7 +616,7 @@ def _ffmpeg_extract(
}
if not got:
raise SubtitleUnavailable("extract_failed")
if r.returncode != 0:
# Partial output with a non-zero exit still beats nothing (a later pass widens it) — say so.
log.warning("ffmpeg subtitle pass exited %s: %s", r.returncode, (r.stderr or b"")[:200])
return got
clean = proc.returncode == 0
if not clean:
log.warning("ffmpeg subtitle pass exited %s: %s", proc.returncode, (err or b"")[:200])
return got, clean