fix(downloads): R5 S1 — downloads robustness

- pick the produced file safely: Path("") is Path("."), which passes .exists(),
  so a missing yt-dlp filepath skipped the staging fallback and crashed in
  with_name ("PosixPath('.') has an empty name") on non-YouTube sources
- unwrap a playlist/multi_video extraction to the entry that carries the media
- run_ffmpeg: drain stderr while it runs (a full 64KB pipe blocked ffmpeg
  forever and hung the worker thread), 1s cancel polling, stall watchdog and a
  bounded wait on exit
- escape apostrophes/backslashes in the ffmpeg concat list (pre-0.51.0 names)
- sweep orphaned HLS segment dirs at startup (nothing else ever reaped them)
This commit is contained in:
2026-07-23 22:32:25 +02:00
parent 08049d2aa5
commit 2f46947dcf
5 changed files with 249 additions and 30 deletions
+105 -19
View File
@@ -24,7 +24,10 @@ import hashlib
import json
import logging
import math
import queue
import subprocess
import threading
import time
from pathlib import Path
log = logging.getLogger("siftlode.edit")
@@ -178,6 +181,16 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
return cmd
def _concat_quote(path: Path) -> str:
"""Escape a path for a single-quoted `file '…'` line of an ffmpeg concat list.
The demuxer's own rule: inside single quotes a literal `'` is written `'\\''` (close, escaped
quote, reopen) and a `\\` must be doubled. On-disk names have been ASCII-slugged since 0.51.0,
but that change is FORWARD-ONLY — a pre-0.51.0 file still on disk can be named `Don't_….mp4`,
and an unescaped apostrophe there makes the whole list unparseable."""
return path.as_posix().replace("\\", "\\\\").replace("'", "'\\''")
def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging: Path):
"""Plan a multi-segment JOIN (cut-list → one file). Returns (cmd, prep_files) where prep_files
maps a path → text content the worker must write before running (a concat list, if any).
@@ -211,7 +224,7 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
listp = staging / "concat.txt"
lines = []
for s in segs:
lines.append(f"file '{src.as_posix()}'")
lines.append(f"file '{_concat_quote(src)}'")
lines.append(f"inpoint {s['start_s']}")
lines.append(f"outpoint {s['end_s']}")
prep[listp] = "\n".join(lines) + "\n"
@@ -238,34 +251,107 @@ def _parse_out_time(line: str) -> float | None:
return None
def run_ffmpeg(cmd, total_s, on_progress, should_cancel) -> None:
# No output on `-progress pipe:1` for this long means ffmpeg is wedged (a stuck network source, a
# hung filter) — a download job may legitimately be slow, but it is never SILENT for minutes.
_FFMPEG_STALL_TIMEOUT_S = 600.0
# Hard cap after we ask the process to go away; past it we stop waiting on it altogether.
_FFMPEG_EXIT_GRACE_S = 10.0
def _pump(stream, sink: "queue.Queue[str | None]") -> None:
"""Read a text pipe to EOF into a queue (None = EOF). Runs in its own thread so the main loop
can time out and poll cancel instead of blocking forever on a silent ffmpeg."""
try:
for line in stream:
sink.put(line)
except (ValueError, OSError): # pipe closed under us by the kill path
pass
finally:
sink.put(None)
def _drain_tail(stream, sink: list[str]) -> None:
"""Consume a pipe to EOF, keeping only its tail.
ffmpeg's stderr MUST be read *while it runs*: the OS pipe buffer is ~64KB, and once it fills,
ffmpeg blocks forever on its next write — the process never exits, so `proc.wait()` never
returns and the worker thread is gone for good. Reading it only after the exit (as this used
to) is exactly the deadlock."""
try:
for line in stream:
sink.append(line)
if len(sink) > 200:
del sink[:100]
except (ValueError, OSError):
pass
def _stop(proc: subprocess.Popen) -> None:
"""Terminate, then kill — never wait unbounded on a process we've given up on."""
proc.terminate()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
log.error("ffmpeg pid %s survived SIGKILL", proc.pid)
def run_ffmpeg(cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S) -> None:
"""Run an ffmpeg edit, reporting 0–100 progress and honouring cooperative cancel.
Raises EditAborted if should_cancel() turns True mid-run, RuntimeError on a nonzero exit."""
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
cancel is honoured within a second even while ffmpeg is quiet, and a wedged process is killed
by the stall watchdog instead of pinning a worker thread forever.
Raises EditAborted if should_cancel() turns True mid-run, RuntimeError on a nonzero exit or a
stall."""
proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
)
err_tail: list[str] = []
lines: queue.Queue[str | None] = queue.Queue()
readers = [
threading.Thread(target=_pump, args=(proc.stdout, lines), daemon=True),
threading.Thread(target=_drain_tail, args=(proc.stderr, err_tail), daemon=True),
]
for t in readers:
t.start()
try:
assert proc.stdout is not None
for line in proc.stdout:
line = line.strip()
last_output = time.monotonic()
while True:
try:
line = lines.get(timeout=1.0)
except queue.Empty:
line = ""
if line is None: # stdout hit EOF — ffmpeg is done (or dying)
break
if line:
last_output = time.monotonic()
secs = _parse_out_time(line.strip())
if secs is not None and total_s:
on_progress(max(0.0, min(99.0, secs * 100.0 / total_s)))
if should_cancel():
proc.terminate()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
_stop(proc)
raise EditAborted
secs = _parse_out_time(line)
if secs is not None and total_s:
on_progress(max(0.0, min(99.0, secs * 100.0 / total_s)))
if time.monotonic() - last_output > stall_timeout_s:
_stop(proc)
raise RuntimeError(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
try:
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
except subprocess.TimeoutExpired: # pipes closed but the process lingers
_stop(proc)
raise RuntimeError("ffmpeg did not exit after closing its output")
finally:
if proc.stdout:
proc.stdout.close()
ret = proc.wait()
for t in readers:
t.join(timeout=_FFMPEG_EXIT_GRACE_S)
for pipe in (proc.stdout, proc.stderr):
if pipe:
pipe.close()
if ret != 0:
err = (proc.stderr.read() if proc.stderr else "") or ""
raise RuntimeError(f"ffmpeg failed ({ret}): {err[:300]}")
raise RuntimeError(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
# --- ffmpeg: universal browser playability --------------------------------------------------