Merge R5: data integrity (downloads & sync) + player big-view overhaul → 0.54.0

This commit is contained in:
2026-07-26 03:38:40 +02:00
28 changed files with 1967 additions and 153 deletions
+1 -1
View File
@@ -1 +1 @@
0.53.1
0.54.0
+139 -33
View File
@@ -24,7 +24,10 @@ import hashlib
import json
import logging
import math
import queue
import subprocess
import threading
import time
from pathlib import Path
log = logging.getLogger("siftlode.edit")
@@ -178,6 +181,17 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
return cmd
def _concat_quote(path: Path) -> str:
"""Escape a path for a single-quoted `file ''` line of an ffmpeg concat list.
ffmpeg's quoting rule: inside single quotes EVERY character is literal except `'` itself, which
is written `'\\''` (close the quote, emit an escaped quote, reopen). Backslashes are literal
there — doubling them would corrupt a path that contains one. On-disk names have been
ASCII-slugged since 0.51.0, but that change is FORWARD-ONLY: a pre-0.51.0 file still on disk can
be named `Don't_….mp4`, and an unescaped apostrophe makes the whole list unparseable."""
return path.as_posix().replace("'", "'\\''")
def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging: Path):
"""Plan a multi-segment JOIN (cut-list → one file). Returns (cmd, prep_files) where prep_files
maps a path → text content the worker must write before running (a concat list, if any).
@@ -211,7 +225,7 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
listp = staging / "concat.txt"
lines = []
for s in segs:
lines.append(f"file '{src.as_posix()}'")
lines.append(f"file '{_concat_quote(src)}'")
lines.append(f"inpoint {s['start_s']}")
lines.append(f"outpoint {s['end_s']}")
prep[listp] = "\n".join(lines) + "\n"
@@ -238,34 +252,114 @@ def _parse_out_time(line: str) -> float | None:
return None
def run_ffmpeg(cmd, total_s, on_progress, should_cancel) -> None:
# No output on `-progress pipe:1` for this long means ffmpeg is wedged (a stuck network source, a
# hung filter) — a download job may legitimately be slow, but it is never SILENT for minutes.
_FFMPEG_STALL_TIMEOUT_S = 600.0
# Hard cap after we ask the process to go away; past it we stop waiting on it altogether.
_FFMPEG_EXIT_GRACE_S = 10.0
def _pump(stream, sink: "queue.Queue[str | None]") -> None:
"""Read a text pipe to EOF into a queue (None = EOF). Runs in its own thread so the main loop
can time out and poll cancel instead of blocking forever on a silent ffmpeg."""
try:
for line in stream:
sink.put(line)
except (ValueError, OSError): # pipe closed under us by the kill path
pass
finally:
sink.put(None)
def _drain_tail(stream, sink: list[str]) -> None:
"""Consume a pipe to EOF, keeping only its tail.
ffmpeg's stderr MUST be read *while it runs*: the OS pipe buffer is ~64KB, and once it fills,
ffmpeg blocks forever on its next write — the process never exits, so `proc.wait()` never
returns and the worker thread is gone for good. Reading it only after the exit (as this used
to) is exactly the deadlock."""
try:
for line in stream:
sink.append(line)
if len(sink) > 200:
del sink[:100]
except (ValueError, OSError):
pass
def _stop(proc: subprocess.Popen) -> None:
"""Terminate, then kill — never wait unbounded on a process we've given up on. Safe to call on
a process that already exited (the single exit path below may reach it after a clean wait)."""
if proc.poll() is not None:
return
proc.terminate()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
log.error("ffmpeg pid %s survived SIGKILL", proc.pid)
def run_ffmpeg(cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S) -> None:
"""Run an ffmpeg edit, reporting 0100 progress and honouring cooperative cancel.
Raises EditAborted if should_cancel() turns True mid-run, RuntimeError on a nonzero exit."""
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
cancel is honoured within a second even while ffmpeg is quiet, and a wedged process is killed
by the stall watchdog instead of pinning a worker thread forever.
Raises EditAborted if should_cancel() turns True mid-run, RuntimeError on a nonzero exit or a
stall."""
proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
)
err_tail: list[str] = []
lines: queue.Queue[str | None] = queue.Queue()
readers = [
threading.Thread(target=_pump, args=(proc.stdout, lines), daemon=True),
threading.Thread(target=_drain_tail, args=(proc.stderr, err_tail), daemon=True),
]
for t in readers:
t.start()
try:
assert proc.stdout is not None
for line in proc.stdout:
line = line.strip()
last_output = time.monotonic()
while True:
try:
line = lines.get(timeout=1.0)
except queue.Empty:
line = ""
if line is None: # stdout hit EOF — ffmpeg is done (or dying)
break
if line:
last_output = time.monotonic()
secs = _parse_out_time(line.strip())
if secs is not None and total_s:
on_progress(max(0.0, min(99.0, secs * 100.0 / total_s)))
if should_cancel():
proc.terminate()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
raise EditAborted
secs = _parse_out_time(line)
if secs is not None and total_s:
on_progress(max(0.0, min(99.0, secs * 100.0 / total_s)))
if time.monotonic() - last_output > stall_timeout_s:
raise RuntimeError(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
try:
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
except subprocess.TimeoutExpired: # pipes closed but the process lingers
raise RuntimeError("ffmpeg did not exit after closing its output")
except BaseException:
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
# writes to the database, and an exception from THERE used to unwind past a live process,
# orphaning it (still encoding, no one left to reap it) after a 10s join on its pipes.
_stop(proc)
raise
finally:
if proc.stdout:
proc.stdout.close()
ret = proc.wait()
for t in readers:
t.join(timeout=_FFMPEG_EXIT_GRACE_S)
for pipe in (proc.stdout, proc.stderr):
if pipe:
pipe.close()
if ret != 0:
err = (proc.stderr.read() if proc.stderr else "") or ""
raise RuntimeError(f"ffmpeg failed ({ret}): {err[:300]}")
raise RuntimeError(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
# --- ffmpeg: universal browser playability --------------------------------------------------
@@ -278,29 +372,41 @@ _BROWSER_SAFE_VCODEC = ("h264",)
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
"""(video_codec, audio_codec) out of ffprobe's JSON stream list, lowercased.
JSON, not the flat `default=nw=1` output: there the fields arrive in ffprobe's own struct
order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line "remember the last
codec_type" parser attributes every name to the PREVIOUS stream. Measured on real downloads:
an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
believed the video stream was AAC and re-encoded a perfectly fine file on every download."""
try:
streams = json.loads(payload or "{}").get("streams") or []
except ValueError:
return None, None
vcodec = acodec = None
for s in streams:
name = (s.get("codec_name") or "").lower() or None
ctype = (s.get("codec_type") or "").lower()
if ctype == "video" and vcodec is None:
vcodec = name
elif ctype == "audio" and acodec is None:
acodec = name
return vcodec, acodec
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
try:
out = subprocess.run(
["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name",
"-of", "default=nw=1", str(path)],
"-of", "json", str(path)],
capture_output=True, text=True, timeout=30, check=True,
).stdout.lower()
).stdout
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
return None, None
vcodec = acodec = None
ctype = None
for line in out.splitlines():
if line.startswith("codec_type="):
ctype = line.split("=", 1)[1].strip()
elif line.startswith("codec_name="):
name = line.split("=", 1)[1].strip() or None
if ctype == "video" and vcodec is None:
vcodec = name
elif ctype == "audio" and acodec is None:
acodec = name
return vcodec, acodec
return parse_probe_streams(out)
def probe_duration(path: Path) -> int:
+32 -1
View File
@@ -1,6 +1,7 @@
import asyncio
import logging
import sys
from contextlib import asynccontextmanager
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from fastapi import FastAPI, HTTPException
@@ -27,6 +28,7 @@ from starlette.middleware.sessions import SessionMiddleware
from app import auth, state
from app.config import settings
from app.db import SessionLocal
from app.plex import stream as plex_stream
from app.routes import (
admin,
audit as audit_routes,
@@ -67,14 +69,43 @@ async def lifespan(app: FastAPI):
" Open the install wizard (internal access only):\n %s",
url,
)
# No HLS session survives a restart (they live in this process), so anything still on disk is
# a crashed run's segments — sweep them or they accumulate forever. FIRE AND FORGET, not
# awaited: uvicorn does not serve a single request until lifespan startup returns, so awaiting
# the sweep would hold the port shut for as long as the rmtree runs (gigabytes of `.ts` after a
# long uptime) — the 502 + failed post-deploy health probe this is meant to avoid. A thread
# alone doesn't buy that; not waiting does. The sweep only touches directories no live session
# owns, so racing it against early playback is safe.
sweep = asyncio.create_task(_sweep_hls_orphans())
start_scheduler()
try:
yield
finally:
log.info("Siftlode shutting down")
# Awaited, not just cancelled: a bare `.cancel()` lets the loop close with the task still
# pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
# triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
# bookkeeping honest.
sweep.cancel()
with suppress(asyncio.CancelledError):
await sweep
shutdown_scheduler()
async def _sweep_hls_orphans() -> None:
"""Background half of the startup HLS sweep (see lifespan). Owns its own errors: a failure here
must never take the app down, and a cancelled shutdown is not one."""
try:
swept = await asyncio.to_thread(plex_stream.sweep_orphans)
except asyncio.CancelledError:
raise
except Exception:
log.exception("HLS orphan sweep failed")
return
if swept:
log.info("swept %d orphaned HLS segment directories", swept)
app = FastAPI(title=settings.app_name, lifespan=lifespan)
app.add_middleware(
+73
View File
@@ -15,6 +15,7 @@ segments. The remux is CPU-light (video copy + a tiny audio transcode), so it ru
CPU-only prod host; only P3's full transcode is CPU-heavy.
"""
import logging
import re
import shutil
import subprocess
import threading
@@ -33,6 +34,32 @@ log = logging.getLogger("siftlode.plex")
_HLS_ROOT = Path(settings.plex_hls_dir) if settings.plex_hls_dir else Path(settings.download_root) / ".plex-hls"
_SEG_SECONDS = 6
_SESSION_IDLE_S = 600 # reap a session with no access for this long
# A session directory is named `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}` (see start_session);
# before multi-audio it was just `{rating_key}_{int(start_s)}`. `sweep_orphans` deletes ONLY names
# matching one of those two shapes: `_HLS_ROOT` is admin-configurable (PLEX_HLS_DIR), so it may
# legitimately point at a shared scratch path we do not own, and a blanket "delete every directory
# under the root" would take the neighbours with it.
#
# The KEY is anchored to digits (a Plex rating_key is a number) — that is what keeps foreign names
# out. A `.+` there let `snapshot_2024_backup_+1.00` through, and pairing `.+` with an optional tail
# degenerated all the way to "anything ending in _<digits>" (`backup_2024`, `release_10`). A
# non-numeric rating_key would merely leave its directory unswept, which is the safe direction to
# fail. Covered by test_plex_stream.py — this regex is the only thing standing between the sweep
# and someone else's data.
_SESSION_DIR_RE = re.compile(r"^\d+_\d+(_[^_]+_[+-]\d+\.\d{2})?$")
# When this process started. `sweep_orphans` only deletes directories older than this: everything
# newer belongs to a session THIS run created, and since `start_session` reuses a directory path
# for the same (rating_key, offset, tag), "stale name" is not the same as "stale directory".
_PROCESS_START = time.time()
# Slack on that comparison, because the two sides come from DIFFERENT clocks. A file's mtime is
# stamped from the kernel's COARSE clock (updated once per timer tick), while `time.time()` reads
# the fine-grained one — measured in this very container, a directory created 2ms AFTER
# `_PROCESS_START` got an mtime 2ms BEFORE it. Without slack the sweep would classify a session
# started in the first instants after boot as a leftover and delete it while ffmpeg writes into it
# — and "the first instants after boot" is exactly when the sweep runs. Erring on the keep side
# costs nothing: a genuine leftover from the last seconds before the restart simply survives until
# the next one.
_MTIME_GRACE_S = 2.0
_lock = threading.Lock()
_sessions: dict[str, "HlsSession"] = {}
@@ -254,6 +281,52 @@ def wait_for(path: Path, timeout: float = 20.0) -> bool:
return path.exists()
def sweep_orphans(older_than: float | None = None) -> int:
"""Delete OUR segment directories that no live session owns.
Sessions live only in THIS process, so at startup every session directory under `_HLS_ROOT` is
a leftover from a crashed or killed run — and nothing else ever reaps them (the download GC
deliberately skips `.plex-hls` as a system tree), so a few restarts mid-playback leave
gigabytes of `.ts` segments behind forever.
Only entries whose name matches `_SESSION_DIR_RE` are touched: the root is admin-configurable,
so anything else under it belongs to someone else and is left alone. Blocking (rmtree) — call
it off the event loop.
TWO guards against deleting a LIVE session, because this now runs CONCURRENTLY with request
handling (the caller stopped awaiting it so startup isn't held up):
* `older_than` (default: this process's start, minus `_MTIME_GRACE_S` of cross-clock slack) —
a directory touched since we booted was made by THIS run, so it is by definition not a
leftover. This is the guard that matters, because `start_session` REUSES the same path for
the same (key, offset, tag): a stale directory can become a live one at any moment.
* the live-session check and the rmtree happen together under `_lock`, so a session can't be
registered in the gap between them. The lock is held for one directory at a time — long
enough to be atomic, short enough that a boot-time sweep of a big backlog doesn't stall
playback."""
cutoff = (_PROCESS_START - _MTIME_GRACE_S) if older_than is None else older_than
dropped = 0
try:
entries = list(_HLS_ROOT.iterdir())
except OSError: # the root doesn't exist yet — nothing to sweep
return 0
for p in entries:
if not _SESSION_DIR_RE.match(p.name):
continue
if not p.is_dir():
continue
try:
if p.stat().st_mtime >= cutoff:
continue # this run made (or reused) it — never ours to delete
except OSError: # vanished under us (a concurrent restart of that session)
continue
with _lock:
if any(s.dir == p for s in _sessions.values()):
continue
shutil.rmtree(p, ignore_errors=True)
dropped += 1
return dropped
def reap_idle() -> int:
now = time.time()
dropped = 0
+5 -2
View File
@@ -52,8 +52,11 @@ def sync_subscriptions(
def sync_rss(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
new = run_rss_poll(db, _user_channels(db, user))
return {"new_videos": new}
# A partial failure comes back in `failed_feeds` so a manual poll reports it instead of an
# innocent-looking "0 new"; a TOTAL outage raises out of run_rss_poll and surfaces as a 500,
# which is the honest answer for a poll that reached nothing.
result = run_rss_poll(db, _user_channels(db, user))
return {"new_videos": result["new"], "failed_feeds": result["failed"]}
@router.post("/backfill")
+94 -17
View File
@@ -7,7 +7,9 @@ never synced. Videos in a playlist that aren't in our shared catalog yet are fet
ingested (with a stub channel) so the mirror is faithful."""
import hashlib
import logging
from collections.abc import Callable
from datetime import datetime, timezone
from typing import TypeVar
from sqlalchemy import delete, select
from sqlalchemy.orm import Session
@@ -263,6 +265,71 @@ def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
}
# After this many consecutive write failures we stop calling YouTube for the rest of the push.
# The failure modes that matter here are all account-wide, not per-video: quota exhausted, the
# write scope revoked, the token dead. Every remaining write then fails identically — but each
# one still SPENDS 50 units, so a long playlist burns through the whole (already dead) budget,
# and tomorrow's too, to produce a failure list we knew after the fifth item.
_MAX_CONSECUTIVE_FAILURES = 5
class _Failed:
"""Type of the `FAILED` sentinel, so `run()` can stay generic in the op's return type: a call
site gets back `str | _Failed` (not a bare `object`) and keeps its item id typed."""
#: Returned by `_Breaker.run` when the write did not happen (refused or failed).
FAILED = _Failed()
T = TypeVar("T")
class _Breaker:
"""Consecutive-failure circuit breaker for a single push run. Any success re-arms it.
Drive it with `run()` — it owns the whole open-check / call / record-outcome sequence, so a
write site can't accidentally implement two thirds of the protocol (forgetting to record a
failure would silently disable the breaker for that operation)."""
def __init__(self, limit: int = _MAX_CONSECUTIVE_FAILURES) -> None:
self.limit = limit
self.streak = 0
self.tripped = False
def open(self) -> bool:
"""True while writes may still be attempted."""
return not self.tripped
def hit(self, ok: bool) -> None:
if ok:
self.streak = 0
return
self.streak += 1
if self.streak >= self.limit and not self.tripped:
self.tripped = True
log.warning(
"playlist push: %d consecutive YouTube write failures — stopping this run "
"(quota, scope or token); the remaining items are reported as failures",
self.limit,
)
def run(self, op: Callable[[], T], on_fail: Callable[[], None]) -> T | _Failed:
"""Attempt one YouTube write. Returns the op's result, or `FAILED` when the breaker is
already tripped (op not called) or the op raised YouTubeError. `on_fail` records the item
in the caller's failure list either way."""
if self.tripped:
on_fail()
return FAILED
try:
out = op()
except YouTubeError:
on_fail()
self.hit(False)
return FAILED
self.hit(True)
return out
def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"""Push a local playlist to YouTube: create it if needed, then reconcile membership
and order so YouTube matches local (local wins). Marks the playlist clean on success.
@@ -271,6 +338,7 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
desired = _local_video_ids(db, pl)
inserted = deleted = reordered = created = 0
failures: list[str] = []
breaker = _Breaker()
with YouTubeClient(db, user) as yt:
if not pl.yt_playlist_id:
pl.yt_playlist_id = yt.create_playlist(pl.name)
@@ -278,11 +346,12 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
created = 1
# Brand-new playlist: just append every video in order.
for vid in desired:
try:
yt.add_playlist_item(pl.yt_playlist_id, vid)
added = breaker.run(
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
lambda v=vid: failures.append(v),
)
if not isinstance(added, _Failed):
inserted += 1
except YouTubeError:
failures.append(vid)
if not failures:
pl.synced_fingerprint = fingerprint(pl.name, desired)
pl.dirty = False
@@ -293,6 +362,7 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"deleted": deleted,
"reordered": reordered,
"failures": failures,
"aborted": breaker.tripped,
"yt_playlist_id": pl.yt_playlist_id,
}
@@ -313,20 +383,23 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
# Delete extras (local is authoritative).
for it in current:
if it["video_id"] not in desired_set:
try:
yt.delete_playlist_item(it["item_id"])
dropped = breaker.run(
lambda i=it: yt.delete_playlist_item(i["item_id"]),
lambda i=it: failures.append(i["video_id"]),
)
if not isinstance(dropped, _Failed):
deleted += 1
except YouTubeError:
failures.append(it["video_id"])
# Insert missing (append; the reorder pass below fixes positions).
cur_set = set(cur_vids)
for vid in desired:
if vid not in cur_set:
try:
item_id_by_vid[vid] = yt.add_playlist_item(pl.yt_playlist_id, vid)
item_id = breaker.run(
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
lambda v=vid: failures.append(v),
)
if not isinstance(item_id, _Failed):
item_id_by_vid[vid] = item_id
inserted += 1
except YouTubeError:
failures.append(vid)
# Reorder to match `desired` via insertion-sort moves over a local model.
model = [v for v in cur_vids if v in desired_set] + [
v for v in desired if v not in cur_set and v in item_id_by_vid
@@ -336,12 +409,15 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
continue # failed insert; skip
if i < len(model) and model[i] == want:
continue
try:
yt.move_playlist_item(item_id_by_vid[want], pl.yt_playlist_id, want, i)
reordered += 1
except YouTubeError:
failures.append(want)
moved = breaker.run(
lambda w=want, pos=i: yt.move_playlist_item(
item_id_by_vid[w], pl.yt_playlist_id, w, pos
),
lambda w=want: failures.append(w),
)
if isinstance(moved, _Failed):
continue
reordered += 1
j = model.index(want)
model.pop(j)
model.insert(i, want)
@@ -355,6 +431,7 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
"deleted": deleted,
"reordered": reordered,
"failures": failures,
"aborted": breaker.tripped,
"yt_playlist_id": pl.yt_playlist_id,
}
+56 -10
View File
@@ -25,7 +25,7 @@ from app.sync.videos import (
run_shorts_classification,
)
from app.youtube.client import YouTubeClient
from app.youtube.rss import fetch_channel_feed
from app.youtube.rss import RssError, fetch_channel_feed
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
RSS_POLL_WORKERS = 16
@@ -48,12 +48,31 @@ def get_service_user(db: Session) -> User | None:
return users[0] if users else None
def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> int:
def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str, int]:
"""Poll every channel's free RSS feed. Returns `{"new": inserted, "failed": channels_that_did
_not_complete}` — the failure count is part of the RESULT, not just a log line, so the scheduler
card and the audit summary can tell "no new uploads" apart from "nothing worked" (an egress
outage, or a database that can't accept writes, otherwise renders identically to a quiet hour).
`failed` counts a channel whose feed could not be READ *or* whose rows could not be APPLIED —
both leave the channel un-polled, and counting only the first would move the blind spot rather
than remove it.
Raises RuntimeError when NOTHING could be polled (`failed == total`, total > 0): a run that
achieved literally nothing is a failed run, and only an exception turns the scheduler card red
— a returned count leaves it green with the outage buried in the summary line. The message
carries the FIRST underlying error, because on a small instance "all channels failed" is also
what a single dead feed id looks like."""
if channels is None:
channels = db.execute(select(Channel)).scalars().all()
total = len(channels)
new = 0
done = 0
failed = 0
# Kept for the total-outage message: "failed for all N channels" alone can't tell an egress
# outage from one permanently dead feed id — which is the whole difference on an instance with
# a single subscription, where BOTH render as 100% failed.
first_error: str | None = None
# Fetch feeds concurrently (network-bound, no DB), then apply inserts on this thread as
# each fetch returns — a SQLAlchemy session isn't thread-safe. Pass the channel id (a
# plain str) to the workers; the ORM instance is touched only here on the main thread.
@@ -65,17 +84,44 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> int:
channel = futures[fut]
try:
entries = fut.result()
except Exception:
# A failed poll is NOT an empty feed: skip the apply so `last_rss_at` stays where it
# was (stamping it made a permanently-broken feed look freshly polled). The shared
# counter/progress tail lives after the statement, so the two outcomes can stay two
# clauses: an RssError is EXPECTED (404, timeout) and gets one warning line; anything
# else is a bug on our side and earns a traceback.
except RssError as exc:
log.warning("RSS poll skipped for channel %s: %s", channel.id, exc)
failed += 1
first_error = first_error or f"{channel.id}: {exc}"
except Exception as exc:
log.exception("RSS fetch failed for channel %s", channel.id)
entries = []
try:
new += apply_rss_feed(db, channel, entries)
except Exception:
db.rollback()
log.exception("RSS apply failed for channel %s", channel.id)
failed += 1
first_error = first_error or f"{channel.id}: {exc}"
else:
try:
new += apply_rss_feed(db, channel, entries)
except Exception as exc:
# A write failure (read-only DB, failover) leaves the channel just as un-polled
# as an unreachable feed — count it, or a wholly broken run reports "failed=0".
db.rollback()
failed += 1
first_error = first_error or f"{channel.id} (apply): {exc}"
log.exception("RSS apply failed for channel %s", channel.id)
# One place that advances the counter, whatever happened above.
done += 1
progress.report(done, total, "rss_poll")
return new
if failed:
log.warning("RSS poll: %d/%d channels could not be polled", failed, total)
if total and failed == total:
# Nothing was polled at all — an egress outage (the tinyproxy-after-reboot incident), a
# dead DNS, a read-only database. Returning normally would record the run as `ok` with the
# failure hidden inside the result string, i.e. a green pill for a total outage.
# The first error TRAVELS WITH the message on purpose: on an instance with one or two
# subscriptions "all channels failed" is also what ONE dead feed id looks like, and a red
# card the user can't act on is its own kind of dishonest. `channel: HTTP 404` says
# "fix this subscription"; `Connection refused` says "the egress is down".
raise RuntimeError(f"RSS poll failed for all {total} channels — first: {first_error}")
return {"new": new, "failed": failed}
def run_enrich(db: Session, max_batches: int = 200, floor: int = 200) -> int:
+27 -5
View File
@@ -73,6 +73,18 @@ def apply_channel_details(db: Session, items: list[dict]) -> None:
channel.details_synced_at = now
def should_prune(fetched_count: int, existing_count: int) -> bool:
"""Whether an import may delete the local subscriptions missing from this fetch.
NEVER on an empty fetch that contradicts local state: the API can answer 200 with no items (a
transient blip, a scope/token edge), and pruning against that wipes every subscription the user
has — taking each row's `deep_requested` flag with it — in a single run. Someone who genuinely
unsubscribed from everything simply keeps their rows until a sync that returns something.
Pure so it can be tested without a database; `import_subscriptions` is the only caller."""
return fetched_count > 0 or existing_count == 0
def import_subscriptions(db: Session, user: User) -> dict:
"""Pull all subscriptions for `user`, upsert channels + subscription links, prune
channels the user has unsubscribed from on YouTube, and fetch channel details for
@@ -108,17 +120,26 @@ def import_subscriptions(db: Session, user: User) -> dict:
sub_row.yt_subscription_id = sub.get("yt_subscription_id")
db.flush()
# Prune subscriptions removed on YouTube.
# Prune subscriptions removed on YouTube — but only when this fetch is trustworthy
# (see `should_prune`).
removed = 0
existing = (
db.execute(select(Subscription).where(Subscription.user_id == user.id))
.scalars()
.all()
)
for sub_row in existing:
if sub_row.channel_id not in fetched:
db.delete(sub_row)
removed += 1
prune_skipped = not should_prune(len(fetched), len(existing))
if prune_skipped:
log.warning(
"Subscription import (user %s): YouTube returned 0 subscriptions while %d exist "
"locally — skipping the prune (treating it as a transient empty response)",
user.id, len(existing),
)
else:
for sub_row in existing:
if sub_row.channel_id not in fetched:
db.delete(sub_row)
removed += 1
db.commit()
# Fetch details for channels that don't have them yet.
@@ -144,6 +165,7 @@ def import_subscriptions(db: Session, user: User) -> dict:
"channels_new": new_channels,
"channels_detailed": detailed,
"removed_stale": removed,
"prune_skipped": prune_skipped,
}
log.info("Subscription import (user %s): %s", user.id, result)
return result
-5
View File
@@ -14,7 +14,6 @@ from app import progress, sysconfig
from app.models import LIVE_OR_UPCOMING, Channel, Video
from app.titles import normalize_title
from app.youtube.client import YouTubeClient, best_thumbnail
from app.youtube.rss import fetch_channel_feed
from app.youtube.shorts import make_client, probe_is_short
_DURATION_RE = re.compile(
@@ -84,10 +83,6 @@ def apply_rss_feed(db: Session, channel: Channel, entries: list[dict]) -> int:
return inserted
def poll_rss_channel(db: Session, channel: Channel) -> int:
return apply_rss_feed(db, channel, fetch_channel_feed(channel.id))
# --- Backfill (uploads playlist; costs quota) ---
def _stub_from_playlist_item(item: dict, channel_id: str) -> dict | None:
content = item.get("contentDetails", {})
+155 -32
View File
@@ -16,6 +16,7 @@ Design:
* Crash-safe: on startup, orphaned `running` jobs are requeued and `downloading` assets reset.
"""
import logging
import re
import shutil
import signal
import threading
@@ -93,24 +94,33 @@ def _parse_upload_date(raw) -> date | None:
# --- claim + recovery -----------------------------------------------------------------------
def _wait_for_schema(timeout: float = 300.0) -> None:
"""Block until the download tables exist before doing any DB work.
def _wait_for_schema(timeout: float = 300.0) -> bool:
"""Block until the download tables exist before doing any DB work. Returns False if they never
appeared (or we were asked to stop while waiting).
The API applies migrations on its OWN startup, and the worker is a separate container that may
boot first (they start in parallel). Without this, the worker would hit a missing `download_jobs`
table and crash-loop until the API's migrations land. Poll patiently instead."""
table and crash-loop until the API's migrations land. Poll patiently instead — and if the schema
is still absent when the timeout runs out, say so and let the caller exit, because "continuing"
only moved the same crash one call further down (into `_recover_orphans`), where it read as an
unrelated failure."""
deadline = time.monotonic() + timeout
while not _stop.is_set():
try:
with SessionLocal() as db:
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
return
return True
except Exception as exc: # noqa: BLE001 — table not migrated yet / DB not up yet
if time.monotonic() > deadline:
log.warning("download schema still absent after %ss; continuing: %s", timeout, str(exc)[:120])
return
log.error(
"download schema still absent after %ss — the API's migrations have not landed; "
"exiting so the container restarts: %s",
timeout, str(exc)[:120],
)
return False
log.info("waiting for the database schema (API migrations)…")
_stop.wait(3.0)
return False
def _recover_orphans() -> None:
@@ -263,15 +273,14 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
def _make_progress_hook(job_id: int, asset_id: int):
state = {"last_db": 0.0, "last_check": 0.0, "meta_done": False}
state = {"meta_done": False}
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S)
def hook(d: dict) -> None:
now = time.monotonic()
# Cooperative pause/cancel: if the job is no longer 'running', abort the download.
if now - state["last_check"] >= 2.0:
state["last_check"] = now
if _job_status(job_id) not in ("running", None):
raise _Aborted
if cancelled():
raise _Aborted
if not state["meta_done"] and d.get("info_dict"):
state["meta_done"] = True
try:
@@ -280,9 +289,8 @@ def _make_progress_hook(job_id: int, asset_id: int):
pass
if d.get("status") != "downloading":
return
if now - state["last_db"] < 1.0:
if not allow_write():
return
state["last_db"] = now
# yt-dlp downloads the video and audio streams SEPARATELY (each 0→100%), then merges.
# Label the phase so the user understands the bar resetting between streams instead of
# seeing a mysterious 95%→5% jump.
@@ -309,6 +317,68 @@ def _make_progress_hook(job_id: int, asset_id: int):
return hook
# Both progress sources call back FAR faster than the database should be touched: yt-dlp fires its
# hook per chunk, and ffmpeg's `-progress pipe:1` emits a block of ~12 lines every ~0.5s while
# run_ffmpeg calls its callbacks once per LINE. Every callback here is a round-trip (`_job_status`
# opens a session and SELECTs; `_set_job` opens one and UPDATE+COMMITs), so unthrottled they cost
# ~10^5 queries on a long job. ONE interval, ONE implementation, used by both paths — three
# hand-rolled copies is how the ffmpeg side ended up unthrottled while the download side was fine.
_DB_WRITE_INTERVAL_S = 1.0
_CANCEL_POLL_INTERVAL_S = 1.0
def _rate_limiter(interval_s: float, *, now=time.monotonic):
"""`should_run()` → True at most once per `interval_s` (the first call always passes).
`now` is injectable so tests drive a clock instead of patching the global one."""
# `None`, not 0.0: "never ran" has to be a VALUE, not a falsy timestamp. A truthiness test here
# reads a legitimate `now() == 0.0` as "never ran" and lets a second call straight through —
# unreachable with real `time.monotonic()`, but `now` exists to be injected, and a test clock
# starting at 0 is the obvious choice. That would let a throttle test encode the wrong
# behaviour and still pass.
state: dict[str, float | None] = {"at": None}
def should_run() -> bool:
t = now()
last = state["at"]
if last is not None and t - last < interval_s:
return False
state["at"] = t
return True
return should_run
def _cancel_poll(job_id: int, interval_s: float = _CANCEL_POLL_INTERVAL_S, *, now=time.monotonic):
"""Cooperative "has this job been paused/cancelled?" poll, rate-limited and sticky: once it
says yes it never queries again. Shared by the yt-dlp hook and run_ffmpeg so cancel latency is
one policy, not one per call site."""
allow = _rate_limiter(interval_s, now=now)
state = {"cancelled": False}
def cancelled() -> bool:
if not state["cancelled"] and allow():
state["cancelled"] = _job_status(job_id) not in ("running", None)
return state["cancelled"]
return cancelled
def _ffmpeg_progress_writer(job_id: int, phase: str, *, now=time.monotonic):
"""`on_progress` for run_ffmpeg: persist at most one row write per interval, and only when the
whole-percent value actually changed (the bar shows integers)."""
allow = _rate_limiter(_DB_WRITE_INTERVAL_S, now=now)
state = {"pct": -1}
def write(pct: float) -> None:
whole = int(pct)
if whole == state["pct"] or not allow():
return
state["pct"] = whole
_set_job(job_id, progress=whole, phase=phase)
return write
def _pp_phase(pp: str) -> str:
"""Map a yt-dlp postprocessor class name to a user-facing phase label. ffmpeg post-steps
aren't byte-progress operations (no %), so the UI shows the step name + an indeterminate
@@ -341,13 +411,69 @@ def _staging_dir(asset_id: int) -> Path:
return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
_THUMB_SUFFIXES = (".jpg", ".jpeg", ".png", ".webp")
# yt-dlp's working files in the staging dir. None of them is ever the finished media, but several
# CAN be the biggest file there (a video-only per-format stream, an abandoned fragment run), so the
# staging fallback must exclude them by name — picking one would store e.g. a silent video-only
# stream as a ready asset.
_YTDLP_TEMP_RE = re.compile(
r"\.part(-Frag\d+)?$" # in-flight download / fragment
r"|\.ytdl$" # resume state
r"|\.temp\.[^.]+$" # merge/postprocess temp
r"|\.f\d+\.[^.]+$", # per-format stream kept before the merge
re.IGNORECASE,
)
def _find_thumbnail(staging: Path, media: Path) -> Path | None:
for p in staging.iterdir():
if p != media and p.suffix.lower() in (".jpg", ".jpeg", ".png", ".webp"):
if p != media and p.suffix.lower() in _THUMB_SUFFIXES:
return p
return None
def _produced_file(info: dict, staging: Path) -> Path:
"""The media file yt-dlp actually wrote.
`requested_downloads[0].filepath` is the authoritative answer on YouTube, but an arbitrary-URL
extractor may not fill it in at all — and `Path("")` is `Path(".")`, which *passes* `.exists()`,
so a missing value used to sail past the fallback and blow up two steps later (`with_name` on an
empty name: the BBC-news failure). Anything that isn't a real file counts as absent; then fall
back to the largest file left in staging that is neither a thumbnail nor one of yt-dlp's own
working files (see `_YTDLP_TEMP_RE`)."""
req = (info.get("requested_downloads") or [{}])[0]
fp = req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
if fp:
produced = Path(str(fp))
if produced.is_file():
return produced
cands = [
p
for p in staging.iterdir()
if p.is_file()
and p.suffix.lower() not in _THUMB_SUFFIXES
and not _YTDLP_TEMP_RE.search(p.name)
]
if not cands:
raise RuntimeError("yt-dlp produced no output file")
return max(cands, key=lambda p: p.stat().st_size)
def _media_info(info: dict) -> dict:
"""Unwrap a playlist/multi_video extraction to the entry that carries the download.
A single non-YouTube page (a news article's video, a multi-rendition embed) can extract as a
wrapper whose own dict has no `requested_downloads` — the media lives on an entry. Pick the
FIRST entry that actually downloaded something: on a page with a skipped teaser in front of the
real video, entries[0] carries no file, and taking it blindly would label the produced file
with the wrong id/title/duration."""
entries = [e for e in (info.get("entries") or []) if e]
if not entries or info.get("requested_downloads"):
return info
chosen = next((e for e in entries if e.get("requested_downloads")), entries[0])
return {**info, **chosen}
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
_RETRIABLE_MARKERS = (
"not a bot",
@@ -426,8 +552,8 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
editmod.run_ffmpeg(
editmod.build_browser_safe_cmd(produced, dest, reenc_v, reenc_a),
total_s=total_s,
on_progress=lambda pct: _set_job(job_id, progress=int(pct), phase="processing"),
should_cancel=lambda: _job_status(job_id) not in ("running", None),
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
should_cancel=_cancel_poll(job_id),
)
produced.unlink(missing_ok=True)
# Keep the stored asset metadata truthful about what's now on disk.
@@ -447,17 +573,9 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
_set_job(job_id, phase="downloading", progress=0)
url = service.source_url(source_kind, source_ref)
info = _extract_with_fallback(job_id, asset_id, url, spec, staging)
req = (info.get("requested_downloads") or [{}])[0]
produced = Path(req.get("filepath") or "")
if not produced.exists():
# Fallback: the single media file left in staging (ignore the thumbnail).
cands = [p for p in staging.iterdir() if p.suffix.lower() not in (".jpg", ".jpeg", ".png", ".webp")]
if not cands:
raise RuntimeError("yt-dlp produced no output file")
produced = cands[0]
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging))
produced = _produced_file(info, staging)
produced = _ensure_browser_playable(job_id, produced, spec, info)
ext = produced.suffix.lstrip(".").lower()
meta = storage.MediaMeta(
@@ -613,8 +731,8 @@ def _process_edit(job_id: int, asset_id: int) -> None:
editmod.run_ffmpeg(
cmd,
total_s,
on_progress=lambda pct: _set_job(job_id, progress=int(pct), phase="editing"),
should_cancel=lambda: _job_status(job_id) not in ("running", None),
on_progress=_ffmpeg_progress_writer(job_id, "editing"),
should_cancel=_cancel_poll(job_id),
)
if not dest_staging.exists():
raise RuntimeError("ffmpeg produced no output file")
@@ -689,9 +807,14 @@ def main() -> None:
log.info("WORKER_ENABLED is off; worker exiting (nothing to do).")
return
_wait_for_schema() # the API may still be applying migrations when we boot
if _stop.is_set():
return
# The API may still be applying migrations when we boot. If they never arrive, exit instead of
# pressing on into a guaranteed crash — with a FAILURE status, so the container comes back under
# any restart policy (`on-failure` too, not just the `unless-stopped` our composes ship) and the
# exit code doesn't claim success for a worker that never started working.
if not _wait_for_schema():
if _stop.is_set():
return # asked to shut down while waiting — that IS a clean exit
raise SystemExit(1)
_recover_orphans()
n = max(1, settings.download_worker_concurrency)
log.info("Download worker started (concurrency=%d, root=%s).", n, settings.download_root)
+13 -3
View File
@@ -8,14 +8,24 @@ import httpx
RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
class RssError(Exception):
"""The feed could not be read (transport failure or a non-200 answer).
Deliberately NOT the same as an empty feed: a failed poll used to return `[]`, which the
caller stamped as a successful `last_rss_at` — so a channel whose feed 404s permanently
(deleted, renamed id) looked like it was being polled fine forever, and nothing surfaced it."""
def fetch_channel_feed(channel_id: str) -> list[dict]:
"""The channel's recent uploads as stubs. `[]` means the feed really is empty; a failure
raises RssError so the caller can leave the channel un-stamped and log it."""
url = RSS_URL.format(channel_id=channel_id)
try:
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"})
except httpx.HTTPError:
return []
except httpx.HTTPError as exc:
raise RssError(f"RSS fetch failed for {channel_id}: {exc}") from exc
if resp.status_code != 200:
return []
raise RssError(f"RSS fetch for {channel_id} returned HTTP {resp.status_code}")
parsed = feedparser.parse(resp.content)
out: list[dict] = []
+139
View File
@@ -0,0 +1,139 @@
"""Picking the file yt-dlp actually produced, for sources that aren't YouTube.
The regression these cover: `Path(info["filepath"] or "")` is `Path(".")`, which PASSES `.exists()`,
so a missing filepath (an arbitrary-URL extractor) skipped the staging fallback and crashed later
with "PosixPath('.') has an empty name" instead of downloading."""
import pytest
from app.downloads.edit import _concat_quote, parse_probe_streams
from app.worker import _media_info, _produced_file
from pathlib import Path, PurePosixPath
def _staging(tmp_path: Path, *names: str) -> Path:
for n in names:
p = tmp_path / n
p.write_bytes(b"x" * (10 * (names.index(n) + 1)))
return tmp_path
class TestProducedFile:
def test_uses_the_reported_filepath(self, tmp_path):
staging = _staging(tmp_path, "v.mp4", "v.jpg")
info = {"requested_downloads": [{"filepath": str(tmp_path / "v.mp4")}]}
assert _produced_file(info, staging) == tmp_path / "v.mp4"
@pytest.mark.parametrize("info", [{}, {"requested_downloads": [{}]}, {"requested_downloads": [{"filepath": ""}]}])
def test_missing_filepath_falls_back_to_staging(self, tmp_path, info):
staging = _staging(tmp_path, "thumb.jpg", "clip.mp4")
assert _produced_file(info, staging) == tmp_path / "clip.mp4"
def test_stale_filepath_falls_back_to_staging(self, tmp_path):
staging = _staging(tmp_path, "clip.mp4")
info = {"requested_downloads": [{"filepath": str(tmp_path / "gone.mp4")}]}
assert _produced_file(info, staging) == tmp_path / "clip.mp4"
def test_picks_the_largest_media_file_ignoring_thumbs_and_partials(self, tmp_path):
staging = _staging(tmp_path, "small.mp4", "big.mp4", "cover.webp")
(tmp_path / "huge.mp4.part").write_bytes(b"x" * 999)
assert _produced_file({}, staging) == tmp_path / "big.mp4"
@pytest.mark.parametrize(
"leftover",
["vid.f137.mp4", "vid.mp4.part-Frag12", "vid.ytdl", "vid.temp.mp4", "vid.mp4.part"],
)
def test_ytdlp_working_files_never_win(self, tmp_path, leftover):
# Each of these can legitimately be the BIGGEST file in staging (a video-only per-format
# stream especially) — storing one would serve a silent or truncated "download".
(tmp_path / leftover).write_bytes(b"x" * 5000)
(tmp_path / "vid.mp4").write_bytes(b"x" * 10)
assert _produced_file({}, tmp_path) == tmp_path / "vid.mp4"
def test_only_leftovers_is_treated_as_no_output(self, tmp_path):
(tmp_path / "vid.f251.webm").write_bytes(b"x" * 5000)
with pytest.raises(RuntimeError):
_produced_file({}, tmp_path)
def test_no_output_at_all_raises(self, tmp_path):
staging = _staging(tmp_path, "only.jpg")
with pytest.raises(RuntimeError):
_produced_file({}, staging)
class TestMediaInfo:
def test_playlist_wrapper_unwraps_to_the_entry_that_downloaded(self):
info = {"id": "page", "title": "Article", "entries": [{"id": "vid", "requested_downloads": [{"filepath": "/x.mp4"}]}]}
assert _media_info(info)["id"] == "vid"
def test_plain_result_is_untouched(self):
info = {"id": "vid", "requested_downloads": [{"filepath": "/x.mp4"}]}
assert _media_info(info) is info
def test_picks_the_entry_that_actually_downloaded(self):
# A page whose first entry was skipped: taking entries[0] would label the produced file
# with the teaser's id/title.
info = {
"id": "page",
"entries": [
{"id": "teaser", "title": "Teaser"},
{"id": "main", "title": "Main", "requested_downloads": [{"filepath": "/x.mp4"}]},
],
}
assert _media_info(info)["id"] == "main"
def test_falls_back_to_the_first_entry_when_none_downloaded(self):
info = {"id": "page", "entries": [{"id": "a"}, {"id": "b"}]}
assert _media_info(info)["id"] == "a"
def test_wrapper_that_downloaded_itself_wins_over_its_entries(self):
info = {"id": "vid", "requested_downloads": [{"filepath": "/x.mp4"}], "entries": [{"id": "other"}]}
assert _media_info(info)["id"] == "vid"
def test_empty_entries_are_ignored(self):
info = {"id": "vid", "entries": [None]}
assert _media_info(info)["id"] == "vid"
class TestParseProbeStreams:
# ffprobe emits codec_name BEFORE codec_type, so the old line parser shifted every name onto
# the previous stream and an H.264+AAC mp4 came back as (aac, None) → a needless re-encode.
def test_reads_each_stream_by_its_own_type(self):
payload = '{"streams":[{"codec_name":"h264","codec_type":"video"},{"codec_name":"aac","codec_type":"audio"}]}'
assert parse_probe_streams(payload) == ("h264", "aac")
def test_audio_first_ordering_is_handled(self):
payload = '{"streams":[{"codec_name":"AAC","codec_type":"audio"},{"codec_name":"H264","codec_type":"video"}]}'
assert parse_probe_streams(payload) == ("h264", "aac")
def test_extra_streams_do_not_win_over_the_first_of_their_kind(self):
payload = (
'{"streams":[{"codec_name":"h264","codec_type":"video"},'
'{"codec_name":"vp9","codec_type":"video"},'
'{"codec_name":"mov_text","codec_type":"subtitle"},'
'{"codec_name":"opus","codec_type":"audio"}]}'
)
assert parse_probe_streams(payload) == ("h264", "opus")
def test_audio_only_and_video_only(self):
assert parse_probe_streams('{"streams":[{"codec_name":"opus","codec_type":"audio"}]}') == (None, "opus")
assert parse_probe_streams('{"streams":[{"codec_name":"av1","codec_type":"video"}]}') == ("av1", None)
def test_garbage_is_not_a_crash(self):
assert parse_probe_streams("") == (None, None)
assert parse_probe_streams("not json") == (None, None)
class TestConcatQuote:
# PurePosixPath so the escaping is asserted identically on a Windows dev box and in the
# Linux test container (a WindowsPath would eat the backslash as a separator).
def test_apostrophe_is_closed_escaped_and_reopened(self):
# A pre-0.51.0 file can still be named with an apostrophe; unescaped it breaks the list.
assert _concat_quote(PurePosixPath("/m/Don't_Look.mp4")) == "/m/Don'\\''t_Look.mp4"
def test_backslash_stays_literal(self):
# Inside single quotes ffmpeg treats `\` literally — doubling it would point at a
# different (non-existent) path.
assert _concat_quote(PurePosixPath("/m/a\\b.mp4")) == "/m/a\\b.mp4"
def test_ordinary_path_is_unchanged(self):
assert _concat_quote(PurePosixPath("/m/clip.mp4")) == "/m/clip.mp4"
+79
View File
@@ -0,0 +1,79 @@
"""The rate limit in front of run_ffmpeg's callbacks.
run_ffmpeg calls `on_progress`/`should_cancel` once per ffmpeg output LINE (`-progress pipe:1`
emits a block of ~12 every half second), and both callbacks are database round-trips: `_set_job`
opens a session and UPDATE+COMMITs, `_job_status` opens one and SELECTs. Unthrottled, a long
re-encode is ~10^5 of each."""
from app import worker
class TestProgressWriter:
def test_writes_the_first_value_immediately(self, monkeypatch):
writes = []
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
worker._ffmpeg_progress_writer(7, "processing")(12.4)
assert writes == [{"progress": 12, "phase": "processing"}]
def test_a_burst_of_lines_costs_one_write(self, monkeypatch):
writes = []
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
write = worker._ffmpeg_progress_writer(7, "processing")
for pct in range(0, 40): # what one second of ffmpeg output looks like
write(pct)
assert len(writes) == 1
def test_the_next_second_writes_again(self, monkeypatch):
writes = []
clock = {"t": 1000.0}
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
write = worker._ffmpeg_progress_writer(7, "processing", now=lambda: clock["t"])
write(10)
write(11) # same second → dropped
clock["t"] += 1.5
write(12)
assert [w["progress"] for w in writes] == [10, 12]
def test_an_unchanged_whole_percent_is_never_rewritten(self, monkeypatch):
writes = []
clock = {"t": 1000.0}
monkeypatch.setattr(worker, "_set_job", lambda job_id, **f: writes.append(f))
write = worker._ffmpeg_progress_writer(7, "processing", now=lambda: clock["t"])
write(10.1)
clock["t"] += 5
write(10.9) # still 10% — the bar shows integers
assert [w["progress"] for w in writes] == [10]
class TestCancelPoll:
def test_asks_once_then_reuses_the_answer(self, monkeypatch):
calls = []
monkeypatch.setattr(worker, "_job_status", lambda job_id: calls.append(job_id) or "running")
cancelled = worker._cancel_poll(7)
for _ in range(40):
assert cancelled() is False
assert len(calls) == 1
def test_a_cancelled_job_is_reported_and_never_queried_again(self, monkeypatch):
calls = []
clock = {"t": 1000.0}
def status(job_id):
calls.append(job_id)
return "cancelled"
monkeypatch.setattr(worker, "_job_status", status)
cancelled = worker._cancel_poll(7, now=lambda: clock["t"])
assert cancelled() is True
clock["t"] += 60 # cancellation is sticky: no re-query, and no un-cancelling
assert cancelled() is True
assert len(calls) == 1
def test_a_cancel_arriving_later_is_seen_on_the_next_tick(self, monkeypatch):
clock = {"t": 1000.0}
state = {"status": "running"}
monkeypatch.setattr(worker, "_job_status", lambda job_id: state["status"])
cancelled = worker._cancel_poll(7, now=lambda: clock["t"])
assert cancelled() is False
state["status"] = "paused"
clock["t"] += 1.0 # the throttle must not delay a cancel by more than its interval
assert cancelled() is True
+155
View File
@@ -0,0 +1,155 @@
"""The guards in front of a recursive delete.
`sweep_orphans` rmtree's directories under `_HLS_ROOT`, and that root is admin-configurable
(PLEX_HLS_DIR) — it can legitimately be a shared scratch path. `_SESSION_DIR_RE` is the ONLY thing
that keeps the sweep off a neighbour's data, so the names it accepts are pinned here.
Since the sweep stopped being awaited at startup it also runs CONCURRENTLY with playback, so the
second half of this file pins what keeps it off a LIVE session's segments."""
import os
import pytest
from app.plex import stream
from app.plex.stream import _SESSION_DIR_RE
@pytest.mark.parametrize(
"name",
[
"12345_0", # legacy shape (pre multi-audio): {rating_key}_{start}
"12345_842",
"12345_0_None_+0.00", # current shape, single audio track (audio_ord None)
"12345_0_2_+0.00", # current shape, explicit audio_ord
"12345_842_multi_-1.25", # multi-audio, negative offset
"12345_842_multi_+12.50",
],
)
def test_our_session_directories_are_swept(name):
assert _SESSION_DIR_RE.match(name)
@pytest.mark.parametrize(
"name",
[
"backup_2024", # the class an "optional tail" pattern wrongly matched
"release_10",
"important_data_2024",
"node_modules",
"logs",
"plex-hls",
"12345", # no start offset — not a session dir
"12345_0_multi", # truncated: missing the offset
"12345_0_multi_1.25", # offset without its sign
"snapshot_2024_backup_+1.00", # session-ish tail, but the key isn't a rating_key
"backup_2024_multi_+0.00",
".config",
],
)
def test_foreign_directories_are_left_alone(name):
assert not _SESSION_DIR_RE.match(name)
class _FakeSession:
def __init__(self, directory):
self.dir = directory
@pytest.fixture
def hls_root(tmp_path, monkeypatch):
"""Point the sweep at a temp root with no sessions registered."""
monkeypatch.setattr(stream, "_HLS_ROOT", tmp_path)
monkeypatch.setattr(stream, "_sessions", {})
return tmp_path
def _session_dir(root, name, *, age_s=0.0):
"""A session directory, optionally backdated by `age_s` to look like a previous run's."""
d = root / name
d.mkdir()
(d / "seg_0.ts").write_bytes(b"x")
if age_s:
t = d.stat().st_mtime - age_s
os.utime(d, (t, t))
return d
def _cutoff(root):
"""A "process start" read from the FILESYSTEM, not from a clock.
The guard compares a directory's mtime against a timestamp, and the two do not have to come
from the same clock: on a bind-mounted dev volume the file times are stamped by the host while
`time.time()` runs in the container VM, and the drift between them is real (it made an earlier
version of these tests flaky). Anchoring the cutoff to a file this test just created removes
the clock from the assertion; that the DEFAULT cutoff is `_PROCESS_START` is pinned separately
below."""
probe = root / ".cutoff-probe"
probe.write_bytes(b"")
at = probe.stat().st_mtime
probe.unlink()
return at
class TestSweepOrphans:
def test_a_previous_runs_leftovers_are_deleted(self, hls_root):
d = _session_dir(hls_root, "12345_0_None_+0.00", age_s=3600)
assert stream.sweep_orphans(older_than=_cutoff(hls_root)) == 1
assert not d.exists()
def test_foreign_directories_survive(self, hls_root):
keep = hls_root / "backup_2024"
keep.mkdir()
(keep / "important.tar").write_bytes(b"x")
assert stream.sweep_orphans(older_than=_cutoff(hls_root)) == 0
assert (keep / "important.tar").exists()
def test_a_live_sessions_directory_is_never_deleted(self, hls_root):
# Registered session, but with an OLD directory — only the live check can save it.
d = _session_dir(hls_root, "12345_0_multi_+0.00", age_s=3600)
stream._sessions["k"] = _FakeSession(d)
assert stream.sweep_orphans(older_than=_cutoff(hls_root)) == 0
assert d.exists()
def test_a_directory_this_run_created_is_never_deleted(self, hls_root):
# THE RACE: the sweep is no longer awaited before the app serves, and `start_session`
# REUSES the path of a same-key session — so between our listing and our rmtree, a stale
# name can become a live directory. A fresh mtime is what tells them apart, and it must
# hold even while `_sessions` is still empty (the session registers a beat later).
cutoff = _cutoff(hls_root)
fresh = _session_dir(hls_root, "12345_0_None_+0.00") # created "after boot"
old = _session_dir(hls_root, "999_0_None_+0.00", age_s=3600)
assert stream.sweep_orphans(older_than=cutoff) == 1 # only the leftover
assert fresh.exists()
assert not old.exists()
def test_the_default_cutoff_is_this_processs_start(self, hls_root, monkeypatch):
# The parameter exists for the tests; production calls it with no argument, so pin what
# that means — otherwise the guard above could be right and unused.
d = _session_dir(hls_root, "12345_0_None_+0.00")
monkeypatch.setattr(stream, "_MTIME_GRACE_S", 0.0)
monkeypatch.setattr(stream, "_PROCESS_START", d.stat().st_mtime + 1)
assert stream.sweep_orphans() == 1 # "older than boot" → swept
assert not d.exists()
def test_a_directory_stamped_just_before_boot_survives(self, hls_root, monkeypatch):
# MEASURED in the container, not theory: a file's mtime comes from the kernel's COARSE
# clock and `time.time()` from the fine one, so a directory created 2ms AFTER
# `_PROCESS_START` was stamped 2ms BEFORE it. Without the grace the sweep deletes a session
# that started in the first instants after boot — which is exactly when the sweep runs.
d = _session_dir(hls_root, "12345_0_None_+0.00")
monkeypatch.setattr(stream, "_PROCESS_START", d.stat().st_mtime + 0.05) # the skew
assert stream.sweep_orphans() == 0
assert d.exists()
def test_a_missing_root_is_not_an_error(self, tmp_path, monkeypatch):
monkeypatch.setattr(stream, "_HLS_ROOT", tmp_path / "nope")
monkeypatch.setattr(stream, "_sessions", {})
assert stream.sweep_orphans() == 0
def test_a_stray_file_with_a_session_name_is_left_alone(self, hls_root):
# Only directories are ours; a same-named FILE belongs to someone else.
f = hls_root / "12345_0"
f.write_bytes(b"x")
os.utime(f, (f.stat().st_mtime - 3600,) * 2) # old enough that only is_dir() saves it
assert stream.sweep_orphans(older_than=_cutoff(hls_root)) == 0
assert f.exists()
+182
View File
@@ -0,0 +1,182 @@
"""run_ffmpeg's process plumbing — the part that can hang a worker thread forever.
No ffmpeg here: a fake Popen stands in, so the reader threads, the 1s poll loop, the stall watchdog
and the bounded exit wait are all exercised on any machine. These are regression tests for a real
deadlock: stderr was opened and never read, so a chatty ffmpeg filled the ~64KB pipe, blocked on its
next write and never exited, and `proc.wait()` never returned."""
import subprocess
import time
import pytest
from app.downloads import edit
class _FakeStream:
"""A pipe. `block=True` = the process is alive but silent (a wedged ffmpeg): iteration parks
until the stream is closed, which is what terminate()/kill() does to a real pipe."""
def __init__(self, lines=(), block=False):
self._lines = list(lines)
self._block = block
self.closed = False
self.consumed = 0
def __iter__(self):
return self
def __next__(self):
if self._lines:
self.consumed += 1
return self._lines.pop(0)
while self._block and not self.closed:
time.sleep(0.02)
raise StopIteration
def close(self):
self.closed = True
class _FakeProc:
pid = 4242
def __init__(self, stdout, stderr, returncode=0, wait_timeouts=0):
self.stdout = stdout
self.stderr = stderr
self._rc = returncode
# How many of the first `wait()` calls raise TimeoutExpired — a process ignoring SIGTERM,
# or one that lingers after closing its pipes.
self._wait_timeouts = wait_timeouts
self.terminated = False
self.killed = False
self.waits = 0
def poll(self):
# Popen.poll(): None while the process is alive. Our fake stays alive until it is stopped —
# `_stop` uses this to skip a process that has already gone.
return self._rc if self.terminated else None
def terminate(self):
self.terminated = True
# A terminated process's pipes hit EOF — that is how the reader threads finish.
self.stdout.close()
self.stderr.close()
def kill(self):
self.killed = True
self.terminate()
def wait(self, timeout=None):
self.waits += 1
if self._wait_timeouts > 0:
self._wait_timeouts -= 1
raise subprocess.TimeoutExpired("ffmpeg", timeout or 0)
return self._rc
@pytest.fixture
def fake_popen(monkeypatch):
"""Install a fake Popen and hand the test the process it will produce."""
def install(proc):
monkeypatch.setattr(edit.subprocess, "Popen", lambda *a, **k: proc)
return proc
return install
def test_progress_is_reported_and_a_clean_exit_returns(fake_popen):
proc = fake_popen(
_FakeProc(_FakeStream(["out_time_us=5000000\n", "progress=continue\n"]), _FakeStream())
)
seen = []
edit.run_ffmpeg(["ffmpeg"], total_s=10, on_progress=seen.append, should_cancel=lambda: False)
assert seen == [50.0] # 5s of 10s
assert not proc.terminated
def test_progress_is_capped_below_100(fake_popen):
fake_popen(_FakeProc(_FakeStream(["out_time_us=99000000\n"]), _FakeStream()))
seen = []
edit.run_ffmpeg(["ffmpeg"], total_s=10, on_progress=seen.append, should_cancel=lambda: False)
assert seen == [99.0] # never reports done before the process actually exits
def test_a_nonzero_exit_raises_with_the_stderr_tail(fake_popen):
# The tail must come from the drained stderr — reading the pipe only AFTER the exit was the
# original deadlock.
proc = fake_popen(
_FakeProc(_FakeStream([]), _FakeStream(["Invalid data found\n"]), returncode=2)
)
with pytest.raises(RuntimeError, match="Invalid data found"):
edit.run_ffmpeg(["ffmpeg"], total_s=10, on_progress=lambda _p: None, should_cancel=lambda: False)
assert not proc.terminated # it exited on its own; nothing to kill
def test_cancel_is_honoured_while_the_process_is_silent(fake_popen):
# The pre-fix loop only checked cancel when a progress line arrived, so a quiet ffmpeg was
# uncancellable. The 1s poll tick is what makes this terminate.
proc = fake_popen(_FakeProc(_FakeStream(block=True), _FakeStream(block=True)))
started = time.monotonic()
with pytest.raises(edit.EditAborted):
edit.run_ffmpeg(["ffmpeg"], total_s=10, on_progress=lambda _p: None, should_cancel=lambda: True)
assert proc.terminated
assert time.monotonic() - started < 5 # not "whenever ffmpeg next speaks"
def test_a_wedged_process_is_killed_by_the_stall_watchdog(fake_popen):
proc = fake_popen(_FakeProc(_FakeStream(block=True), _FakeStream(block=True)))
with pytest.raises(RuntimeError, match="stalled"):
edit.run_ffmpeg(
["ffmpeg"],
total_s=10,
on_progress=lambda _p: None,
should_cancel=lambda: False,
stall_timeout_s=0.5,
)
assert proc.terminated # the worker thread is freed, not parked forever
def test_a_callback_exception_still_stops_the_process(fake_popen):
# `on_progress` is a database write in production. A failure THERE used to unwind past a live
# ffmpeg — orphaning it, still encoding, after a 10s join on its pipes.
proc = fake_popen(_FakeProc(_FakeStream(["out_time_us=1000000\n"]), _FakeStream(block=True)))
def boom(_pct):
raise RuntimeError("database is down")
with pytest.raises(RuntimeError, match="database is down"):
edit.run_ffmpeg(["ffmpeg"], total_s=10, on_progress=boom, should_cancel=lambda: False)
assert proc.terminated
def test_a_process_that_ignores_sigterm_is_killed(fake_popen):
# `_stop`'s escalation: terminate, wait 5s, then SIGKILL. Without it a cancel could park the
# worker thread on a process that simply refuses to go.
proc = fake_popen(
_FakeProc(_FakeStream(block=True), _FakeStream(block=True), wait_timeouts=1)
)
with pytest.raises(edit.EditAborted):
edit.run_ffmpeg(["ffmpeg"], total_s=10, on_progress=lambda _p: None, should_cancel=lambda: True)
assert proc.terminated and proc.killed
def test_a_process_lingering_after_eof_is_stopped_and_reported(fake_popen):
# Pipes closed but the process still around: the bounded wait must give up rather than block
# the worker thread on `proc.wait()` forever (the old code waited with no timeout at all).
proc = fake_popen(_FakeProc(_FakeStream([]), _FakeStream(), wait_timeouts=1))
with pytest.raises(RuntimeError, match="did not exit"):
edit.run_ffmpeg(["ffmpeg"], total_s=10, on_progress=lambda _p: None, should_cancel=lambda: False)
assert proc.terminated
# Two waits: the bounded grace wait that timed out, then _stop's own — never an unbounded one.
assert proc.waits == 2
def test_stderr_is_drained_while_the_process_runs(fake_popen):
# The deadlock case: a chatty ffmpeg that exits 0. EVERY stderr line must be consumed — with a
# real pipe, an unread one fills at ~64KB and blocks ffmpeg's next write forever.
err = _FakeStream([f"warning {i}\n" for i in range(500)])
fake_popen(_FakeProc(_FakeStream(["out_time_us=1000000\n"]), err))
edit.run_ffmpeg(["ffmpeg"], total_s=10, on_progress=lambda _p: None, should_cancel=lambda: False)
assert err.consumed == 500
assert err.closed
+225
View File
@@ -0,0 +1,225 @@
"""Guards that stop one bad YouTube answer from cascading: the subscription prune guard, the
playlist-push circuit breaker, and RSS telling a failure apart from an empty feed."""
import httpx
import pytest
from app.sync import runner
from app.sync.playlists import FAILED, _Breaker
from app.sync.subscriptions import should_prune
from app.youtube import rss
from app.youtube.client import YouTubeError
class TestShouldPrune:
def test_a_normal_fetch_prunes(self):
assert should_prune(fetched_count=12, existing_count=15) is True
def test_an_empty_fetch_against_existing_rows_never_prunes(self):
# The data-loss case: one anomalous empty-but-200 response must not wipe 15 rows.
assert should_prune(fetched_count=0, existing_count=15) is False
def test_an_empty_fetch_with_nothing_local_is_fine(self):
# A brand-new account: nothing to lose, so the (no-op) prune may run.
assert should_prune(fetched_count=0, existing_count=0) is True
def test_a_single_remaining_subscription_still_prunes(self):
assert should_prune(fetched_count=1, existing_count=200) is True
class TestBreaker:
def test_open_until_the_limit_is_reached(self):
b = _Breaker(limit=3)
for _ in range(2):
b.hit(False)
assert b.open()
b.hit(False)
assert not b.open() and b.tripped
def test_a_success_re_arms_the_streak(self):
b = _Breaker(limit=3)
b.hit(False)
b.hit(False)
b.hit(True) # one write went through — this isn't an account-wide failure
b.hit(False)
b.hit(False)
assert b.open()
def test_stays_tripped_once_tripped(self):
b = _Breaker(limit=2)
b.hit(False)
b.hit(False)
b.hit(True)
assert not b.open()
def _quota_dead():
"""A write op that fails the way a quota-exhausted / scope-revoked account fails."""
raise YouTubeError("quota exceeded")
class TestBreakerRun:
def test_returns_the_ops_result_and_leaves_failures_alone(self):
b, failed = _Breaker(), []
assert b.run(lambda: "item-1", lambda: failed.append("v1")) == "item-1"
assert failed == []
def test_a_youtube_error_records_the_failure_and_counts_towards_the_trip(self):
b, failed = _Breaker(limit=2), []
assert b.run(_quota_dead, lambda: failed.append("v1")) is FAILED
assert failed == ["v1"] and b.open()
b.run(_quota_dead, lambda: failed.append("v2"))
assert not b.open()
def test_a_tripped_breaker_never_calls_the_op(self):
b, failed, calls = _Breaker(limit=1), [], []
b.run(_quota_dead, lambda: failed.append("v1"))
assert b.run(lambda: calls.append("called"), lambda: failed.append("v2")) is FAILED
assert calls == [] # the whole point: no more quota spent
assert failed == ["v1", "v2"] # …but the item is still reported
def test_other_exceptions_are_not_swallowed(self):
# Only YouTubeError is a "write failed" signal; a bug must still surface.
def boom():
raise ValueError("bug")
with pytest.raises(ValueError):
_Breaker().run(boom, lambda: None)
class _StubDb:
"""Just enough Session for run_rss_poll when the caller supplies the channel list."""
def __init__(self):
self.rollbacks = 0
def rollback(self):
self.rollbacks += 1
class _StubChannel:
def __init__(self, cid):
self.id = cid
class TestRssPollResult:
"""`failed` must count every channel that did not get polled — feed-side AND apply-side.
Counting only the fetch failures would move the "a broken run looks like a quiet hour" blind
spot rather than remove it."""
def _run(self, monkeypatch, *, fetch, apply_, channels=("UC1", "UC2")):
monkeypatch.setattr(runner, "fetch_channel_feed", fetch)
monkeypatch.setattr(runner, "apply_rss_feed", apply_)
return runner.run_rss_poll(_StubDb(), [_StubChannel(c) for c in channels])
def test_a_healthy_run_reports_no_failures(self, monkeypatch):
out = self._run(monkeypatch, fetch=lambda cid: [{"id": "v"}], apply_=lambda db, ch, e: 1)
assert out == {"new": 2, "failed": 0}
def test_unreadable_feeds_are_counted(self, monkeypatch):
# One dead channel among healthy ones: counted, but the run itself is still a run.
def fetch(cid):
if cid == "UC_dead":
raise rss.RssError("404")
return [{"id": "v"}]
out = self._run(
monkeypatch, fetch=fetch, apply_=lambda db, ch, e: 1, channels=("UC1", "UC_dead")
)
assert out == {"new": 1, "failed": 1}
def test_apply_failures_are_counted_too(self, monkeypatch):
# A per-row write failure (a constraint clash on one channel) while the rest store fine.
def apply_(db, ch, entries):
if ch.id == "UC_bad":
raise RuntimeError("constraint")
return 1
out = self._run(
monkeypatch,
fetch=lambda cid: [{"id": "v"}],
apply_=apply_,
channels=("UC1", "UC_bad"),
)
assert out == {"new": 1, "failed": 1}, "an unstored channel must not read as ok"
def test_a_quiet_hour_is_distinguishable_from_an_outage(self, monkeypatch):
quiet = self._run(monkeypatch, fetch=lambda cid: [], apply_=lambda db, ch, e: 0)
assert quiet == {"new": 0, "failed": 0} # same "new", different "failed" — that's the point
def test_a_total_outage_raises_instead_of_returning_ok(self, monkeypatch):
# Every feed unreachable (the tinyproxy-lost-the-boot-race incident). Returning a result
# here recorded the scheduler run as "ok", with the outage only visible inside the summary
# text — a green pill for a run that did nothing at all.
def boom(cid):
raise rss.RssError("Connection refused")
with pytest.raises(RuntimeError, match="all 2 channels"):
self._run(monkeypatch, fetch=boom, apply_=lambda db, ch, e: 1)
def test_the_outage_message_carries_the_underlying_error(self, monkeypatch):
# "all N channels failed" is ALSO what one dead feed id looks like on a small instance.
# The card has to say which: `HTTP 404` = fix the subscription, `Connection refused` =
# the egress is down.
def dead(cid):
raise rss.RssError(f"RSS fetch for {cid} returned HTTP 404")
with pytest.raises(RuntimeError, match="HTTP 404"):
self._run(monkeypatch, fetch=dead, apply_=lambda db, ch, e: 1, channels=("UC_dead",))
def test_a_read_only_database_is_a_total_outage_too(self, monkeypatch):
# Every feed reads fine, nothing can be stored: just as un-polled, just as red.
def boom(db, ch, entries):
raise RuntimeError("read-only transaction")
with pytest.raises(RuntimeError, match="read-only transaction"):
self._run(monkeypatch, fetch=lambda cid: [{"id": "v"}], apply_=boom)
def test_no_channels_at_all_is_not_an_outage(self, monkeypatch):
# A fresh instance with nothing subscribed must not report an error every poll.
out = self._run(
monkeypatch, fetch=lambda cid: [], apply_=lambda db, ch, e: 0, channels=()
)
assert out == {"new": 0, "failed": 0}
class TestRssErrorVsEmpty:
"""A failed poll must RAISE (so the caller leaves `last_rss_at` alone); only a genuinely
empty feed may return []."""
def _patch(self, monkeypatch, **kwargs):
def fake_get(url, **_):
if "exc" in kwargs:
raise kwargs["exc"]
return httpx.Response(
kwargs["status"],
content=kwargs.get("body", b""),
request=httpx.Request("GET", url),
)
monkeypatch.setattr(rss.httpx, "get", fake_get)
def test_non_200_raises(self, monkeypatch):
self._patch(monkeypatch, status=404)
with pytest.raises(rss.RssError):
rss.fetch_channel_feed("UC_dead")
def test_transport_error_raises(self, monkeypatch):
self._patch(monkeypatch, exc=httpx.ConnectError("no route"))
with pytest.raises(rss.RssError):
rss.fetch_channel_feed("UC_offline")
def test_an_empty_but_valid_feed_returns_empty(self, monkeypatch):
feed = b"""<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>"""
self._patch(monkeypatch, status=200, body=feed)
assert rss.fetch_channel_feed("UC_quiet") == []
def test_entries_are_parsed(self, monkeypatch):
feed = b"""<?xml version="1.0"?>
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:yt="http://www.youtube.com/xml/schemas/2015">
<entry><yt:videoId>abc123</yt:videoId><title>Hello</title>
<published>2026-07-23T10:00:00+00:00</published></entry>
</feed>"""
self._patch(monkeypatch, status=200, body=feed)
out = rss.fetch_channel_feed("UC_live")
assert [e["id"] for e in out] == ["abc123"]
assert out[0]["channel_id"] == "UC_live"
+7 -2
View File
@@ -137,6 +137,11 @@ export default function Feed({
(video: Video, startAt: number | null = null) => setActiveVideo({ video, startAt }),
[],
);
// Stable identity, like Playlists' `closePlayer`: PlayerModal's keydown/scroll-lock effect
// depends on `onClose`, so a fresh closure every Feed render re-subscribed those listeners,
// re-ran focusModal() under the open player, dropped its in-flight timers and flushed the
// debounced volume write on each pass.
const closePlayer = useCallback(() => setActiveVideo(null), []);
const ytActive = !!ytSearch;
// How many live-search results to gather (the count selector replaces a manual "load more";
@@ -484,7 +489,7 @@ export default function Feed({
video={activeVideo.video}
startAt={activeVideo.startAt}
queue={ytPlayerQueue}
onClose={() => setActiveVideo(null)}
onClose={closePlayer}
onState={onState}
/>
</Suspense>
@@ -739,7 +744,7 @@ export default function Feed({
video={activeVideo.video}
startAt={activeVideo.startAt}
queue={playerQueue}
onClose={() => setActiveVideo(null)}
onClose={closePlayer}
onState={onState}
/>
</Suspense>
+349 -26
View File
@@ -11,6 +11,9 @@ import {
ChevronLeft,
ChevronRight,
ExternalLink,
Maximize,
Minimize,
Minimize2,
Repeat,
Repeat1,
Settings,
@@ -34,7 +37,15 @@ import {
relativeTime,
} from "../lib/format";
import { renderDescription } from "../lib/descriptionLinks";
import { useBackToClose } from "../lib/history";
import {
exitFullscreen,
fullscreenElement,
onFullscreenChange,
requestFullscreen,
} from "../lib/fullscreen";
import { useBackToClose, useBackToExit } from "../lib/history";
import { DEFAULT_VOLUME, normalizeVolume, stepVolume } from "../lib/playerVolume";
import { LS, readAccountMerged, writeAccount } from "../lib/storage";
import { useScrollFade } from "../lib/useScrollFade";
// Experiment (branch experiment/inline-player): play the video in-app via the
@@ -54,6 +65,21 @@ type LoopMode = "off" | "one" | "all";
const AUTO_MODES: AutoMode[] = ["off", "next", "prev", "random"];
const LOOP_MODES: LoopMode[] = ["off", "one", "all"];
/** Can `el` (or an ancestor inside the modal) still scroll vertically in `dir` (-1 up, 1 down)?
* Used to leave ArrowUp/ArrowDown alone while there is content to scroll — the volume shortcut
* must not eat the only keyboard way to read a long description. Stops at `<body>`: the page
* behind the modal is `overflow:hidden` anyway. */
function canScrollBy(el: HTMLElement | null, dir: -1 | 1): boolean {
for (let node = el; node && node !== document.body; node = node.parentElement) {
const style = getComputedStyle(node);
if (!/(auto|scroll|overlay)/.test(style.overflowY)) continue;
const room =
dir < 0 ? node.scrollTop > 0 : node.scrollTop + node.clientHeight < node.scrollHeight - 1; // -1: sub-pixel heights
if (room) return true;
}
return false;
}
// --- IFrame Player API loader (singleton) ---
let apiPromise: Promise<any> | null = null;
function loadYouTubeApi(): Promise<any> {
@@ -155,6 +181,42 @@ export default function PlayerModal({
// navigable (previously the overlay ate clicks below the top items). Moving the pointer off the
// video, or the window regaining focus, re-arms our click/scroll/keyboard shortcuts.
const [nativeControls, setNativeControls] = useState(false);
// True while OUR stage element is the fullscreen element. In fullscreen the interaction overlay
// yields entirely (see the overlay's pointer-events below): YouTube's control bar only appears
// while the iframe itself sees mouse movement, so an armed overlay kept the bar — and with it
// the native fullscreen button — permanently hidden after pressing F.
const [isFullscreen, setIsFullscreen] = useState(false);
// Our own "fills the viewport" mode (the F key). Not the Fullscreen API — see toggleFullscreen.
const [maximized, setMaximized] = useState(false);
const maximizedRef = useRef(false);
maximizedRef.current = maximized;
// Mirrored into a ref because the keydown handler is bound once (deps: [onClose]) and would
// otherwise read `false` forever — the house pattern here (see navRef/volumeRef).
const isFullscreenRef = useRef(false);
// Nested Back: peel fullscreen → maximise → close, one browser/mouse Back at a time (each is a
// history layer only while active). Mirrors the Esc ladder in the keydown handler below.
useBackToExit(isFullscreen, exitFullscreen);
useBackToExit(maximized, () => setMaximized(false));
// Volume survives the modal, the next video and a reload: a YT.Player is created per video and
// always starts at full volume. Per-account, like the Plex player's prefs. Level 0 IS the muted
// state (that's how YouTube's own control behaves), so there's no separate `muted` flag to
// contradict it.
//
// Ref + explicit read/write rather than `useAccountPersistedObject`: nothing RENDERS the level
// (the on-player flash has its own `volumeUi` state), the once-bound wheel/key listeners need a
// ref anyway, and the hook's write lives inside a `setState` updater — which is both a second
// write on every settle and not something React must run while the component is unmounting.
// Sanitised HERE, at the boundary where the untrusted value enters (`normalizeVolume`, see
// lib/playerVolume): a stored `null` / `"80"` / missing key becomes the default once, instead of
// travelling into the arithmetic. Read via a lazy `useState` initializer so it happens once, not
// on every render (the house pattern — see the frozen queue above).
const [storedVolume] = useState(() =>
normalizeVolume(readAccountMerged(LS.ytPlayerPrefs, { volume: DEFAULT_VOLUME }).volume),
);
const volumeRef = useRef(storedVolume);
const volSaveTimerRef = useRef<number | undefined>(undefined);
// When the last volume gesture happened — see nudgeVolume (a spin must not re-read the player).
const lastNudgeAtRef = useRef(0);
const focusModal = () => cardRef.current?.focus();
const togglePlay = () => {
@@ -164,26 +226,83 @@ export default function PlayerModal({
p.pauseVideo?.(); // 1 === playing
else p.playVideo?.();
};
const toggleFullscreen = () => {
const el = stageRef.current;
if (!el) return;
if (document.fullscreenElement) document.exitFullscreen?.();
else el.requestFullscreen?.();
/** F = maximise, deliberately WITHOUT the Fullscreen API.
*
* The measured reason (Chrome/Brave/Firefox alike, so it is spec behaviour, not a browser bug):
* a cross-origin embed reads its fullscreen state from ITS OWN document. While WE hold the
* browser's fullscreen lock — on our wrapper or even on the iframe element — the YouTube player
* never learns it is fullscreen, so its own fullscreen button stays in the "enter" state and a
* click on it changes nothing visible. That is the reported bug: every other YouTube control
* (gear, CC, volume) works, only its fullscreen button looks dead.
*
* So we don't take the lock at all: `maximized` is a CSS layer that fills the viewport. The page
* stays non-fullscreen, which means the embed's own fullscreen button works exactly as it does
* anywhere else — and OUR shortcuts and overlays (wheel volume, the level flash, the queue bar)
* keep working, because our DOM is what's on screen. The one thing we give up is hiding the
* browser's own chrome; YouTube's button provides that, correctly, when the user wants it.
*/
const toggleFullscreen = () => setMaximized((m) => !m);
// True browser fullscreen that WE own (the stage element). Unlike YouTube's own fullscreen it keeps
// OUR overlay + control bar on screen, so wheel/arrow volume and our controls work inside it too.
// We never surface YouTube's own fullscreen button, so the "a cross-origin embed's own button looks
// dead while we hold the lock" problem that made F avoid the Fullscreen API does not arise.
const toggleTrueFullscreen = () => {
if (fullscreenElement()) exitFullscreen();
else if (stageRef.current) requestFullscreen(stageRef.current);
};
// Leave the big view back to the modal card: drop fullscreen first if held, then un-maximise.
const exitBigView = () => {
if (fullscreenElement()) exitFullscreen();
setMaximized(false);
};
const flashVolume = (level: number) => {
setVolumeUi(level);
window.clearTimeout(volTimerRef.current);
volTimerRef.current = window.setTimeout(() => setVolumeUi(null), 1200);
};
const nudgeVolume = (delta: number) => {
/** Write the current level out now, cancelling any pending debounce. */
const persistVolume = () => {
window.clearTimeout(volSaveTimerRef.current);
volSaveTimerRef.current = undefined;
writeAccount(LS.ytPlayerPrefs, { volume: volumeRef.current });
};
/** Push a level (0100) to the player and remember it; 0 also mutes (a player left at zero
* volume but unmuted shows YouTube's speaker icon as ON, which reads as broken). */
const applyVolume = (level: number, flash = true) => {
// Callers hand over an already-normalised level (stepVolume / the stored value), but the
// player can still answer NaN, so re-normalise here against the LAST GOOD level — never the
// default: jumping a quiet session to full blast is the surprise this feature exists to avoid.
const next = normalizeVolume(level, volumeRef.current);
const p = playerRef.current;
if (!p || typeof p.getVolume !== "function") return;
const cur = p.isMuted?.() ? 0 : (p.getVolume?.() ?? 50);
const next = Math.max(0, Math.min(100, Math.round(cur + delta)));
if (next > 0) p.unMute?.();
p.setVolume?.(next);
if (next === 0) p.mute?.();
flashVolume(next);
if (p && typeof p.setVolume === "function") {
if (next > 0) p.unMute?.();
p.setVolume(next);
if (next === 0) p.mute?.();
}
volumeRef.current = next;
// Debounced: one write when the spin settles, not one per notch. The unmount cleanup flushes
// a pending write, so the last change is never lost.
window.clearTimeout(volSaveTimerRef.current);
volSaveTimerRef.current = window.setTimeout(persistVolume, 400);
if (flash) flashVolume(next);
};
const nudgeVolume = (delta: number) => {
// No early return when the player isn't ready yet: the level lives in `volumeRef` and onReady
// applies it, so a wheel spin in the first second after opening (the natural reaction to a
// video that starts loud) now lands instead of being silently dropped.
const p = playerRef.current;
// Whose level do we step from? Our own ref, EXCEPT when the gesture starts fresh — then read
// the player, because the user may have moved YouTube's own slider in the meantime.
// `getVolume()` is answered across the iframe boundary and lags our `setVolume` by a beat, so
// re-reading it mid-spin resurrects a stale level and silently drops notches (measured: five
// wheel notches moved 100 → 90 instead of 75).
const now = Date.now();
const continuing = now - lastNudgeAtRef.current < 1000;
lastNudgeAtRef.current = now;
const live =
p && typeof p.getVolume === "function" ? (p.isMuted?.() ? 0 : p.getVolume()) : null;
const cur = continuing || live === null ? volumeRef.current : live;
applyVolume(stepVolume(cur, { delta, fallback: volumeRef.current }));
};
// The player can navigate to other videos (YouTube links in a description). The
@@ -375,7 +494,13 @@ export default function PlayerModal({
const onKey = (e: KeyboardEvent) => {
if (e.key === "Escape") {
// In fullscreen, let the browser's Esc exit fullscreen only — don't also close the modal.
if (document.fullscreenElement) return;
if (fullscreenElement()) return;
// Same for our own maximise: Esc steps back out of it before it closes the player.
if (maximizedRef.current) {
e.preventDefault();
setMaximized(false);
return;
}
// Defer to any Modal opened above the player (e.g. the download dialog): its Escape closes
// it, ours would otherwise also fire and close the player underneath it (U-C).
if (modalCount() > 0) return;
@@ -396,6 +521,21 @@ export default function PlayerModal({
if (typing || tag === "button" || tag === "a") return;
e.preventDefault();
togglePlay();
} else if (e.key === "ArrowUp" || e.key === "ArrowDown") {
// Volume by keyboard (YouTube/Plex-style ±5). The only volume control that survives
// fullscreen, where the wheel goes to YouTube's own iframe. But the modal card scrolls
// (a long description) and Up/Down is how you scroll it from the keyboard — so defer
// whenever the focused element can still scroll that way, exactly like Space defers to a
// focused button. NOT while the video fills the screen though — in fullscreen OR in our own
// maximise — because the card is behind/off-screen there: deferring would scroll something
// invisible and leave the viewer with no volume control at all. Both flags, not just
// fullscreen: since F stopped taking the browser's lock, maximise is the common case and
// checking only `isFullscreen` made the keys dead exactly where they matter most.
const up = e.key === "ArrowUp";
const covered = isFullscreenRef.current || maximizedRef.current;
if (typing || (!covered && canScrollBy(el, up ? -1 : 1))) return;
e.preventDefault();
nudgeVolume(up ? 5 : -5);
} else if (e.key === "ArrowLeft" || e.key === "ArrowRight") {
// Plain arrows seek within the video (YouTube-style ±5s); Shift+arrow steps to the
// previous/next video in the queue (the feed's order or a playlist).
@@ -420,6 +560,9 @@ export default function PlayerModal({
window.clearTimeout(closeTimer.current);
window.clearTimeout(openTimer.current);
window.clearTimeout(volTimerRef.current);
// Flush a debounced volume write — closing the player right after a wheel spin (the normal
// way to leave) must not drop the level the user just set.
if (volSaveTimerRef.current != null) persistVolume();
};
}, [onClose]);
@@ -431,6 +574,9 @@ export default function PlayerModal({
const el = dialogRef.current;
if (!el) return;
const onWheel = (e: WheelEvent) => {
// Ctrl/⌘+wheel is the browser's zoom gesture — never swallow it. Volume is the PLAIN wheel
// (and ↑/↓); taking the modifier too left zoom unreachable while the player was open.
if (e.ctrlKey || e.metaKey) return;
e.preventDefault();
focusModal(); // re-grab focus if it had slipped into the native controls
nudgeVolume(e.deltaY < 0 ? 5 : -5);
@@ -439,6 +585,27 @@ export default function PlayerModal({
return () => el.removeEventListener("wheel", onWheel);
}, []);
// Track the browser's fullscreen state. WE never request it any more (F maximises via CSS), so
// this is YouTube's own fullscreen — entered from its control bar, left by its button or Esc.
useEffect(() => {
const onChange = () => {
const fsEl = fullscreenElement();
// Refs only — this listener is bound once, so it must not close over render values.
const p = playerRef.current;
const frame = p && typeof p.getIframe === "function" ? p.getIframe() : null;
const ours = !!fsEl && (fsEl === stageRef.current || fsEl === frame);
isFullscreenRef.current = ours; // read by the once-bound keydown handler
setIsFullscreen(ours); // drives the overlay's pointer-events
// LEAVING fullscreen is where the user got stranded: to reach YouTube's fullscreen button
// they had to click INTO the player, which moves keyboard focus into the cross-origin iframe
// — and it stays there afterwards. Our window-level shortcuts then never fire (F went to
// YouTube instead, giving a second, control-less fullscreen), so the maximised view had no
// way out. Taking focus back here restores F/Space/arrows the moment fullscreen ends.
if (!fsEl) focusModal();
};
return onFullscreenChange(onChange); // both spellings — see lib/fullscreen
}, []);
// Yield the interaction overlay to YouTube's native UI. Clicking a native control (gear / seek
// bar / CC) moves focus into the player iframe — the only cross-origin signal we get. While the
// iframe holds focus we drop the overlay's pointer-events so the (arbitrarily tall) settings
@@ -532,8 +699,12 @@ export default function PlayerModal({
vq: "hd1080",
},
events: {
// Keep keyboard focus on the modal, not the iframe, so shortcuts work right away.
onReady: () => focusModal(),
// Keep keyboard focus on the modal, not the iframe, so shortcuts work right away, and
// restore the remembered volume — a fresh player always starts at 100.
onReady: () => {
focusModal();
applyVolume(volumeRef.current, false);
},
onStateChange: (e: any) => {
// 1 === playing → sync the navigated-to video's title/author for display.
if (e?.data === 1) {
@@ -653,19 +824,34 @@ export default function PlayerModal({
>
<div
ref={stageRef}
className="player-stage relative aspect-video w-full bg-black rounded-t-2xl overflow-hidden"
className={`player-stage relative aspect-video w-full bg-black rounded-t-2xl overflow-hidden ${
maximized ? "player-stage--max" : ""
}`}
>
{/* Hide the iframe entirely on error so YouTube's own error screen can't bleed
through our overlay. */}
{/* In the big views the video area is inset by the control bar's height (`bottom-14`) so
the bar below never overlaps it. The inset lives on THIS wrapper, not on `mountRef`:
YT.Player REPLACES the mount element with its iframe (width/height 100% of the parent),
which would drop any class on mountRef itself — the iframe then sizes to this wrapper. */}
<div
ref={mountRef}
className={`w-full h-full ${playerError != null ? "invisible" : ""}`}
/>
{/* Interaction layer over the CENTRE of the video: catches click (play/pause) and stops
the iframe stealing keyboard focus. It deliberately leaves the top AND bottom edges
uncovered so YouTube's native controls — the top-right cluster (volume / CC / settings)
and the bottom bar (seek / More videos / fullscreen) — stay clickable. Hidden on error
so the "Open on YouTube" CTA is clickable. */}
className={
maximized || isFullscreen ? "absolute inset-x-0 top-0 bottom-14" : "w-full h-full"
}
>
<div
ref={mountRef}
className={`w-full h-full ${playerError != null ? "invisible" : ""}`}
/>
</div>
{/* Interaction layer over the CENTRE of the video: catches click (play/pause) + wheel
volume (the wheel event bubbles from here to the modal's listener — over the bare
cross-origin iframe it would be swallowed) and stops the iframe stealing keyboard focus.
It leaves the top AND bottom edges uncovered so YouTube's edge controls stay reachable.
Armed in the big views too (that is what keeps wheel volume working after F, the reported
gap) — it only yields to `nativeControls`, i.e. once the user opens YouTube's own
settings menu (focus moves into the iframe), so that menu is navigable end to end.
Hidden on error so the "Open on YouTube" CTA is clickable. */}
{playerError == null && (
<div
onClick={() => {
@@ -679,6 +865,143 @@ export default function PlayerModal({
}`}
/>
)}
{/* Our own control bar for the big views (maximise / our fullscreen), where the card's
controls are off-screen. It's IN the stage (so it survives true fullscreen, where a
<body> portal would not) and DOCKED in its own strip: the video area above is inset by
this bar's height (see mountRef's `bottom-14`), so the bar NEVER overlaps YouTube's seek
bar — that stays reachable by hovering the video's bottom edge, just above ours. Always
visible (no auto-hide) so there's no guessing whether it will appear and no fight with the
seek bar over the same pixels. stopPropagation so a click here doesn't toggle playback;
the exit button also re-arms our shortcuts. Add-to-playlist / download are hidden in true
fullscreen — their popovers portal to <body>, outside the fullscreen element. */}
{(maximized || isFullscreen) && (
<div
onClick={(e) => e.stopPropagation()}
className="absolute inset-x-0 bottom-0 z-menu flex h-14 items-center justify-start gap-x-3 overflow-x-auto border-t border-white/10 bg-neutral-950/95 px-3 text-sm text-white"
>
{hasQueue && (
<>
<button
onClick={goPrev}
disabled={index === 0}
title={`${t("player.previous")} · Shift+←`}
className="inline-flex items-center gap-1 text-white/80 enabled:hover:text-white disabled:opacity-30 transition"
>
<SkipBack className="h-4 w-4" />
<span className="hidden sm:inline">{t("player.previous")}</span>
</button>
<span className="tabular-nums text-white/70">
{index + 1} / {queue!.length}
</span>
<button
onClick={goNext}
disabled={index === queue!.length - 1}
title={`${t("player.next")} · Shift+→`}
className="inline-flex items-center gap-1 text-white/80 enabled:hover:text-white disabled:opacity-30 transition"
>
<span className="hidden sm:inline">{t("player.next")}</span>
<SkipForward className="h-4 w-4" />
</button>
<span className="h-4 w-px bg-white/20" />
<button
onClick={cycleAuto}
title={t("player.autoAdvance.hint")}
className={`inline-flex items-center gap-1 rounded-lg px-2 py-1 transition ${
autoMode !== "off"
? "bg-white/20 text-white"
: "text-white/70 hover:bg-white/10 hover:text-white"
}`}
>
{autoMode === "prev" ? (
<SkipBack className="h-3.5 w-3.5" />
) : autoMode === "random" ? (
<Shuffle className="h-3.5 w-3.5" />
) : (
<SkipForward className="h-3.5 w-3.5" />
)}
<span className="hidden md:inline">
{t("player.autoAdvance.label")}: {t(`player.autoAdvance.${autoMode}`)}
</span>
</button>
<button
onClick={cycleLoop}
title={t("player.loop.hint")}
className={`inline-flex items-center gap-1 rounded-lg px-2 py-1 transition ${
loopMode !== "off"
? "bg-white/20 text-white"
: "text-white/70 hover:bg-white/10 hover:text-white"
}`}
>
{loopMode === "one" ? (
<Repeat1 className="h-3.5 w-3.5" />
) : (
<Repeat className="h-3.5 w-3.5" />
)}
<span className="hidden md:inline">
{t("player.loop.label")}: {t(`player.loop.${loopMode}`)}
</span>
</button>
<span className="h-4 w-px bg-white/20" />
</>
)}
{!navigated && !isFullscreen && (
<AddToPlaylist
videoId={active.id}
className="inline-flex h-8 w-8 items-center justify-center rounded-lg text-white/80 hover:bg-white/10 hover:text-white transition"
/>
)}
{!navigated && !isFullscreen && (
<DownloadButton
videoId={active.id}
title={active.title}
className="inline-flex h-8 w-8 items-center justify-center rounded-lg text-white/80 hover:bg-white/10 hover:text-white transition"
/>
)}
{!navigated && (
<button
onClick={() => setWatched(!watched)}
title={watched ? t("player.watchedUnmark") : t("player.markWatched")}
className={`inline-flex items-center gap-1 rounded-lg px-2 py-1 transition ${
watched
? "bg-white/25 text-white"
: "text-white/70 hover:bg-white/10 hover:text-white"
}`}
>
{watched ? <CheckCheck className="h-4 w-4" /> : <Check className="h-4 w-4" />}
<span className="hidden md:inline">
{watched ? t("player.watched") : t("player.markWatched")}
</span>
</button>
)}
<span className="h-4 w-px bg-white/20" />
<button
onClick={toggleTrueFullscreen}
title={isFullscreen ? t("player.exitFullscreen") : t("player.fullscreen")}
aria-label={isFullscreen ? t("player.exitFullscreen") : t("player.fullscreen")}
className="inline-flex h-8 w-8 items-center justify-center rounded-lg text-white/80 hover:bg-white/10 hover:text-white transition"
>
{isFullscreen ? (
<Minimize className="h-4 w-4" />
) : (
<Maximize className="h-4 w-4" />
)}
</button>
<button
onClick={(e) => {
e.stopPropagation();
exitBigView();
setNativeControls(false);
focusModal();
}}
title={t("player.exitMaximize")}
aria-label={t("player.exitMaximize")}
className="inline-flex items-center gap-1 rounded-lg px-2 py-1 text-white/80 hover:bg-white/10 hover:text-white transition"
>
<Minimize2 className="h-4 w-4" />
<span className="hidden sm:inline">{t("player.exitMaximize")}</span>
</button>
</div>
)}
{/* Discreet badge while YouTube's own controls have taken over (overlay yielded). */}
{playerError == null && nativeControls && (
<div className="pointer-events-none absolute top-3 left-3 z-menu flex items-center gap-1.5 rounded-full bg-black/60 px-2.5 py-1 text-[11px] text-white/90 shadow-lg backdrop-blur-sm">
+14 -8
View File
@@ -35,6 +35,12 @@ import {
} from "lucide-react";
import { api, type PlexMarker } from "../lib/api";
import { formatDuration } from "../lib/format";
import {
exitFullscreen,
fullscreenElement,
onFullscreenChange,
requestFullscreen,
} from "../lib/fullscreen";
import { LS, useAccountPersistedObject } from "../lib/storage";
import { useDismiss } from "../lib/useDismiss";
import { useScrollFade } from "../lib/useScrollFade";
@@ -579,17 +585,17 @@ export default function PlexPlayer({ itemId, onClose, queue }: Props) {
[nudgeVolume],
);
// Via lib/fullscreen, not the raw API: WebKit (Safari/iPadOS) ships only the prefixed spelling,
// so reading `document.fullscreenElement` alone made this player believe it is never fullscreen
// — the button did nothing, `fs` never flipped, and Escape (below) read "not fullscreen" as
// permission to close the whole player instead of just leaving fullscreen.
const toggleFs = useCallback(() => {
const el = wrapRef.current;
if (!el) return;
if (document.fullscreenElement) document.exitFullscreen().catch(() => {});
else el.requestFullscreen().catch(() => {});
}, []);
useEffect(() => {
const onFs = () => setFs(!!document.fullscreenElement);
document.addEventListener("fullscreenchange", onFs);
return () => document.removeEventListener("fullscreenchange", onFs);
if (fullscreenElement()) exitFullscreen();
else requestFullscreen(el);
}, []);
useEffect(() => onFullscreenChange(() => setFs(!!fullscreenElement())), []);
const go = useCallback((otherId: string | null | undefined) => {
if (otherId) {
@@ -876,7 +882,7 @@ export default function PlexPlayer({ itemId, onClose, queue }: Props) {
setTracksOpen(false);
} else if (helpOpenRef.current) setHelpOpen(false);
else if (infoOpenRef.current) setInfoOpen(false);
else if (!document.fullscreenElement) onClose();
else if (!fullscreenElement()) onClose();
break;
}
};
+5 -2
View File
@@ -35,6 +35,9 @@
"unavailableBody": "This video isn't available — it may be private, removed, or region-restricted.",
"embedDisabledBody": "The owner disabled playback on other sites (common for auto-generated “Topic” music tracks). It still plays on YouTube.",
"openOnYoutube": "Open on YouTube",
"shortcutsHint": "Click: play/pause · Scroll: volume · F: fullscreen",
"nativeControls": "YouTube controls active"
"shortcutsHint": "Click: play/pause · Scroll or ↑ ↓: volume · F: fill the window",
"exitMaximize": "Exit full window (Esc)",
"nativeControls": "YouTube controls active",
"fullscreen": "Fullscreen",
"exitFullscreen": "Exit fullscreen"
}
+5 -2
View File
@@ -35,6 +35,9 @@
"unavailableBody": "Ez a videó nem elérhető — lehet privát, eltávolított vagy régiókorlátozott.",
"embedDisabledBody": "A tulajdonos letiltotta a lejátszást más oldalakon (gyakori az auto-generált „Topic” zenei sávoknál). A YouTube-on viszont lejátszható.",
"openOnYoutube": "Megnyitás YouTube-on",
"shortcutsHint": "Kattintás: lejátszás/szünet · Görgő: hangerő · F: teljes képernyő",
"nativeControls": "YouTube vezérlők aktívak"
"shortcutsHint": "Kattintás: lejátszás/szünet · Görgő vagy ↑ ↓: hangerő · F: teljes ablak",
"exitMaximize": "Kilépés a teljes ablakból (Esc)",
"nativeControls": "YouTube vezérlők aktívak",
"fullscreen": "Teljes képernyő",
"exitFullscreen": "Kilépés a teljes képernyőből"
}
+30 -4
View File
@@ -579,13 +579,39 @@ html[data-scheme="matrix"][data-theme="light"] {
}
}
/* The in-app player stage, when fullscreened via the F key / native button, fills the screen
instead of keeping its 16:9 letterbox (and drops the rounded top corners). */
/* The in-app player stage filling the screen: either because something fullscreened it (YouTube's
own button hands the lock to the iframe, not to us — but keep this for any browser that does),
or via our own F-key maximise. `--max` deliberately does NOT use the Fullscreen API: while we
hold the browser's lock, the cross-origin YouTube embed never learns it is fullscreen and its own
fullscreen button stops working (see PlayerModal.toggleFullscreen). */
.player-stage:fullscreen,
.player-stage--max {
aspect-ratio: auto;
border-radius: 0;
}
/* A fullscreened element is already sized by the browser; the viewport units are only for the
`:fullscreen` path, where the stage is not positioned. */
.player-stage:fullscreen {
width: 100vw;
height: 100vh;
aspect-ratio: auto;
border-radius: 0;
}
/* Sizing, precisely: the element keeps Tailwind's `w-full` from its base className, so the WIDTH is
`100%` of the containing block — which, for a fixed element, is the viewport minus any classic
scrollbar. The HEIGHT comes from `inset: 0` (top+bottom, once `aspect-ratio` is released above).
`right: 0` is therefore ignored (over-constrained: left + width win) — harmless, but don't read
this rule as inset-sized. What had to go are the viewport units it used to inherit: `100vw`
INCLUDES the scrollbar (horizontal overflow) and `100vh` includes a mobile browser's retracting
chrome, which pushes the bottom edge — YouTube's own control bar — off screen. */
.player-stage--max {
position: fixed;
inset: 0;
/* rail-level (40): this stage only has to beat the modal chrome it is nested inside (whose own
children top out at z-menu/20). The modal root is a z-overlay stacking context, so nothing
here escapes it — 9999 is the glass tuner's reserved level and claimed an authority this
layer neither has nor needs. */
z-index: 40;
}
/* Thin, theme-aware scrollbars */
+49
View File
@@ -0,0 +1,49 @@
// The browser's Fullscreen API, WebKit included.
//
// Safari and iPadOS still ship only the prefixed spelling: no `fullscreenchange`, no
// `fullscreenElement`, no `requestFullscreen` on a plain element. Reading only the unprefixed names
// there means the page believes it is NEVER fullscreen — which silently breaks whatever the
// fullscreen state drives (an overlay that must yield, a focus reclaim, an Esc branch that would
// otherwise close the whole player instead of just leaving fullscreen).
//
// Both video surfaces (PlayerModal's YouTube embed, PlexPlayer's <video>) go through here, so the
// prefix handling lives in ONE place — the previous shape had it in neither, then in one.
type WebkitDocument = Document & {
webkitFullscreenElement?: Element | null;
webkitExitFullscreen?: () => void;
};
type WebkitElement = HTMLElement & { webkitRequestFullscreen?: () => void };
/** The element currently holding the fullscreen lock, or null. */
export function fullscreenElement(): Element | null {
const d = document as WebkitDocument;
return d.fullscreenElement ?? d.webkitFullscreenElement ?? null;
}
/** Ask for fullscreen on `el`. Errors are swallowed on purpose: a refused request (no user
* activation, an iOS device that only fullscreens <video>) is not something a caller can fix. */
export function requestFullscreen(el: HTMLElement): void {
const e = el as WebkitElement;
if (e.requestFullscreen) void e.requestFullscreen().catch(() => {});
else e.webkitRequestFullscreen?.();
}
/** Leave fullscreen, whoever holds it. */
export function exitFullscreen(): void {
const d = document as WebkitDocument;
if (d.exitFullscreen) void d.exitFullscreen().catch(() => {});
else d.webkitExitFullscreen?.();
}
/** Subscribe to fullscreen transitions; returns the unsubscribe. Registers BOTH spellings — a
* browser that supports both never fires both for the same transition, so the handler still runs
* once. */
export function onFullscreenChange(handler: () => void): () => void {
document.addEventListener("fullscreenchange", handler);
document.addEventListener("webkitfullscreenchange", handler);
return () => {
document.removeEventListener("fullscreenchange", handler);
document.removeEventListener("webkitfullscreenchange", handler);
};
}
+23
View File
@@ -114,3 +114,26 @@ export function useBackToClose(onClose: () => void): void {
};
}, []);
}
/** A Back-closable layer that exists only WHILE `active`. Stacks on top of `useBackToClose` and any
* other active `useBackToExit` (LIFO) so nested views peel off one Back at a time — e.g. a player's
* fullscreen sits above its maximise, which sits above the player itself: Back exits fullscreen,
* then maximise, then closes the player. Leaving a layer by its own control (button/Esc) flips
* `active` false, whose cleanup drops the entry and its history state on the next tick — exactly the
* useBackToClose lifecycle, just gated on a flag. `onExit` runs on a genuine Back while active. */
export function useBackToExit(active: boolean, onExit: () => void): void {
const cb = useRef(onExit);
cb.current = onExit;
useEffect(() => {
if (!active) return;
ensureOverlayListener();
const entry: Overlay = { token: ++overlayCounter, cb: () => cb.current() };
overlayStack.push(entry);
scheduleReconcile();
return () => {
const idx = overlayStack.indexOf(entry);
if (idx !== -1) overlayStack.splice(idx, 1);
scheduleReconcile();
};
}, [active]);
}
+58
View File
@@ -0,0 +1,58 @@
import { describe, expect, it } from "vitest";
import { DEFAULT_VOLUME, normalizeVolume, stepVolume } from "./playerVolume";
describe("normalizeVolume", () => {
it("keeps a valid level", () => {
expect(normalizeVolume(0)).toBe(0);
expect(normalizeVolume(37)).toBe(37);
expect(normalizeVolume(100)).toBe(100);
});
it("clamps out-of-range levels and rounds to whole percents", () => {
expect(normalizeVolume(-20)).toBe(0);
expect(normalizeVolume(150)).toBe(100);
expect(normalizeVolume(42.6)).toBe(43);
});
it.each([
["a missing key", undefined],
["an explicit null", null],
["a stringified number", "80"],
["NaN from a broken player", NaN],
["a whole object", { volume: 50 }],
])("falls back on %s", (_label, raw) => {
// The regression: a partial stored object made the level undefined, which Math.round turned
// into NaN — muting playback and persisting `null` for every later session.
expect(normalizeVolume(raw, 15)).toBe(15);
});
it("prefers the last known level over the default, so a quiet session stays quiet", () => {
expect(normalizeVolume(NaN, 15)).toBe(15);
expect(normalizeVolume(NaN, 15)).not.toBe(DEFAULT_VOLUME);
});
it("uses the default when the fallback is unusable too", () => {
expect(normalizeVolume(NaN, NaN)).toBe(DEFAULT_VOLUME);
expect(normalizeVolume(null, undefined)).toBe(DEFAULT_VOLUME);
expect(normalizeVolume(null, "80")).toBe(DEFAULT_VOLUME);
});
});
describe("stepVolume", () => {
it("steps up and down from the current level", () => {
expect(stepVolume(60, { delta: 5, fallback: 100 })).toBe(65);
expect(stepVolume(60, { delta: -5, fallback: 100 })).toBe(55);
});
it("clamps at both ends instead of wrapping", () => {
expect(stepVolume(98, { delta: 5, fallback: 100 })).toBe(100);
expect(stepVolume(3, { delta: -5, fallback: 100 })).toBe(0);
});
it("steps from the fallback when the player answers garbage", () => {
// A spin must not lose notches to a stale/unusable reading — it continues from what we last set.
expect(stepVolume(undefined, { delta: -5, fallback: 90 })).toBe(85);
expect(stepVolume(NaN, { delta: -5, fallback: 90 })).toBe(85);
});
});
+33
View File
@@ -0,0 +1,33 @@
// The YouTube player's volume arithmetic, kept out of PlayerModal so it can be unit-tested. Every
// rule here was established by a bug: a wheel spin that lost notches to the iframe's lagging
// getVolume, a partial stored object that became NaN and persisted as `null`, and a "corrupt →
// default" fallback that jumped an already-quiet session to full blast.
/** Level used when nothing trustworthy is available. */
export const DEFAULT_VOLUME = 100;
const clamp = (n: number) => Math.max(0, Math.min(100, Math.round(n)));
/**
* Coerce anything that claims to be a volume into a usable 0100 level.
*
* Only a real, finite `number` is accepted: the stored blob is JSON, so a level is a number or it
* is corrupt — `null`, `undefined`, `"80"` and `NaN` all mean "we don't know", and `Number(null)`
* being `0` (silently muting the player) is exactly the coercion to avoid. `fallback` is the last
* level known to work; it is itself validated, so a corrupt value can never travel through it.
*/
export function normalizeVolume(raw: unknown, fallback: unknown = DEFAULT_VOLUME): number {
if (typeof raw === "number" && Number.isFinite(raw)) return clamp(raw);
if (typeof fallback === "number" && Number.isFinite(fallback)) return clamp(fallback);
return DEFAULT_VOLUME;
}
/** One volume step (wheel notch / arrow key) from `current`.
*
* `delta` and `fallback` ride in an options object on purpose: as two adjacent `number`
* parameters they were transposable without a type error, and swapping them silently steps the
* volume BY the current level. */
export function stepVolume(current: unknown, opts: { delta: number; fallback: unknown }): number {
const base = normalizeVolume(current, opts.fallback);
return clamp(base + opts.delta);
}
+15
View File
@@ -14,6 +14,21 @@ export interface ReleaseEntry {
}
export const RELEASE_NOTES: ReleaseEntry[] = [
{
version: "0.54.0",
date: "2026-07-26",
summary:
"A sturdier player — press F to go big, arrow-key volume that's remembered — plus downloads and subscription sync that hold up when something goes wrong.",
features: [
"Player: press F to maximise the video to a full-window view — now with its own control bar (previous/next, auto-advance, loop, add-to-playlist, download, mark watched) that auto-hides while you watch, plus a button for true fullscreen. Scroll-wheel and ↑/↓ volume work in every mode (the full window and fullscreen too), and your volume is remembered per account.",
"Volume: Ctrl+scroll zooms the page as usual again instead of being captured by the player. Keyboard shortcuts keep working after you click into the video, and the mouse/browser Back button now steps out one level at a time (fullscreen → full window → the normal player → closed) instead of closing the whole player at once.",
],
fixes: [
"Downloads: fixed a class of downloads that could fail with an internal error (a produced file the downloader couldn't locate), and mp4 downloads are no longer needlessly re-encoded — they finish faster and keep their original quality.",
"Downloads: leftover temporary playback files from an interrupted session are now cleaned up automatically on restart, and a cancelled download stops reliably instead of hanging.",
"Subscription sync: a one-off empty response from YouTube can no longer wipe your subscription list; playlist sync now stops safely once a daily quota limit is hit instead of burning through the rest; and a permanently unavailable channel is reported instead of silently looking healthy.",
],
},
{
version: "0.53.1",
date: "2026-07-25",
+4
View File
@@ -50,6 +50,10 @@ export const LS = {
plexQ: "siftlode.plexQ",
plexPlaylistLayout: "siftlode.plexPlaylistLayout",
plexPlayerPrefs: "siftlode.plexPlayerPrefs",
// The YouTube modal player's volume (`{ volume: 0100 }`; 0 IS muted — there is no separate
// flag). A YT.Player instance is created per video and starts at full volume, so without this
// every next video is loud again (same idea as plexPlayerPrefs).
ytPlayerPrefs: "siftlode.ytPlayerPrefs",
notifHistory: "siftlode.notifications",
notifSettings: "siftlode.notifSettings",
onboardingDismissed: "siftlode.onboarding.dismissed",