peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

style: apply ruff format across backend + scripts (line-length 100)

Browse Source 
Mechanical, whole-tree `ruff format` pass — no behaviour change. Recorded in
.git-blame-ignore-revs (next commit) so git blame skips it.
This commit is contained in:
peter
2026-08-23 07:43:06 +02:00
parent 434a98d6ed
commit b1832afd18
217 changed files with 3149 additions and 1744 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/audit.py
+1
View File
@@ -9,6 +9,7 @@ runs are always logged. Secrets are never stored (config changes log the key nam
mutation it describes, so the trail can't drift from reality (and a rolled-back mutation drops
its audit row too).
"""
from datetime import datetime, timedelta, timezone
from sqlalchemy import delete
backend/app/auth.py
+6 -9
View File
@@ -41,8 +41,7 @@ log = logging.getLogger("siftlode.auth")
def google_enabled(db: Session) -> bool:
"""Whether Sign in with Google is available (both client id + secret resolve to non-empty)."""
return bool(
sysconfig.get_str(db, "google_client_id")
and sysconfig.get_str(db, "google_client_secret")
sysconfig.get_str(db, "google_client_id") and sysconfig.get_str(db, "google_client_secret")
)
@@ -125,7 +124,9 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
# ciphertext, so a token row with no refresh token silently failed to revoke). decrypt_optional so a
# TOKEN_ENCRYPTION_KEY mismatch can't 500 the purge — a best-effort revoke degrades to "skip". (B-core-2)
google_token = (
(decrypt_optional(tok.refresh_token_enc) or decrypt_optional(tok.access_token)) if tok else None
(decrypt_optional(tok.refresh_token_enc) or decrypt_optional(tok.access_token))
if tok
else None
)
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
db.execute(delete(User).where(User.id == user_id))
@@ -197,9 +198,7 @@ def current_user(request: Request, db: Session = Depends(get_db)) -> User:
return user
def optional_current_user(
request: Request, db: Session = Depends(get_db)
) -> User | None:
def optional_current_user(request: Request, db: Session = Depends(get_db)) -> User | None:
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
@@ -215,9 +214,7 @@ def require_human(user: User = Depends(current_user)) -> User:
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
where there's no scope check to lean on."""
if user.is_demo:
raise HTTPException(
status_code=403, detail="Not available in the demo account."
)
raise HTTPException(status_code=403, detail="Not available in the demo account.")
return user
backend/app/config.py
+1
View File
@@ -169,6 +169,7 @@ class Settings(BaseSettings):
@property
def player_client_fallback_list(self) -> list[str]:
return self._clients(self.download_player_clients_fallback)
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
download_worker_concurrency: int = 2
# How often (empty pending queue) the worker polls for a new job, in seconds.
backend/app/demo.py
+12 -7
View File
@@ -4,6 +4,7 @@ Shared by the admin's manual reset endpoint (routes/admin.py) and the scheduled
(scheduler.py). Lives in the service layer, not routes, so the scheduler can call it without an
`app.routes -> app.scheduler` import cycle (the reason its import used to be lazy).
"""
from sqlalchemy import delete, func, select
from sqlalchemy.orm import Session
@@ -49,13 +50,17 @@ def _seed_demo_subscriptions(db: Session, demo: User, n: int = 14) -> int:
Channel manager isn't empty — gives visitors something to explore and matches the landing-page
screenshot. Idempotent: clears the demo's subscriptions first, then re-adds the top N."""
db.execute(delete(Subscription).where(Subscription.user_id == demo.id))
top = db.execute(
select(Video.channel_id)
.where(Video.channel_id.is_not(None))
.group_by(Video.channel_id)
.order_by(func.count().desc())
.limit(n)
).scalars().all()
top = (
db.execute(
select(Video.channel_id)
.where(Video.channel_id.is_not(None))
.group_by(Video.channel_id)
.order_by(func.count().desc())
.limit(n)
)
.scalars()
.all()
)
for channel_id in top:
db.add(Subscription(user_id=demo.id, channel_id=channel_id))
return len(top)
backend/app/downloads/adapters.py
+5 -2
View File
@@ -10,6 +10,7 @@ Adapters are stateless singletons; get one with `resolve_adapter(source_kind, so
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
"""
from urllib.parse import urlparse
from app.config import settings
@@ -140,8 +141,10 @@ def resolve_adapter(
def host_is(*domains: str) -> bool:
return any(_host_is(h, *domains) for h in hosts)
if source_kind == "youtube" or "youtube" in extractor or host_is(
"youtube.com", "youtu.be", "youtube-nocookie.com"
if (
source_kind == "youtube"
or "youtube" in extractor
or host_is("youtube.com", "youtu.be", "youtube-nocookie.com")
):
return _YOUTUBE
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
backend/app/downloads/backfill.py
+1
View File
@@ -10,6 +10,7 @@ URL may have expired — those just get a locally-generated poster instead).
Run once per environment:
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
"""
import logging
import yt_dlp
backend/app/downloads/edit.py
+151 -29
View File
@@ -20,6 +20,7 @@ Also builds the editor **filmstrip** — a single tiled sprite of the source vid
scrub track. Cached on the source asset's `storyboard_path` (reserved in phase 1) and reusable
for a player hover-preview later.
"""
import hashlib
import json
import logging
@@ -40,6 +41,7 @@ class EditAborted(Exception):
# --- edit spec ------------------------------------------------------------------------------
def normalize_edit_spec(spec: dict | None) -> dict:
"""Canonicalize a raw edit spec: clamp/round trim, coerce crop to ints, drop empties.
@@ -140,6 +142,7 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
# --- ffmpeg: trim / crop --------------------------------------------------------------------
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
@@ -153,8 +156,15 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
reencode = needs_reencode(spec)
cmd = [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-progress", "pipe:1", "-nostats",
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-progress",
"pipe:1",
"-nostats",
]
if start:
cmd += ["-ss", f"{start:.3f}"]
@@ -170,8 +180,16 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
if crop:
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
cmd += [
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
"-c:a", "aac", "-b:a", "192k",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"20",
"-c:a",
"aac",
"-b:a",
"192k",
]
else:
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
@@ -204,8 +222,15 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
crop = spec.get("crop")
prep: dict[Path, str] = {}
head = [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-progress", "pipe:1", "-nostats",
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-progress",
"pipe:1",
"-nostats",
]
if needs_reencode(spec):
@@ -218,9 +243,24 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
labels.append(f"[v{i}][a{i}]")
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
cmd = head + [
"-i", str(src), "-filter_complex", ";".join(parts),
"-map", "[v]", "-map", "[a]",
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "192k",
"-i",
str(src),
"-filter_complex",
";".join(parts),
"-map",
"[v]",
"-map",
"[a]",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"20",
"-c:a",
"aac",
"-b:a",
"192k",
]
else:
listp = staging / "concat.txt"
@@ -314,7 +354,9 @@ def _stop(proc: subprocess.Popen) -> None:
escalate_kill(proc, proc.terminate, proc.kill, grace_s=5, label="ffmpeg")
def run_ffmpeg(cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S) -> None:
def run_ffmpeg(
cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S
) -> None:
"""Run an ffmpeg edit, reporting 0–100 progress and honouring cooperative cancel.
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
@@ -411,9 +453,20 @@ def probe_codecs(path: Path) -> tuple[str | None, str | None]:
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
try:
out = subprocess.run(
["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name",
"-of", "json", str(path)],
capture_output=True, text=True, timeout=30, check=True,
[
"ffprobe",
"-v",
"error",
"-show_entries",
"stream=codec_type,codec_name",
"-of",
"json",
str(path),
],
capture_output=True,
text=True,
timeout=30,
check=True,
).stdout
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
return None, None
@@ -425,16 +478,29 @@ def probe_duration(path: Path) -> int:
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
try:
out = subprocess.run(
["ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "default=nw=1:nk=1", str(path)],
capture_output=True, text=True, timeout=30, check=True,
[
"ffprobe",
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=nw=1:nk=1",
str(path),
],
capture_output=True,
text=True,
timeout=30,
check=True,
).stdout.strip()
return int(float(out)) if out else 0
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
return 0
def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool) -> tuple[bool, bool, bool]:
def browser_transcode_flags(
vcodec: str | None, acodec: str | None, is_mp4: bool
) -> tuple[bool, bool, bool]:
"""Decide what (if anything) makes a file universally browser-playable. Returns
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
@@ -443,15 +509,30 @@ def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool
return (rv or ra or not is_mp4), rv, ra
def build_browser_safe_cmd(src: Path, dest: Path, reencode_video: bool, reencode_audio: bool) -> list[str]:
def build_browser_safe_cmd(
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool
) -> list[str]:
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
pipe:1` streams machine-readable progress for run_ffmpeg."""
cmd = [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-progress", "pipe:1", "-nostats", "-i", str(src),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-progress",
"pipe:1",
"-nostats",
"-i",
str(src),
]
cmd += ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"] if reencode_video else ["-c:v", "copy"]
cmd += (
["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"]
if reencode_video
else ["-c:v", "copy"]
)
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
cmd += ["-movflags", "+faststart", str(dest)]
return cmd
@@ -486,25 +567,63 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
fps = geom["count"] / dur
vf = f"fps={fps:.6f},scale={_SB_TILE_W}:-2,tile={geom['cols']}x{geom['rows']}"
return [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-i", str(src), "-frames:v", "1", "-q:v", "5", "-vf", vf, str(dest),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-i",
str(src),
"-frames:v",
"1",
"-q:v",
"5",
"-vf",
vf,
str(dest),
]
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
return [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-ss", f"{max(0.0, at_s):.3f}", "-i", str(src),
"-frames:v", "1", "-q:v", "3", "-vf", "scale=640:-2", str(dest),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-ss",
f"{max(0.0, at_s):.3f}",
"-i",
str(src),
"-frames:v",
"1",
"-q:v",
"3",
"-vf",
"scale=640:-2",
str(dest),
]
def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
"""Decode a downloaded thumbnail (webp/png/…) to a single baseline JPEG frame."""
return [
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
"-i", str(src), "-frames:v", "1", "-update", "1", str(dest),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-i",
str(src),
"-frames:v",
"1",
"-update",
"1",
str(dest),
]
@@ -550,7 +669,10 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
return None
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
if err is not None:
log.info("thumbnail->jpg conversion failed (%s); poster falls back to a cut frame", str(err)[:120])
log.info(
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
str(err)[:120],
)
return None
return dest if dest.exists() else None
backend/app/downloads/formats.py
+2 -3
View File
@@ -13,6 +13,7 @@ spec shape (all keys optional; defaults applied by `normalize`):
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
"""
import hashlib
import json
@@ -215,8 +216,6 @@ def build_ydl_opts(
# jpg/png thumbnail directly and converts anything else to PNG (not the crashing mjpeg2jpeg-to-jpg
# path), so it never needed the convertor and does not reintroduce the crash.
if s["embed_thumbnail"]:
opts["postprocessors"].append(
{"key": "EmbedThumbnail", "already_have_thumbnail": True}
)
opts["postprocessors"].append({"key": "EmbedThumbnail", "already_have_thumbnail": True})
return opts
backend/app/downloads/gc.py
+59 -35
View File
@@ -15,6 +15,7 @@ Five passes each run:
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
"""
import logging
import time
from datetime import datetime, timedelta, timezone
@@ -40,9 +41,7 @@ _ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight do
def _holders(db: Session, asset_id: int) -> list[int]:
return list(
db.execute(
select(DownloadJob.user_id)
.where(DownloadJob.asset_id == asset_id)
.distinct()
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
).scalars()
)
@@ -101,7 +100,9 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
pass
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
for d in sorted(
(p for p in entry.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True
(p for p in entry.rglob("*") if p.is_dir()),
key=lambda p: len(p.parts),
reverse=True,
):
try:
d.rmdir()
@@ -121,15 +122,19 @@ def run_download_gc(db: Session) -> dict:
# 1. Pre-expiry warnings.
warn_before = now + timedelta(days=grace_days)
soon = db.execute(
select(MediaAsset).where(
MediaAsset.status == "ready",
MediaAsset.gc_notified.is_(False),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at > now,
MediaAsset.expires_at <= warn_before,
soon = (
db.execute(
select(MediaAsset).where(
MediaAsset.status == "ready",
MediaAsset.gc_notified.is_(False),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at > now,
MediaAsset.expires_at <= warn_before,
)
)
).scalars().all()
.scalars()
.all()
)
for asset in soon:
_notify_holders(db, asset, "download_expiring")
asset.gc_notified = True
@@ -138,13 +143,17 @@ def run_download_gc(db: Session) -> dict:
db.commit()
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
dead = db.execute(
select(MediaAsset).where(
MediaAsset.status.in_(("ready", "error")),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at <= now,
dead = (
db.execute(
select(MediaAsset).where(
MediaAsset.status.in_(("ready", "error")),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at <= now,
)
)
).scalars().all()
.scalars()
.all()
)
for asset in dead:
_delete_asset(db, asset, "download_expired")
expired += 1
@@ -154,17 +163,24 @@ def run_download_gc(db: Session) -> dict:
# 3. LRU eviction over the total cache cap.
cap = sysconfig.get_int(db, "download_total_max_bytes")
if cap > 0:
total = db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.status == "ready"
)
).scalar() or 0
total = (
db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.status == "ready"
)
).scalar()
or 0
)
if total > cap:
candidates = db.execute(
select(MediaAsset)
.where(MediaAsset.status == "ready")
.order_by(MediaAsset.last_access_at.asc().nulls_first())
).scalars().all()
candidates = (
db.execute(
select(MediaAsset)
.where(MediaAsset.status == "ready")
.order_by(MediaAsset.last_access_at.asc().nulls_first())
)
.scalars()
.all()
)
for asset in candidates:
if total <= cap:
break
@@ -176,13 +192,17 @@ def run_download_gc(db: Session) -> dict:
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
# referenced errored row to pending on re-enqueue, so these only accumulate once every
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
orphans = db.execute(
select(MediaAsset).where(
MediaAsset.status == "error",
MediaAsset.ref_count == 0,
MediaAsset.rel_path.is_(None),
orphans = (
db.execute(
select(MediaAsset).where(
MediaAsset.status == "error",
MediaAsset.ref_count == 0,
MediaAsset.rel_path.is_(None),
)
)
).scalars().all()
.scalars()
.all()
)
for asset in orphans:
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
reaped += 1
@@ -194,7 +214,11 @@ def run_download_gc(db: Session) -> dict:
swept = _sweep_orphan_files(db, settings.download_root)
result = {
"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped, "swept": swept
"warned": warned,
"expired": expired,
"evicted": evicted,
"reaped": reaped,
"swept": swept,
}
log.info("download_gc %s", result)
return result
backend/app/downloads/jobdb.py
+2
View File
@@ -1,4 +1,5 @@
"""Short-lived-session DB helpers + staging paths shared by the worker parent and the child pipeline."""
from pathlib import Path
from sqlalchemy import text
@@ -10,6 +11,7 @@ from app.models import DownloadJob, MediaAsset
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
def _set_job(job_id: int, **fields) -> None:
with SessionLocal() as db:
job = db.get(DownloadJob, job_id)
backend/app/downloads/links.py
+7 -4
View File
@@ -6,6 +6,7 @@ require a short-lived signed **grant** on the file request — because a `<video
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
so the raw password never rides in the media URL (or server logs).
"""
import hashlib
import hmac
import secrets
@@ -27,10 +28,9 @@ def is_expired(link: DownloadLink) -> bool:
# --- signed grants (password-protected file access) ----------------------------------------
def _sign(msg: str) -> str:
return hmac.new(
settings.secret_key.encode(), msg.encode(), hashlib.sha256
).hexdigest()[:32]
return hmac.new(settings.secret_key.encode(), msg.encode(), hashlib.sha256).hexdigest()[:32]
def make_grant(token: str, password_version: int, now: float | None = None) -> str:
@@ -58,6 +58,7 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
# --- serialization -------------------------------------------------------------------------
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
"""What the file owner sees when managing their links."""
return {
@@ -72,7 +73,9 @@ def owner_view(link: DownloadLink, base_url: str = "") -> dict:
}
def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None) -> dict:
def public_meta(
link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None
) -> dict:
"""Metadata for the public watch page (only ever returned once access is authorized).
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
backend/app/downloads/og.py
+2 -3
View File
@@ -6,6 +6,7 @@ This injects per-video OG/Twitter tags into the served HTML so a shared link unf
video's title, channel and thumbnail. Real browsers ignore the extra tags and hydrate the SPA as
usual. Password-protected / expired / invalid links fall back to the generic card (no metadata leak).
"""
import html
import re
from pathlib import Path
@@ -71,9 +72,7 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
def _watch_tags(token: str, db: Session) -> str | None:
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
link = db.execute(
select(DownloadLink).where(DownloadLink.token == token)
).scalar_one_or_none()
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
if link is None or linksmod.is_expired(link):
return None
url = f"{settings.app_base}/watch/{token}"
backend/app/downloads/pipeline.py
+52 -19
View File
@@ -1,4 +1,5 @@
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
import logging
import re
import shutil
@@ -57,9 +58,7 @@ _YTDLP_TEMP_RE = re.compile(
)
def _notify_download_done(
db, user_id: int | None, title: str | None, job_id, *, kind: str
) -> None:
def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, kind: str) -> None:
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
silently while scheduler jobs and playlist-push already notified (R11 feedback symmetry). A
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
@@ -113,7 +112,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
def _make_progress_hook(job_id: int, asset_id: int):
state = {"meta_done": False}
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S)
def hook(d: dict) -> None:
@@ -247,7 +246,9 @@ def _make_pp_hook(job_id: int):
def hook(d: dict) -> None:
if d.get("status") in ("started", "processing"):
_set_job(job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None)
_set_job(
job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None
)
return hook
@@ -269,7 +270,9 @@ def _produced_file(info: dict, staging: Path) -> Path:
back to the largest file left in staging that is neither a thumbnail nor one of yt-dlp's own
working files (see `_YTDLP_TEMP_RE`)."""
req = (info.get("requested_downloads") or [{}])[0]
fp = req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
fp = (
req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
)
if fp:
produced = Path(str(fp))
if produced.is_file():
@@ -310,7 +313,9 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
last_exc: Exception | None = None
for i, clients in enumerate(client_sets):
opts = formats.build_ydl_opts(
spec, outtmpl, _make_progress_hook(job_id, asset_id),
spec,
outtmpl,
_make_progress_hook(job_id, asset_id),
extra=adapter.ydl_opts_extra(clients),
)
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
@@ -324,7 +329,12 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
except yt_dlp.utils.DownloadError as exc:
last_exc = exc
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
log.info(
"job %s: clients %s failed (%s); retrying with fallback",
job_id,
clients,
str(exc)[:70],
)
for p in list(staging.iterdir()): # drop partials before the retry
try:
p.unlink()
@@ -363,7 +373,9 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
dest = produced.with_name(produced.stem + ".browser.mp4")
action = "re-encode" if (reenc_v or reenc_a) else "remux"
log.info("job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4)
log.info(
"job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4
)
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
# re-encode still shows real progress instead of a frozen 0%.
@@ -436,8 +448,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
asset.width = int(info["width"]) if info.get("width") else None
asset.height = int(info["height"]) if info.get("height") else None
asset.container = ext
asset.vcodec = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
asset.vcodec = (
info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
)
asset.acodec = (
info.get("acodec") if info.get("acodec") not in (None, "none") else None
)
asset.title = meta.title
asset.uploader = meta.uploader
asset.uploader_url = adapter.uploader_url(info)
@@ -456,8 +472,13 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
job = db.get(DownloadJob, job_id)
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
owner_id = job.user_id if job else None
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None, "speed_bps": None,
"eta_s": None}
fields = {
"status": JobStatus.DONE,
"progress": 100,
"phase": None,
"speed_bps": None,
"eta_s": None,
}
if job and not job.display_name and meta.title:
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
fields["display_name"] = meta.title[:255]
@@ -484,8 +505,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
# with `error` (error also holds the asset → would re-open the double-release class).
service.finalize_if_running(
db, job_id, status=JobStatus.ERROR, error=msg,
error_code=dlerrors.classify(msg), phase=None,
db,
job_id,
status=JobStatus.ERROR,
error=msg,
error_code=dlerrors.classify(msg),
phase=None,
)
db.commit()
log.warning("job %s failed: %s", job_id, msg)
@@ -501,8 +526,12 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
asset.error = msg
# Atomic fail: only while still running (see the download error path).
service.finalize_if_running(
db, job_id, status=JobStatus.ERROR, error=msg,
error_code=dlerrors.classify(msg), phase=None,
db,
job_id,
status=JobStatus.ERROR,
error=msg,
error_code=dlerrors.classify(msg),
phase=None,
)
db.commit()
log.warning("edit job %s failed: %s", job_id, msg)
@@ -517,7 +546,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
user_id = job.user_id if job else None
src_ready = src is not None and src.status == "ready" and bool(src.rel_path)
src_rel = src.rel_path if src else None
src_w, src_h, src_dur = (src.width, src.height, src.duration_s) if src else (None, None, None)
src_w, src_h, src_dur = (
(src.width, src.height, src.duration_s) if src else (None, None, None)
)
if not src_ready or not src_rel:
_fail_edit(job_id, asset_id, "The source download is no longer available.")
@@ -541,7 +572,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
if edit_spec.get("segments"):
cmd, prep = editmod.build_concat_plan(src_path, dest_staging, edit_spec, out_ext, staging)
cmd, prep = editmod.build_concat_plan(
src_path, dest_staging, edit_spec, out_ext, staging
)
for p, content in prep.items():
p.write_text(content, encoding="utf-8")
else:
backend/app/downloads/quota.py
+1
View File
@@ -8,6 +8,7 @@ Limits come from a per-user DownloadQuota row if the admin set one, else the sys
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
"""
from dataclasses import dataclass
from sqlalchemy import func, select
backend/app/downloads/runner.py
+1
View File
@@ -10,6 +10,7 @@ Contract: the asset is already claimed ('downloading') by the parent. This proce
writes the terminal job/asset state itself (done / error, or a cooperative pause via the in-flight
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
"""
import logging
import sys
backend/app/downloads/service.py
+3 -1
View File
@@ -6,6 +6,7 @@ cache hit (asset already ready) — marks the job done instantly with no re-down
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
"""
from datetime import datetime, timezone
from sqlalchemy import func, select, update
@@ -166,7 +167,8 @@ def enqueue(
source_ref=source_ref,
profile_id=profile_id,
profile_snapshot=spec,
display_name=display_name or (asset.title[:255] if asset.status == "ready" and asset.title else None),
display_name=display_name
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
status="queued",
queue_pos=next_queue_pos(db),
)
backend/app/downloads/storage.py
+24 -7
View File
@@ -10,6 +10,7 @@ Plus a Kodi/Plex-readable `<basename>.nfo` (episodedetails) and `<basename>.jpg`
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
date-based — the community-standard mapping for YouTube-in-Plex.
"""
import re
import shutil
import unicodedata
@@ -36,9 +37,27 @@ _DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"}
# that accented names show in non-UTF-8 tools.
_FOLD_MAP = str.maketrans(
{
"ß": "ss", "ẞ": "SS", "ø": "o", "Ø": "O", "ł": "l", "Ł": "L", "đ": "d", "Đ": "D",
"ð": "d", "Ð": "D", "þ": "th", "Þ": "Th", "æ": "ae", "Æ": "AE", "œ": "oe", "Œ": "OE",
"ı": "i", "İ": "I", "·": " ", "–": "-", "—": "-",
"ß": "ss",
"ẞ": "SS",
"ø": "o",
"Ø": "O",
"ł": "l",
"Ł": "L",
"đ": "d",
"Đ": "D",
"ð": "d",
"Ð": "D",
"þ": "th",
"Þ": "Th",
"æ": "ae",
"Æ": "AE",
"œ": "oe",
"Œ": "OE",
"ı": "i",
"İ": "I",
"·": " ",
"–": "-",
"—": "-",
}
)
@@ -174,7 +193,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
f" <premiered>{aired}</premiered>",
f" <year>{year}</year>",
f" <plot>{escape(meta.description or '')}</plot>",
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
f' <uniqueid type="{escape(meta.uniqueid_type)}" default="true">{escape(meta.video_id)}</uniqueid>',
]
if meta.duration_s:
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
@@ -190,9 +209,7 @@ def _sidecar(media: Path, ext: str) -> Path:
return media.with_suffix(ext)
def write_sidecars(
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
) -> bool:
def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
media = abs_path(download_root, rel)
try:
backend/app/email.py
+6 -13
View File
@@ -4,6 +4,7 @@ Gmail SMTP + App Password by default. Deliberately fail-soft: if SMTP isn't conf
or a send errors, we log and return False so the caller can fall back to in-app
notifications — email is never load-bearing for the app's core flow.
"""
import logging
import smtplib
import ssl
@@ -77,8 +78,7 @@ def send_access_approved(email: str) -> bool:
"Open Siftlode and sign in, and your YouTube subscriptions feed will be ready:\n\n"
f"{settings.app_base}\n\n"
"If you weren't expecting this, you can ignore the message.\n\n"
"— Siftlode"
+ (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
"— Siftlode" + (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
)
return _send([email], "You're in — your Siftlode access is approved", body, reply_to=admin)
@@ -90,8 +90,7 @@ def send_access_denied(email: str) -> bool:
"Thanks for your interest in Siftlode. After review, your access request wasn't approved "
"this time.\n\n"
"If you think this was a mistake, you're welcome to reply and let us know.\n\n"
"— Siftlode"
+ (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
"— Siftlode" + (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
)
return _send([email], "About your Siftlode access request", body, reply_to=admin)
@@ -113,9 +112,7 @@ def send_account_unsuspended(to: str, operator: str | None) -> bool:
body = (
"Hi,\n\n"
"Your Siftlode account has been reinstated — the suspension was lifted and you can sign "
"in again."
+ contact
+ "\n\n— Siftlode"
"in again." + contact + "\n\n— Siftlode"
)
return _send([to], "Your Siftlode account has been reinstated", body, reply_to=operator)
@@ -130,9 +127,7 @@ def send_account_deleted(to: str, operator: str | None) -> bool:
"Hi,\n\n"
"Your Siftlode account and all associated data — subscriptions, tags, watch history, "
"playlists and settings — have been permanently deleted. Nothing is retained.\n\n"
"Thanks for having used Siftlode."
+ contact
+ "\n\n— Siftlode"
"Thanks for having used Siftlode." + contact + "\n\n— Siftlode"
)
return _send([to], "Your Siftlode account has been deleted", body, reply_to=operator)
@@ -147,9 +142,7 @@ def send_account_suspended(to: str, operator: str | None) -> bool:
body = (
"Hi,\n\n"
"A sign-in to your Siftlode account was just attempted, but the account is currently "
"suspended, so access was denied.\n\n"
+ contact
+ "— Siftlode"
"suspended, so access was denied.\n\n" + contact + "— Siftlode"
)
return _send([to], "Your Siftlode account is suspended", body, reply_to=operator)
backend/app/feedquery.py
+51 -27
View File
@@ -144,9 +144,7 @@ def filtered_query(
state = aliased(VideoState)
status_expr = func.coalesce(state.status, "new")
query = select(*feed_columns(user, state)).join(
Channel, Channel.id == Video.channel_id
)
query = select(*feed_columns(user, state)).join(Channel, Channel.id == Video.channel_id)
if scope == "all":
# Whole shared catalog; subscription is optional (only for priority sort).
@@ -181,9 +179,7 @@ def filtered_query(
# exactly. 'unwatched'/'all' include row-less 'new' videos, so they keep the outer join.
state_required = show in ("watched", "hidden", "in_progress")
join_state = query.join if state_required else query.outerjoin
query = join_state(
state, and_(state.video_id == Video.id, state.user_id == user.id)
)
query = join_state(state, and_(state.video_id == Video.id, state.user_id == user.id))
# Hide videos the maintenance job has flagged as currently unplayable (deleted/private/
# abandoned). They're either deleted after a grace period or unhidden if they recover;
@@ -197,9 +193,7 @@ def filtered_query(
# asking for scope=all + channel_ids (the channel is in this accessible set).
accessible_explore = (
select(Subscription.channel_id).where(Subscription.user_id == user.id)
).union(
select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id)
)
).union(select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id))
query = query.where(
or_(
Video.via_explore.is_(False),
@@ -244,13 +238,9 @@ def filtered_query(
query = query.where(Video.duration_seconds <= max_duration)
if max_age_days is not None:
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
query = query.where(
Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)
)
query = query.where(Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc))
if published_after is not None:
start = datetime.combine(
published_after, datetime.min.time(), tzinfo=timezone.utc
)
start = datetime.combine(published_after, datetime.min.time(), tzinfo=timezone.utc)
query = query.where(Video.published_at >= start)
if published_before is not None:
end = datetime.combine(
@@ -314,9 +304,7 @@ def filtered_query(
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
)
else:
sub = select(ChannelTag.channel_id).where(
ChannelTag.tag_id.in_(ids), visible
)
sub = select(ChannelTag.channel_id).where(ChannelTag.tag_id.in_(ids), visible)
query = query.where(Video.channel_id.in_(sub))
# Content type: Normal / Shorts / Live·Upcoming as a union of enabled types.
@@ -403,7 +391,9 @@ def feed_summary(db: Session, user: User, **params) -> dict:
if cat == exclude_cat:
continue
if tag_modes.get(cat, "or") == "and" and len(sel) > 1:
if not sel <= ch_tags: # conjunctive: channel must carry ALL this category's selected tags
if (
not sel <= ch_tags
): # conjunctive: channel must carry ALL this category's selected tags
return False
elif not (sel & ch_tags): # disjunctive: channel must carry ANY selected tag
return False
@@ -443,19 +433,52 @@ def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
"oldest": [{**pub, "asc": True}],
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
"duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}],
"duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}],
"duration_desc": [
{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}
],
"duration_asc": [
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
],
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
"title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}],
"subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}],
"subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}],
"title_desc": [
{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}
],
"subscribers": [
{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}
],
"subscribers_asc": [
{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}
],
# Your per-channel priority (set in the channel manager), newest first within a tier.
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
"priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub],
"priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub],
"priority": [
{
"expr": func.coalesce(Subscription.priority, 0),
"asc": False,
"nullable": False,
"kind": "num",
},
pub,
],
"priority_asc": [
{
"expr": func.coalesce(Subscription.priority, 0),
"asc": True,
"nullable": False,
"kind": "num",
},
pub,
],
# Deterministic per-seed shuffle: the same seed yields the same total order, so
# the cursor stays valid across pages of one shuffled run.
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
"shuffle": [
{
"expr": func.md5(func.concat(Video.id, str(seed))),
"asc": True,
"nullable": False,
"kind": "str",
}
],
}
# Relevance (search-result ranking) only exists when there's a query to rank against;
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
@@ -490,6 +513,7 @@ def keyset_predicate(keys: list[dict], cursor_id: str):
`newest` pages went 51.8k -> ~0.5k buffers). A NULLABLE key can't take that bound (the
NULL group sorts after every value and `col <= v` would drop it), so it keeps the pure
NULL-aware OR form."""
def eq(k):
return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
backend/app/main.py
+2 -6
View File
@@ -12,9 +12,7 @@ from fastapi import FastAPI, HTTPException
_siftlode_logger = logging.getLogger("siftlode")
if not _siftlode_logger.handlers:
_handler = logging.StreamHandler(sys.stdout)
_handler.setFormatter(
logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s")
)
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s"))
_siftlode_logger.addHandler(_handler)
_siftlode_logger.setLevel(logging.INFO)
_siftlode_logger.propagate = False
@@ -140,9 +138,7 @@ async def setup_gate(request, call_next):
with SessionLocal() as db:
configured = state.is_configured_cached(db)
if not configured:
return JSONResponse(
{"detail": "This instance isn't set up yet."}, status_code=503
)
return JSONResponse({"detail": "This instance isn't set up yet."}, status_code=503)
return await call_next(request)
backend/app/models.py
+101 -183
View File
@@ -26,9 +26,7 @@ class TimestampMixin:
"""Adds a server-defaulted `created_at`. Models that index created_at or order it
differently (QuotaEvent, Notification, Message) keep their own column on purpose."""
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class UpdatedAtMixin:
@@ -51,18 +49,14 @@ class User(Base, TimestampMixin):
password_hash: Mapped[str | None] = mapped_column(String(255))
# Email ownership proven (Google sign-in is implicitly verified; password registration
# verifies via a link). Login requires this AND is_active.
email_verified: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
email_verified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Admin-approved & enabled. A pending password registration starts inactive; admin
# approval activates it. A deactivated account can't log in.
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
# Admin-imposed access block, distinct from is_active (approval lifecycle). A suspended
# account can't sign in by any method and its live sessions are rejected; the user is told
# why and pointed at the operator's contact. The demo account is never suspendable.
is_suspended: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
is_suspended: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Server-side session revocation (SA4): a monotonic counter bumped on password reset, password
# change, or "log out everywhere". A signed session cookie records the epoch it was minted under;
# current_user rejects any cookie whose epoch is stale, so a stolen/copied cookie dies on those
@@ -95,9 +89,7 @@ class OAuthToken(Base, UpdatedAtMixin):
__tablename__ = "oauth_tokens"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), unique=True
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), unique=True)
# Refresh token encrypted at rest (Fernet). Access token is short-lived and low risk.
refresh_token_enc: Mapped[str | None] = mapped_column(String)
access_token: Mapped[str | None] = mapped_column(String)
@@ -115,9 +107,7 @@ class AuthToken(Base, TimestampMixin):
__tablename__ = "auth_tokens"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
kind: Mapped[str] = mapped_column(String(16)) # "verify" | "reset"
token_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
@@ -133,9 +123,7 @@ class Invite(Base):
id: Mapped[int] = mapped_column(primary_key=True)
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
# pending | approved | denied
status: Mapped[str] = mapped_column(
String(16), default="pending", server_default="pending"
)
status: Mapped[str] = mapped_column(String(16), default="pending", server_default="pending")
note: Mapped[str | None] = mapped_column(Text)
requested_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
@@ -189,9 +177,7 @@ class Channel(Base, TimestampMixin):
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
backfill_done: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# True when this channel only entered the catalog because a live YouTube search
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
# Lets the Library hide search-only "noise" by default.
@@ -218,9 +204,7 @@ class Subscription(Base, TimestampMixin):
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
@@ -231,9 +215,7 @@ class Subscription(Base, TimestampMixin):
# Per-user opt-in to full-history (deep) backfill of this channel. Default off so a
# new user's unique channels don't trigger a big shared-quota burst; the deep
# scheduler only picks up channels at least one user has requested.
deep_requested: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
deep_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
user: Mapped["User"] = relationship(back_populates="subscriptions")
@@ -255,9 +237,7 @@ class Video(Base, TimestampMixin):
# (channel_id, published_at DESC NULLS LAST, id) from migration 0067 leads with channel_id, so it
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) AND the per-channel feed
# sort. Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE")
)
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
description: Mapped[str | None] = mapped_column(Text)
@@ -289,27 +269,19 @@ class Video(Base, TimestampMixin):
# boolean btrees and replaced the useful ones with partial indexes over the target subset
# (via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — migration-defined, like the
# search_vector GIN.
is_short: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
is_short: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Whether the youtube.com/shorts/<id> probe has run for this video.
shorts_probed: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
shorts_probed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# True when this video first entered the catalog via a live YouTube search (not an
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
# Library by default; left False by the organic ingest paths.
via_search: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
via_search: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
# channel. Like via_search it's hidden from the shared Library by default, but it's
# additionally PER-USER private: a via_explore video is only visible to users who
# explored or subscribe to its channel (a gate in the feed query), so one user's
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
via_explore: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
via_explore: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# none | live | upcoming | premiere | was_live
live_status: Mapped[str] = mapped_column(
String(16), default="none", server_default="none", index=True
@@ -319,7 +291,9 @@ class Video(Base, TimestampMixin):
# maintenance/validation job to decide whether a video is playable anywhere.
embeddable: Mapped[bool | None] = mapped_column(Boolean)
privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private
upload_status: Mapped[str | None] = mapped_column(String(16)) # processed|failed|rejected|deleted
upload_status: Mapped[str | None] = mapped_column(
String(16)
) # processed|failed|rejected|deleted
# Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation
# (least-recently-checked first); unavailable_since marks a video as currently
# unplayable (hidden from the feed immediately, hard-deleted after a grace period);
@@ -327,12 +301,8 @@ class Video(Base, TimestampMixin):
# last_checked_at: no plain btree — migration 0069 (PERF-18) replaced it with the partial
# ix_videos_recheck_order (last_checked_at NULLS FIRST WHERE unavailable_since IS NULL) that the
# revalidate job's exact ORDER BY rides; unavailable_since is a partial WHERE IS NOT NULL (PERF-3).
last_checked_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True)
)
unavailable_since: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True)
)
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
@@ -366,9 +336,7 @@ class Tag(Base, TimestampMixin):
name: Mapped[str] = mapped_column(String(64))
color: Mapped[str | None] = mapped_column(String(16))
# language | topic | other
category: Mapped[str] = mapped_column(
String(16), default="other", server_default="other"
)
category: Mapped[str] = mapped_column(String(16), default="other", server_default="other")
class ChannelTag(Base):
@@ -376,21 +344,15 @@ class ChannelTag(Base):
regenerated freely; source='user' rows are never touched by the system."""
__tablename__ = "channel_tags"
__table_args__ = (
UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),
)
__table_args__ = (UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),)
id: Mapped[int] = mapped_column(primary_key=True)
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
tag_id: Mapped[int] = mapped_column(
ForeignKey("tags.id", ondelete="CASCADE"), index=True
)
tag_id: Mapped[int] = mapped_column(ForeignKey("tags.id", ondelete="CASCADE"), index=True)
# NULL = system/auto link visible to everyone; set = a specific user's link.
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
confidence: Mapped[float | None] = mapped_column(Float)
@@ -403,12 +365,8 @@ class VideoState(Base, UpdatedAtMixin):
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
video_id: Mapped[str] = mapped_column(
ForeignKey("videos.id", ondelete="CASCADE"), index=True
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
# new | watched | hidden ("saved" is now Watch-later playlist membership, not a status)
status: Mapped[str] = mapped_column(String(12), default="new", server_default="new")
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
@@ -433,12 +391,8 @@ class SearchFind(Base, TimestampMixin):
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
video_id: Mapped[str] = mapped_column(
ForeignKey("videos.id", ondelete="CASCADE"), index=True
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
class ExploredChannel(Base, TimestampMixin):
@@ -453,14 +407,10 @@ class ExploredChannel(Base, TimestampMixin):
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
__tablename__ = "explored_channels"
__table_args__ = (
UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),
)
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
@@ -472,14 +422,10 @@ class BlockedChannel(Base, TimestampMixin):
views — a personal "never show me this channel again" that doesn't affect other users."""
__tablename__ = "blocked_channels"
__table_args__ = (
UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),
)
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
@@ -544,16 +490,12 @@ class AppState(Base):
__tablename__ = "app_state"
id: Mapped[int] = mapped_column(primary_key=True, default=1)
sync_paused: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
sync_paused: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# First-run install wizard. `configured` is False on a fresh instance (the app runs in
# setup mode: only the wizard works) and flipped True when the wizard finishes. While
# unconfigured, a one-time setup token (only its SHA-256 hash stored) gates the wizard;
# the raw token + URL are printed to the container logs on each startup.
configured: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
configured: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
setup_token_hash: Mapped[str | None] = mapped_column(String(64))
@@ -599,14 +541,10 @@ class Playlist(Base, TimestampMixin, UpdatedAtMixin):
__tablename__ = "playlists"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
name: Mapped[str] = mapped_column(String(255))
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
source: Mapped[str] = mapped_column(
String(16), default="local", server_default="local"
)
source: Mapped[str] = mapped_column(String(16), default="local", server_default="local")
yt_playlist_id: Mapped[str | None] = mapped_column(String(64))
# True when the current items/name differ from `synced_fingerprint` (unpushed edits).
dirty: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
@@ -626,21 +564,13 @@ class PlaylistItem(Base):
"""A video's membership in a playlist, with its order within that playlist."""
__tablename__ = "playlist_items"
__table_args__ = (
UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),
)
__table_args__ = (UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
playlist_id: Mapped[int] = mapped_column(
ForeignKey("playlists.id", ondelete="CASCADE")
)
video_id: Mapped[str] = mapped_column(
ForeignKey("videos.id", ondelete="CASCADE"), index=True
)
playlist_id: Mapped[int] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
added_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
playlist: Mapped["Playlist"] = relationship(back_populates="items")
@@ -659,16 +589,12 @@ class SavedView(Base, TimestampMixin, UpdatedAtMixin):
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
source: Mapped[str] = mapped_column(String(16), default="feed", server_default="feed")
name: Mapped[str] = mapped_column(String(80))
filters: Mapped[dict] = mapped_column(JSON)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
is_default: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
is_default: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class Notification(Base):
@@ -686,19 +612,13 @@ class Notification(Base):
__tablename__ = "notifications"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
type: Mapped[str] = mapped_column(String(32), index=True) # e.g. "maintenance"
title: Mapped[str] = mapped_column(String(255))
body: Mapped[str | None] = mapped_column(Text)
data: Mapped[dict | None] = mapped_column(JSON)
read: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
dismissed: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
read: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false", index=True)
dismissed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), index=True
)
@@ -724,9 +644,7 @@ class Message(Base):
__tablename__ = "messages"
id: Mapped[int] = mapped_column(primary_key=True)
kind: Mapped[str] = mapped_column(
String(16), default="user", server_default="user"
)
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
# NULL for system messages (no human sender).
sender_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
@@ -786,13 +704,9 @@ class MediaAsset(Base, TimestampMixin):
container: Mapped[str | None] = mapped_column(String(16))
vcodec: Mapped[str | None] = mapped_column(String(32))
acodec: Mapped[str | None] = mapped_column(String(32))
nfo_written: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
gc_notified: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
title: Mapped[str | None] = mapped_column(Text)
uploader: Mapped[str | None] = mapped_column(String(255))
@@ -828,9 +742,7 @@ class DownloadProfile(Base, TimestampMixin):
) # NULL for a builtin preset
name: Mapped[str] = mapped_column(String(80))
spec: Mapped[dict] = mapped_column(JSON)
is_builtin: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
@@ -855,18 +767,14 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
__table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512))
# "download" (default) or "edit" (a per-user trim/crop derivative of another job).
job_kind: Mapped[str] = mapped_column(
String(16), default="download", server_default="download"
)
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
source_job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="SET NULL")
@@ -895,7 +803,9 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
eta_s: Mapped[int | None] = mapped_column(Integer)
phase: Mapped[str | None] = mapped_column(String(32))
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
error_code: Mapped[str | None] = mapped_column(String(24)) # category, see downloads.errors.classify
error_code: Mapped[str | None] = mapped_column(
String(24)
) # category, see downloads.errors.classify
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
@@ -904,20 +814,14 @@ class DownloadShare(Base, TimestampMixin):
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
__tablename__ = "download_shares"
__table_args__ = (
UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),
)
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE")
)
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
from_user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
to_user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
class DownloadQuota(Base, UpdatedAtMixin):
@@ -933,9 +837,7 @@ class DownloadQuota(Base, UpdatedAtMixin):
max_bytes: Mapped[int] = mapped_column(BigInteger)
max_concurrent: Mapped[int] = mapped_column(Integer)
max_jobs: Mapped[int] = mapped_column(Integer)
unlimited: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class DownloadLink(Base, TimestampMixin):
@@ -954,12 +856,8 @@ class DownloadLink(Base, TimestampMixin):
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
token: Mapped[str] = mapped_column(String(64), unique=True)
created_by: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
allow_download: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
password_hash: Mapped[str | None] = mapped_column(Text)
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
# password immediately invalidates any grants issued under the old one (instead of them living
@@ -1046,10 +944,14 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
year: Mapped[int | None] = mapped_column(Integer)
thumb_key: Mapped[str | None] = mapped_column(String(512)) # Plex thumb path (proxied, not stored)
thumb_key: Mapped[str | None] = mapped_column(
String(512)
) # Plex thumb path (proxied, not stored)
art_key: Mapped[str | None] = mapped_column(String(512))
child_count: Mapped[int | None] = mapped_column(Integer) # seasons
collection_keys: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # Plex collections this show is in
collection_keys: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # Plex collections this show is in
added_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
# Filterable / orderable metadata mirrored from the show section listing (parallels PlexItem).
rating: Mapped[float | None] = mapped_column(Float, index=True) # Plex audienceRating (~IMDb)
@@ -1062,15 +964,21 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
genres: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
directors: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
cast_names: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
people_text: Mapped[str | None] = mapped_column(Text) # cast+directors, folded into search_vector
people_text: Mapped[str | None] = mapped_column(
Text
) # cast+directors, folded into search_vector
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
imdb_rating: Mapped[float | None] = mapped_column(Float) # IMDb-specific score (vs `rating` = audienceRating)
imdb_rating: Mapped[float | None] = mapped_column(
Float
) # IMDb-specific score (vs `rating` = audienceRating)
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
tagline: Mapped[str | None] = mapped_column(Text)
full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}]
full_cast: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # ordered [{name, role, thumb}]
# --- TMDB enrichment (E5 S2): see the identical block on plex_items for the full rationale. Additive;
# served from the row; NULL until enriched. (`budget`/`revenue` stay NULL for shows — TV has none.) ---
tmdb_id: Mapped[int | None] = mapped_column(Integer)
@@ -1080,8 +988,12 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
budget: Mapped[int | None] = mapped_column(BigInteger)
revenue: Mapped[int | None] = mapped_column(BigInteger)
keywords: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
crew: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [{name, job, tmdb_id, thumb}]
studios: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB production companies [str]
crew: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # [{name, job, tmdb_id, thumb}]
studios: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # TMDB production companies [str]
production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
@@ -1198,10 +1110,14 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
imdb_rating: Mapped[float | None] = mapped_column(Float) # IMDb-specific score (vs `rating` = audienceRating)
imdb_rating: Mapped[float | None] = mapped_column(
Float
) # IMDb-specific score (vs `rating` = audienceRating)
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
tagline: Mapped[str | None] = mapped_column(Text)
full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}]
full_cast: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # ordered [{name, role, thumb}]
# --- TMDB enrichment (E5 S2): matched by the canonical `guid` ("tmdb:12345"), filled by the TMDB
# backfill job, SERVED FROM THIS ROW (never a live TMDB call). Additive — imdb_rating/rating
# above stay Plex-sourced; `tmdb_rating` is the distinct TMDb community score. NULL until enriched;
@@ -1214,12 +1130,18 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
budget: Mapped[int | None] = mapped_column(BigInteger) # USD (movies; TV has none)
revenue: Mapped[int | None] = mapped_column(BigInteger)
keywords: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
crew: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [{name, job, tmdb_id, thumb}]
studios: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB production companies [str]
crew: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # [{name, job, tmdb_id, thumb}]
studios: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # TMDB production companies [str]
production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
# [{tmdb_id, title, poster, media_type}] — resolved to library rows (by guid) at serve time.
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # TMDB backdrop file_paths [str] (crossfade pool, S8)
backdrops: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # TMDB backdrop file_paths [str] (crossfade pool, S8)
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
title_hu: Mapped[str | None] = mapped_column(String(512))
@@ -1273,7 +1195,9 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
place_of_birth: Mapped[str | None] = mapped_column(Text)
known_for_department: Mapped[str | None] = mapped_column(String(64))
also_known_as: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
known_for: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [{tmdb_id, media_type, title}]
known_for: Mapped[list | None] = mapped_column(
JSONB(none_as_null=True)
) # [{tmdb_id, media_type, title}]
tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
@@ -1309,9 +1233,7 @@ class PlexState(Base, UpdatedAtMixin):
__table_args__ = (UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
item_id: Mapped[int] = mapped_column(
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
)
@@ -1321,9 +1243,7 @@ class PlexState(Base, UpdatedAtMixin):
)
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
synced_to_plex: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
synced_to_plex: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class PlexLink(Base, UpdatedAtMixin):
@@ -1378,9 +1298,7 @@ class PlexPlaylistItem(Base):
__table_args__ = (UniqueConstraint("playlist_id", "item_id", name="uq_plex_playlist_item"),)
id: Mapped[int] = mapped_column(primary_key=True)
playlist_id: Mapped[int] = mapped_column(
ForeignKey("plex_playlists.id", ondelete="CASCADE")
)
playlist_id: Mapped[int] = mapped_column(ForeignKey("plex_playlists.id", ondelete="CASCADE"))
item_id: Mapped[int] = mapped_column(
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
)
backend/app/notifications.py
+2 -3
View File
@@ -5,6 +5,7 @@ survive reloads/devices and are produced server-side. First producer is the main
job. Unread rows never auto-expire; read rows are trimmed past a soft per-user cap so the
table can't grow without bound for a heavy user.
"""
from sqlalchemy import delete, select
from sqlalchemy.orm import Session
@@ -37,9 +38,7 @@ def create_notification(
transaction can pass commit=False and commit once themselves."""
if title is not None and len(title) > TITLE_MAX_LEN:
title = title[: TITLE_MAX_LEN - 1] + "…" # clamp to the column limit, add an ellipsis
notif = Notification(
user_id=user_id, type=type, title=title, body=body, data=data
)
notif = Notification(user_id=user_id, type=type, title=title, body=body, data=data)
db.add(notif)
if commit:
db.commit()
backend/app/plex/client.py
+11 -3
View File
@@ -5,6 +5,7 @@ this client — Siftlode plays the LOCAL physical file (see app.plex.paths). Ple
metadata, image proxying, and (P5) optional watch-state write-back. Requests JSON via the
``Accept: application/json`` header (Plex serves XML by default).
"""
import logging
from urllib.parse import quote
@@ -163,7 +164,9 @@ class PlexClient:
self._write("PUT", f"/library/collections/{collection_rating_key}/items?uri={uri}")
def remove_collection_item(self, collection_rating_key: str, item_rating_key: str) -> None:
self._write("DELETE", f"/library/collections/{collection_rating_key}/children/{item_rating_key}")
self._write(
"DELETE", f"/library/collections/{collection_rating_key}/children/{item_rating_key}"
)
def rename_collection(self, section_key: str, collection_rating_key: str, title: str) -> None:
self._write(
@@ -221,7 +224,11 @@ class PlexClient:
return self._get("/accounts").get("Account", []) or []
def watch_history(
self, min_viewed_at: int = 0, account_id: int | None = None, size: int = 500, max_pages: int = 40
self,
min_viewed_at: int = 0,
account_id: int | None = None,
size: int = 500,
max_pages: int = 40,
) -> list[dict]:
"""The watch-history rows (ratingKey / viewedAt / accountID / type) newer than `min_viewed_at`
(epoch seconds), newest first — works without Plex Pass. The efficient "what changed since T"
@@ -255,7 +262,8 @@ class PlexClient:
else:
log.warning(
"watch_history hit the %d-page scan cap (since=%s); reconcile will catch the tail",
max_pages, min_viewed_at,
max_pages,
min_viewed_at,
)
return out
backend/app/plex/detail.py
+1 -3
View File
@@ -38,9 +38,7 @@ def rich_detail(meta: dict) -> dict:
"imdb_url": f"https://www.imdb.com/title/{imdb_id}/" if imdb_id else None,
"content_rating": meta.get("contentRating"),
"genres": [g.get("tag") for g in (meta.get("Genre") or []) if g.get("tag")][:8],
"directors": [
d.get("tag") for d in (meta.get("Director") or []) if d.get("tag")
][:4],
"directors": [d.get("tag") for d in (meta.get("Director") or []) if d.get("tag")][:4],
"studio": meta.get("studio"),
"tagline": meta.get("tagline"),
"cast": people.cast_from_meta(meta),
backend/app/plex/imgcache.py
+1 -3
View File
@@ -63,6 +63,4 @@ def write(key: str, data: bytes, ct: str) -> None:
def response(data: bytes, ct: str) -> Response:
return Response(
content=data, media_type=ct, headers={"Cache-Control": "public, max-age=86400"}
)
return Response(content=data, media_type=ct, headers={"Cache-Control": "public, max-age=86400"})
backend/app/plex/lookups.py
+2 -3
View File
@@ -1,4 +1,5 @@
"""Small shared Plex row lookups, so the get-by-rating_key pattern lives in one place."""
from typing import TypeVar
from sqlalchemy import select
@@ -11,6 +12,4 @@ def by_rating_key(db: Session, model: type[_M], rating_key: str) -> _M | None:
"""The mirrored row for a Plex rating_key, or None. `model` is any of the rating_key-keyed Plex
tables (PlexItem / PlexShow / PlexSeason / PlexCollection); the return type is that model, so call
sites keep their `Model | None` static type."""
return db.execute(
select(model).where(model.rating_key == str(rating_key))
).scalars().first()
return db.execute(select(model).where(model.rating_key == str(rating_key))).scalars().first()
backend/app/plex/paths.py
+2 -1
View File
@@ -7,6 +7,7 @@ is admin-configured (sysconfig ``plex_path_map``) as newline/comma-separated ``p
pairs; the longest matching Plex prefix wins. An empty map means the paths are identical on both
sides (Plex and Siftlode see the same filesystem).
"""
from pathlib import Path
from sqlalchemy.orm import Session
@@ -33,7 +34,7 @@ def map_path(db: Session, plex_path: str) -> str:
"""Translate a Plex-side absolute path to the local mount path (no mapping = unchanged)."""
for plex_prefix, local_prefix in parse_map(sysconfig.get_str(db, "plex_path_map")):
if plex_path == plex_prefix or plex_path.startswith(plex_prefix.rstrip("/") + "/"):
rest = plex_path[len(plex_prefix):].lstrip("/")
rest = plex_path[len(plex_prefix) :].lstrip("/")
return str(Path(local_prefix) / rest) if rest else local_prefix
return plex_path
backend/app/plex/people.py
+3 -1
View File
@@ -19,7 +19,9 @@ PERSON_IMG_HOST = "https://metadata-static.plex.tv/"
def person_thumb_url(thumb: str | None) -> str | None:
"""Proxied /person-image URL for a Plex CDN headshot, or None when it's missing / off-CDN."""
t = str(thumb or "")
return f"/api/plex/person-image?u={quote(t, safe='')}" if t.startswith(PERSON_IMG_HOST) else None
return (
f"/api/plex/person-image?u={quote(t, safe='')}" if t.startswith(PERSON_IMG_HOST) else None
)
def cast_from_meta(meta: dict, limit: int = 24) -> list[dict]:
backend/app/plex/query.py
+5 -7
View File
@@ -104,9 +104,7 @@ def vector_match(model, lang: str, tsq):
by its English title (search_vector_hu has no title tokens for such a row); EN matches only the EN
vector. Ranking still uses `vector_col` (the HU vector in HU mode)."""
if lang == "hu":
return or_(
model.search_vector_hu.op("@@")(tsq), model.search_vector.op("@@")(tsq)
)
return or_(model.search_vector_hu.op("@@")(tsq), model.search_vector.op("@@")(tsq))
return model.search_vector.op("@@")(tsq)
@@ -195,7 +193,9 @@ def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
for kw in split_csv(p.get("keywords")): # AND — TMDB keyword chips from the detail page (E5 S5)
conds.append(model.keywords.contains([kw]))
csel = split_csv(p.get("collections"))
if csel: # OR (any) or AND (all) — a title can belong to several collections, mirroring genre_mode
if (
csel
): # OR (any) or AND (all) — a title can belong to several collections, mirroring genre_mode
cc = [model.collection_keys.contains([c]) for c in csel]
conds.append(and_(*cc) if p.get("collection_mode") == "all" else or_(*cc))
return conds
@@ -223,9 +223,7 @@ class LibraryFilters:
added_within: str | None = None
directors: str | None = None
actors: str | None = None
actor_mode: str = (
"any" # how multiple actors combine: any (OR, default) | all (AND)
)
actor_mode: str = "any" # how multiple actors combine: any (OR, default) | all (AND)
studios: str | None = None
keywords: str | None = None # TMDB keyword chips (E5 S5), CSV; language-neutral
collections: str | None = None # CSV of collection rating_keys
backend/app/plex/rollup.py
+4 -12
View File
@@ -25,9 +25,7 @@ def rollup(cards: list[dict]) -> dict:
total = len(cards)
watched = sum(1 for c in cards if c.get("status") == "watched")
inprog = [
c
for c in cards
if (c.get("position_seconds") or 0) > 0 and c.get("status") != "watched"
c for c in cards if (c.get("position_seconds") or 0) > 0 and c.get("status") != "watched"
]
resume = None
if inprog:
@@ -77,15 +75,11 @@ def agg_subquery(db: Session, user_id: int, include_resumed: bool = False):
def state_map(db: Session, user_id: int, show_ids: list[int]) -> dict[int, str]:
if not show_ids:
return {}
rows = db.execute(
agg_subquery(db, user_id).where(PlexItem.show_id.in_(show_ids))
).all()
rows = db.execute(agg_subquery(db, user_id).where(PlexItem.show_id.in_(show_ids))).all()
return {r.show_id: show_status(r.total, r.watched, r.inprog) for r in rows}
def item_state_map(
db: Session, user_id: int, item_ids: list[int]
) -> dict[int, PlexState]:
def item_state_map(db: Session, user_id: int, item_ids: list[int]) -> dict[int, PlexState]:
"""A user's PlexState rows for a set of leaf items (movies or episodes), keyed by item_id — the
leaf analogue of `state_map` (which rolls a show's episodes up to a status string). One batched
fetch, so the caller serializes cards without a per-card query. Empty in → empty dict (no `IN ()`)."""
@@ -94,8 +88,6 @@ def item_state_map(
return {
s.item_id: s
for s in db.execute(
select(PlexState).where(
PlexState.user_id == user_id, PlexState.item_id.in_(item_ids)
)
select(PlexState).where(PlexState.user_id == user_id, PlexState.item_id.in_(item_ids))
).scalars()
}
backend/app/plex/stream.py
+82 -23
View File
@@ -14,6 +14,7 @@ Sessions live in this (single) API process; a periodic reaper kills idle ones an
segments. The remux is CPU-light (video copy + a tiny audio transcode), so it runs fine on the
CPU-only prod host; only P3's full transcode is CPU-heavy.
"""
import logging
import re
import shutil
@@ -31,7 +32,11 @@ from app.plex import paths
log = logging.getLogger("siftlode.plex")
_HLS_ROOT = Path(settings.plex_hls_dir) if settings.plex_hls_dir else Path(settings.download_root) / ".plex-hls"
_HLS_ROOT = (
Path(settings.plex_hls_dir)
if settings.plex_hls_dir
else Path(settings.download_root) / ".plex-hls"
)
_SEG_SECONDS = 6
_SESSION_IDLE_S = 600 # reap a session with no access for this long
# A session directory is named `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}` (see start_session);
@@ -66,7 +71,9 @@ _sessions: dict[str, "HlsSession"] = {}
class HlsSession:
def __init__(self, key: str, directory: Path, proc: subprocess.Popen, start_s: float, entry: str):
def __init__(
self, key: str, directory: Path, proc: subprocess.Popen, start_s: float, entry: str
):
self.key = key
self.dir = directory
self.proc = proc
@@ -91,15 +98,21 @@ def _seek_opts(start_s: float) -> list[str]:
_HLS_TAIL = [
"-f", "hls",
"-hls_time", str(_SEG_SECONDS),
"-hls_list_size", "0",
"-f",
"hls",
"-hls_time",
str(_SEG_SECONDS),
"-hls_list_size",
"0",
# EVENT (not VOD): ffmpeg writes/appends the playlist AS segments complete, so playback can
# start immediately from this session's offset. VOD only writes the playlist at the end. The
# full seekbar comes from our known duration + the seek-restart model, not from the playlist.
"-hls_playlist_type", "event",
"-hls_segment_type", "mpegts",
"-hls_flags", "independent_segments+temp_file",
"-hls_playlist_type",
"event",
"-hls_segment_type",
"mpegts",
"-hls_flags",
"independent_segments+temp_file",
]
@@ -108,9 +121,21 @@ def _probe_audio_langs(src: Path) -> list[str]:
Used to build the multi-rendition var_stream_map. Empty on probe failure → single-audio path."""
try:
out = subprocess.run(
["ffprobe", "-v", "error", "-select_streams", "a", "-show_entries", "stream_tags=language",
"-of", "csv=p=0", str(src)],
capture_output=True, text=True, timeout=10,
[
"ffprobe",
"-v",
"error",
"-select_streams",
"a",
"-show_entries",
"stream_tags=language",
"-of",
"csv=p=0",
str(src),
],
capture_output=True,
text=True,
timeout=10,
)
return [ln.strip() for ln in out.stdout.splitlines()]
except (subprocess.SubprocessError, OSError):
@@ -118,8 +143,12 @@ def _probe_audio_langs(src: Path) -> list[str]:
def _ffmpeg_cmd(
src: Path, start_s: float, out_dir: Path, audio_ord: int | None,
aoff: float = 0.0, audio_langs: list[str] | None = None,
src: Path,
start_s: float,
out_dir: Path,
audio_ord: int | None,
aoff: float = 0.0,
audio_langs: list[str] | None = None,
) -> list[str]:
args = ["ffmpeg", "-nostdin", "-loglevel", "error"]
# Keep ORIGINAL timestamps on the output so the first segment carries the true absolute PTS of the
@@ -152,15 +181,23 @@ def _ffmpeg_cmd(
entry += ",default:yes"
var.append(entry)
args += _HLS_TAIL + [
"-master_pl_name", "master.m3u8",
"-var_stream_map", " ".join(var),
"-hls_segment_filename", str(out_dir / "seg_%v_%d.ts"), str(out_dir / "stream_%v.m3u8"),
"-master_pl_name",
"master.m3u8",
"-var_stream_map",
" ".join(var),
"-hls_segment_filename",
str(out_dir / "seg_%v_%d.ts"),
str(out_dir / "stream_%v.m3u8"),
]
else:
ao = audio_ord if audio_ord is not None else 0
args += ["-map", "0:v:0", "-map", f"{ai}:a:{ao}?", "-sn"]
args += ["-c:v", "copy", "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
args += _HLS_TAIL + ["-hls_segment_filename", str(out_dir / "seg_%d.ts"), str(out_dir / "index.m3u8")]
args += _HLS_TAIL + [
"-hls_segment_filename",
str(out_dir / "seg_%d.ts"),
str(out_dir / "index.m3u8"),
]
return args
@@ -170,9 +207,23 @@ def _probe_first_pts(seg: Path) -> float | None:
zero-bases the timeline to. Returns None on any failure (caller falls back to the requested offset)."""
try:
out = subprocess.run(
["ffprobe", "-v", "error", "-select_streams", "v:0", "-read_intervals", "%+#1",
"-show_entries", "packet=pts_time", "-of", "csv=p=0", str(seg)],
capture_output=True, text=True, timeout=10,
[
"ffprobe",
"-v",
"error",
"-select_streams",
"v:0",
"-read_intervals",
"%+#1",
"-show_entries",
"packet=pts_time",
"-of",
"csv=p=0",
str(seg),
],
capture_output=True,
text=True,
timeout=10,
)
for line in out.stdout.splitlines():
tok = line.strip().strip(",")
@@ -199,7 +250,9 @@ def _enforce_cap(cap: int) -> None:
# Caller holds _lock. Drop the least-recently-accessed sessions over the cap.
if len(_sessions) <= cap:
return
for key, s in sorted(_sessions.items(), key=lambda kv: kv[1].last_access)[: len(_sessions) - cap]:
for key, s in sorted(_sessions.items(), key=lambda kv: kv[1].last_access)[
: len(_sessions) - cap
]:
_kill(s)
_sessions.pop(key, None)
@@ -232,7 +285,9 @@ def start_session(
shutil.rmtree(directory, ignore_errors=True)
directory.mkdir(parents=True, exist_ok=True)
proc = subprocess.Popen(
_ffmpeg_cmd(src, start_s, directory, audio_ord, aoff, audio_langs if is_multi else None),
_ffmpeg_cmd(
src, start_s, directory, audio_ord, aoff, audio_langs if is_multi else None
),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
@@ -253,7 +308,11 @@ def start_session(
s.media_start_s = k
log.info(
"plex hls session start key=%s req=%.1f real=%.3f multi=%s audio=%s",
key, start_s, s.media_start_s, is_multi, audio_ord,
key,
start_s,
s.media_start_s,
is_multi,
audio_ord,
)
return s
backend/app/plex/subtitles.py
+2 -5
View File
@@ -26,9 +26,7 @@ BITMAP_SUBS = {
def srt_to_vtt(text: str) -> str:
"""Minimal SubRip→WebVTT: add the header and switch the ms separator (`,`→`.`). SRT cue-index
lines are valid in WebVTT, so they're left as-is."""
body = re.sub(
r"(\d\d:\d\d:\d\d),(\d\d\d)", r"\1.\2", text.replace("\r\n", "\n").lstrip("")
)
body = re.sub(r"(\d\d:\d\d:\d\d),(\d\d\d)", r"\1.\2", text.replace("\r\n", "\n").lstrip(""))
return "WEBVTT\n\n" + body.strip() + "\n"
@@ -114,8 +112,7 @@ def ffmpeg_to_vtt(src: str, sub_ord: int | None) -> str:
r = subprocess.run(cmd, capture_output=True)
if r.returncode != 0 or not r.stdout:
raise PlexError(
(r.stderr or b"").decode("utf-8", "replace")[:200]
or "ffmpeg subtitle convert failed"
(r.stderr or b"").decode("utf-8", "replace")[:200] or "ffmpeg subtitle convert failed"
)
return r.stdout.decode("utf-8", "replace")
backend/app/plex/sync.py
+51 -41
View File
@@ -14,6 +14,7 @@ mass-delete is the only real risk). Library-level reconcile (dropping a whole re
dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are fetched lazily on the
item-detail/player path (P2), not here.
"""
import logging
from datetime import datetime, timezone
@@ -154,8 +155,14 @@ def sync(db: Session) -> dict:
if not sysconfig.get_bool(db, "plex_enabled"):
return {"skipped": "disabled"}
stats = {
"libraries": 0, "movies": 0, "shows": 0, "seasons": 0, "episodes": 0,
"collections": 0, "overlaid": 0, "pruned": 0,
"libraries": 0,
"movies": 0,
"shows": 0,
"seasons": 0,
"episodes": 0,
"collections": 0,
"overlaid": 0,
"pruned": 0,
}
wanted = _enabled_section_keys(db)
overlay_map = _overlay_map(db) # {huKey: enKey}
@@ -163,9 +170,7 @@ def sync(db: Session) -> dict:
try:
with PlexClient(db) as plex:
sections = {
str(s.get("key")): s
for s in plex.sections()
if s.get("type") in ("movie", "show")
str(s.get("key")): s for s in plex.sections() if s.get("type") in ("movie", "show")
}
libs_by_key: dict[str, PlexLibrary] = {}
# Pass 1 — browsable (non-overlay) sections. HU-overlay sections are handled in pass 2 and
@@ -196,7 +201,8 @@ def sync(db: Session) -> dict:
if s is None or en_lib is None or s.get("type") != en_lib.kind:
log.warning(
"Plex overlay skipped: hu=%s en=%s (missing section/EN lib or kind mismatch)",
hu_key, en_key,
hu_key,
en_key,
)
continue
lib = _upsert_library(db, hu_key, s.get("title") or hu_key, s.get("type"))
@@ -237,12 +243,16 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
shared enrich_row_people). Oldest-id first, movies before shows. Stamps people_enriched_at only on
a SUCCESSFUL fetch, so a transient per-item failure retries next run rather than being lost.
Returns how many items were enriched this run."""
movies = db.execute(
select(PlexItem)
.where(PlexItem.kind == "movie", PlexItem.people_enriched_at.is_(None))
.order_by(PlexItem.id)
.limit(limit)
).scalars().all()
movies = (
db.execute(
select(PlexItem)
.where(PlexItem.kind == "movie", PlexItem.people_enriched_at.is_(None))
.order_by(PlexItem.id)
.limit(limit)
)
.scalars()
.all()
)
remaining = limit - len(movies)
shows = (
db.execute(
@@ -250,7 +260,9 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
.where(PlexShow.people_enriched_at.is_(None))
.order_by(PlexShow.id)
.limit(remaining)
).scalars().all()
)
.scalars()
.all()
if remaining > 0
else []
)
@@ -263,7 +275,9 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
log.debug("Plex people enrich fetch failed for %s: %s", row.rating_key, e)
continue # leave people_enriched_at NULL so it's retried on a later run
people.enrich_row_people(db, row, people.cast_from_meta(meta))
detail.apply_rich_detail(row, meta) # imdb/tagline/full_cast → served from the row, not live
detail.apply_rich_detail(
row, meta
) # imdb/tagline/full_cast → served from the row, not live
row.people_enriched_at = now
n += 1
db.commit()
@@ -271,9 +285,7 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
def _upsert_library(db: Session, key: str, title: str, kind: str) -> PlexLibrary:
lib = db.execute(
select(PlexLibrary).where(PlexLibrary.plex_key == key)
).scalars().first()
lib = db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == key)).scalars().first()
if lib is None:
lib = PlexLibrary(plex_key=key)
db.add(lib)
@@ -319,7 +331,7 @@ def _apply_facet_fields(row, meta: dict) -> None:
column names) from the cheap section listing: rating, content_rating, studio, release date,
genres/directors/cast, and the searchable people blob folded into search_vector at sync time."""
row.rating = _rating(meta)
row.content_rating = (meta.get("contentRating") or None)
row.content_rating = meta.get("contentRating") or None
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
row.genres = _tags(meta, "Genre")
@@ -363,7 +375,9 @@ def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
row.codec_video = mf.get("codec_video")
row.codec_audio = mf.get("codec_audio")
row.size_bytes = mf.get("size_bytes")
row.playable = classify_playable(mf.get("codec_video"), mf.get("codec_audio"), mf.get("container"))
row.playable = classify_playable(
mf.get("codec_video"), mf.get("codec_audio"), mf.get("container")
)
def _prune_stale(
@@ -411,9 +425,7 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
# 1) shows
shows = {
r.rating_key: r
for r in db.execute(
select(PlexShow).where(PlexShow.library_id == lib.id)
).scalars()
for r in db.execute(select(PlexShow).where(PlexShow.library_id == lib.id)).scalars()
}
shows_seen: set[str] = set()
shows_progress: dict = {}
@@ -427,7 +439,9 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
db.add(sh)
shows[rk] = sh
sh.library_id = lib.id
sh.guid = _canonical_guid(meta) or sh.guid # EN<->HU pairing key; keep prior if listing has none
sh.guid = (
_canonical_guid(meta) or sh.guid
) # EN<->HU pairing key; keep prior if listing has none
sh.title = (meta.get("title") or "").strip() or "Untitled"
sh.summary = meta.get("summary")
sh.year = meta.get("year")
@@ -495,7 +509,9 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
# Prune leaf-first (episodes -> seasons -> shows) so a stale row is deleted directly, before its
# parent's cascade would reach it — avoids deleting the same ORM object twice. Each level is
# independently guarded on a non-empty listing by _prune_stale.
_prune_stale(db, episodes, episodes_seen, stats, complete=episodes_progress.get("complete", False))
_prune_stale(
db, episodes, episodes_seen, stats, complete=episodes_progress.get("complete", False)
)
_prune_stale(db, seasons, seasons_seen, stats, complete=seasons_progress.get("complete", False))
_prune_stale(db, shows, shows_seen, stats, complete=shows_progress.get("complete", False))
@@ -521,9 +537,7 @@ def _sync_overlay(
item_type = _TYPE_MOVIE
else:
rows = db.execute(
select(PlexShow).where(
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
)
select(PlexShow).where(PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None))
).scalars()
item_type = _TYPE_SHOW
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
@@ -535,7 +549,9 @@ def _sync_overlay(
for group in by_guid.values():
for row in group:
row.title_hu = row.summary_hu = row.genres_hu = None
seen: set[str] = set() # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
seen: set[str] = (
set()
) # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
for meta in _paginate(plex, lib.plex_key, item_type):
g = _canonical_guid(meta)
targets = by_guid.get(g) if g else None
@@ -576,22 +592,18 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
full ~4-min library sync. Upserts the row (preserving `editable`) and reconciles ONLY this
collection's membership on the affected member rows."""
meta = plex.metadata(rk) or {}
col = _upsert_collection(
db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta
)
col = _upsert_collection(db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta)
Model = PlexItem if lib.kind == "movie" else PlexShow
new_members = {str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")}
current = db.execute(
select(Model).where(Model.collection_keys.contains([rk]))
).scalars().all()
new_members = {
str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")
}
current = db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars().all()
for row in current: # drop rk from rows that are no longer members
if row.rating_key not in new_members:
row.collection_keys = [k for k in (row.collection_keys or []) if k != rk] or None
to_add = new_members - {r.rating_key for r in current}
if to_add:
for row in db.execute(
select(Model).where(Model.rating_key.in_(to_add))
).scalars():
for row in db.execute(select(Model).where(Model.rating_key.in_(to_add))).scalars():
row.collection_keys = list(dict.fromkeys([*(row.collection_keys or []), rk]))
db.flush()
return col
@@ -600,9 +612,7 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
Model = PlexItem if lib.kind == "movie" else PlexShow
for row in db.execute(
select(Model).where(Model.collection_keys.contains([rk]))
).scalars():
for row in db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars():
row.collection_keys = [k for k in (row.collection_keys or []) if k != rk] or None
col = by_rating_key(db, PlexCollection, rk)
if col is not None:
backend/app/plex/tmdb.py
+21 -4
View File
@@ -9,6 +9,7 @@ Plex-sourced `imdb_rating` / `rating` are left untouched; `tmdb_rating` is the d
One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `apply_tmdb_detail` writes
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
"""
import logging
import time
from datetime import datetime, timezone
@@ -184,14 +185,24 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
for c in payload.get("created_by") or []:
if c.get("name"):
crew.append(
{"name": c["name"], "job": "Creator", "tmdb_id": c.get("id"), "profile_path": c.get("profile_path")}
{
"name": c["name"],
"job": "Creator",
"tmdb_id": c.get("id"),
"profile_path": c.get("profile_path"),
}
)
for c in credits.get("crew") or []:
# Movie crew entries carry `job`; TV aggregate_credits carry a `jobs` array per person.
job = (c.get("jobs") or [{}])[0].get("job") if is_tv else c.get("job")
if job in priority and c.get("name"):
crew.append(
{"name": c["name"], "job": job, "tmdb_id": c.get("id"), "profile_path": c.get("profile_path")}
{
"name": c["name"],
"job": job,
"tmdb_id": c.get("id"),
"profile_path": c.get("profile_path"),
}
)
# Dedup by (name, job) — a person can hold two distinct key roles (Coppola: Director + Writer) — then
# order by display priority so the lead roles come first, and cap.
@@ -235,7 +246,9 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
"revenue": (payload.get("revenue") or None) if not is_tv else None,
"keywords": keywords or None,
"crew": crew or None,
"studios": [c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")][:8]
"studios": [
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
][:8]
or None,
"production_countries": [
c.get("name") for c in (payload.get("production_countries") or []) if c.get("name")
@@ -274,7 +287,11 @@ def _link_people(db: Session, links: dict[str, int]) -> None:
db.execute(
stmt.on_conflict_do_update(
index_elements=[PlexPerson.name],
set_={"tmdb_person_id": func.coalesce(PlexPerson.tmdb_person_id, stmt.excluded.tmdb_person_id)},
set_={
"tmdb_person_id": func.coalesce(
PlexPerson.tmdb_person_id, stmt.excluded.tmdb_person_id
)
},
)
)
backend/app/plex/watch_sync.py
+57 -34
View File
@@ -13,6 +13,7 @@ exist only in Siftlode are left untouched (union, Plex-authoritative on the inte
Phases B (Siftlode→Plex push via scrobble/timeline) and C (incremental Plex→Siftlode via the watch
history + onDeck, last-write-wins) build on this and land in later ships.
"""
import logging
from datetime import datetime, timedelta, timezone
@@ -61,10 +62,7 @@ def _plex_watch_to_state(
def _rk_to_id(db: Session) -> dict[str, int]:
"""The rating_key → PlexItem.id lookup used to resolve a Plex leaf back to its mirrored row."""
return {
rk: iid
for rk, iid in db.execute(select(PlexItem.rating_key, PlexItem.id)).all()
}
return {rk: iid for rk, iid in db.execute(select(PlexItem.rating_key, PlexItem.id)).all()}
def _scan_plex_states(
@@ -165,9 +163,7 @@ def link_for_push(db: Session, user_id: int) -> PlexLink | None:
"""The active owner watch-push link for this user, or None when push is off / not the owner.
Push is gated on `initial_import_done` so we never push a half-known local state over Plex's
authoritative one before the one-time "Plex is master" import has reconciled them."""
link = db.execute(
select(PlexLink).where(PlexLink.user_id == user_id)
).scalars().first()
link = db.execute(select(PlexLink).where(PlexLink.user_id == user_id)).scalars().first()
if link and link.uses_admin and link.sync_enabled and link.initial_import_done:
return link
return None
@@ -222,11 +218,13 @@ def push_state_to_plex(
# Flag the row synced — but an "unwatched" via status=new deletes the row, so there may be
# nothing to flag; that's fine (unscrobble still happened). Skip the flag if the row was
# re-edited since we captured `expect_ts` (leave it dirty for its own push / the reconcile).
st = db.execute(
select(PlexState).where(
PlexState.user_id == user_id, PlexState.item_id == item_id
st = (
db.execute(
select(PlexState).where(PlexState.user_id == user_id, PlexState.item_id == item_id)
)
).scalars().first()
.scalars()
.first()
)
if st is not None and _row_matches(st, expect_ts):
st.synced_to_plex = True
db.commit()
@@ -270,7 +268,11 @@ def push_bulk_state_to_plex(user_id: int, changes: list[tuple[int, str, str]]) -
# One bad item mustn't abort the rest of the batch; leave its row dirty
# (synced_to_plex=False) for a later reconcile.
log.warning(
"Plex bulk push failed (user %s, item %s, %s): %s", user_id, item_id, action, e
"Plex bulk push failed (user %s, item %s, %s): %s",
user_id,
item_id,
action,
e,
)
continue
synced_ids.append(item_id)
@@ -301,13 +303,17 @@ _LWW_TOLERANCE_S = 90
def _active_sync_links(db: Session) -> list[PlexLink]:
"""Every link with two-way sync actually running (owner MVP = one row). Iterated uniformly so
P5b friend links slot in later without touching this loop."""
return db.execute(
select(PlexLink).where(
PlexLink.uses_admin.is_(True),
PlexLink.sync_enabled.is_(True),
PlexLink.initial_import_done.is_(True),
return (
db.execute(
select(PlexLink).where(
PlexLink.uses_admin.is_(True),
PlexLink.sync_enabled.is_(True),
PlexLink.initial_import_done.is_(True),
)
)
).scalars().all()
.scalars()
.all()
)
def _resolve_account_id(db: Session, plex: PlexClient, link: PlexLink) -> int:
@@ -353,11 +359,13 @@ def _pull_apply(
"""Bring one Plex-side positive signal into Siftlode under last-write-wins. Returns
"watched"/"in_progress" if it changed the row, else None. Never clears a row (un-watch is the
full reconcile's job) — this only pulls in watched/resume."""
st = db.execute(
select(PlexState).where(
PlexState.user_id == user_id, PlexState.item_id == item_id
st = (
db.execute(
select(PlexState).where(PlexState.user_id == user_id, PlexState.item_id == item_id)
)
).scalars().first()
.scalars()
.first()
)
if _same_state(st, status, pos):
if st is not None and not st.synced_to_plex:
st.synced_to_plex = True # already matches Plex → just settle the flag, no churn
@@ -370,7 +378,12 @@ def _pull_apply(
st = PlexState(user_id=user_id, item_id=item_id)
db.add(st)
if status == "watched":
st.status, st.position_seconds, st.watched_at, st.progress_updated_at = "watched", 0, plex_ts, plex_ts
st.status, st.position_seconds, st.watched_at, st.progress_updated_at = (
"watched",
0,
plex_ts,
plex_ts,
)
else:
st.status, st.position_seconds, st.progress_updated_at = "new", pos, plex_ts
st.synced_to_plex = True
@@ -390,13 +403,17 @@ def _repush_dirty(db: Session, plex: PlexClient, link: PlexLink) -> int:
select(PlexItem.id, PlexItem.rating_key, PlexItem.duration_s)
).all()
}
dirty = db.execute(
select(PlexState).where(
PlexState.user_id == link.user_id,
PlexState.synced_to_plex.is_(False),
PlexState.status != "hidden",
dirty = (
db.execute(
select(PlexState).where(
PlexState.user_id == link.user_id,
PlexState.synced_to_plex.is_(False),
PlexState.status != "hidden",
)
)
).scalars().all()
.scalars()
.all()
)
n = 0
for st in dirty:
info = info_by_id.get(st.item_id)
@@ -440,7 +457,9 @@ def run_plex_watch_sync(db: Session) -> dict:
iid = rk_to_id.get(str(row.get("ratingKey") or ""))
if iid is None:
continue
ch = _pull_apply(db, link.user_id, iid, "watched", 0, _epoch(row.get("viewedAt")))
ch = _pull_apply(
db, link.user_id, iid, "watched", 0, _epoch(row.get("viewedAt"))
)
if ch:
total[ch] += 1
@@ -451,7 +470,9 @@ def run_plex_watch_sync(db: Session) -> dict:
off = int(row.get("viewOffset") or 0) // 1000
if off < _PROGRESS_MIN_S:
continue
ch = _pull_apply(db, link.user_id, iid, "new", off, _epoch(row.get("lastViewedAt")))
ch = _pull_apply(
db, link.user_id, iid, "new", off, _epoch(row.get("lastViewedAt"))
)
if ch:
total[ch] += 1
@@ -495,9 +516,11 @@ def run_plex_watch_reconcile(db: Session) -> dict:
# Siftlode rows Plex no longer has a record for: clear the ones that were mirrored
# (Plex un-watched them); leave hidden + dirty (dirty gets re-pushed just below).
for st in db.execute(
select(PlexState).where(PlexState.user_id == link.user_id)
).scalars().all():
for st in (
db.execute(select(PlexState).where(PlexState.user_id == link.user_id))
.scalars()
.all()
):
if st.item_id in plex_states or st.status == "hidden":
continue
if st.synced_to_plex:
backend/app/progress.py
+1
View File
@@ -6,6 +6,7 @@ holds the sink, so the sync code stays unaware of the scheduler and concurrent j
its own thread/context) never cross wires. When no sink is bound (e.g. a manual sync endpoint),
`report` is a cheap no-op.
"""
import contextvars
from typing import Callable, Optional
backend/app/quota.py
+4 -3
View File
@@ -4,6 +4,7 @@ The whole app shares one daily budget (the API resets at midnight US Pacific tim
Steady-state work (RSS detection is free; enrichment of new videos is cheap) should
always be allowed; expensive backfill must yield to the remaining budget.
"""
import contextlib
import contextvars
from datetime import datetime, timezone
@@ -105,9 +106,9 @@ def units_used_today(db: Session) -> int:
# commit in a SEPARATE session (see record_usage), and a cached ORM row would otherwise hide
# their effect from a caller that already loaded today's row (measured()'s before/after diff).
# Under READ COMMITTED a fresh SELECT sees the other session's committed writes.
return db.scalar(
select(ApiQuotaUsage.units_used).where(ApiQuotaUsage.day == pacific_today())
) or 0
return (
db.scalar(select(ApiQuotaUsage.units_used).where(ApiQuotaUsage.day == pacific_today())) or 0
)
def remaining_today(db: Session) -> int:
backend/app/ratelimit.py
+1
View File
@@ -6,6 +6,7 @@ is fine for abuse throttling (not for anything that must survive a deploy). Not
processes — if we ever run multiple workers, swap the backing store for Redis behind the same
``allow()`` interface.
"""
import threading
import time
backend/app/realtime.py
+1
View File
@@ -11,6 +11,7 @@ they schedule the actual send onto the WebSocket event loop via run_coroutine_th
connect/disconnect/_push coroutines all run on that one loop, so the connection map is only ever
mutated from a single thread.
"""
import asyncio
import logging
from collections import defaultdict
backend/app/routes/_common.py
+1
View File
@@ -5,6 +5,7 @@ drift. Keeping them here (imported by the route modules) means the "fetch a row
and "fetch a row this user owns or 404" idioms have one definition, one status code, and
one existence-leak policy.
"""
from typing import Any, TypeVar
from fastapi import HTTPException
backend/app/routes/admin.py
+55 -32
View File
@@ -1,5 +1,6 @@
"""Admin-only onboarding: review access requests (Invites), approve/deny, manual add.
Also the demo-account admin surface: the demo email whitelist + a manual state reset."""
from datetime import datetime, timezone
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
@@ -144,7 +145,9 @@ def add_invite(
inv.status = "approved"
inv.decided_at = datetime.now(timezone.utc)
inv.decided_by = admin.email
audit.record(db, admin.id, AuditAction.INVITE_ADD, target_type="invite", target_id=inv.id, summary=email)
audit.record(
db, admin.id, AuditAction.INVITE_ADD, target_type="invite", target_id=inv.id, summary=email
)
db.commit()
_activate_user_by_email(db, email)
return _serialize(inv)
@@ -152,6 +155,7 @@ def add_invite(
# --- Users & roles --------------------------------------------------------------------
def _serialize_user(u: User) -> dict:
return {
"id": u.id,
@@ -208,9 +212,14 @@ def set_user_role(
changed = target.role != role
if changed:
audit.record(
db, admin.id, AuditAction.USER_ROLE_CHANGE,
target_type="user", target_id=target.id, summary=target.email,
before={"role": target.role}, after={"role": role},
db,
admin.id,
AuditAction.USER_ROLE_CHANGE,
target_type="user",
target_id=target.id,
summary=target.email,
before={"role": target.role},
after={"role": role},
)
target.role = role
db.commit()
@@ -254,16 +263,17 @@ def set_user_suspended(
was_suspended = target.is_suspended
if was_suspended != suspended:
audit.record(
db, admin.id,
db,
admin.id,
AuditAction.USER_SUSPEND if suspended else AuditAction.USER_UNSUSPEND,
target_type="user", target_id=target.id, summary=target.email,
target_type="user",
target_id=target.id,
summary=target.email,
)
target.is_suspended = suspended
db.commit()
if was_suspended and not suspended:
background.add_task(
email_mod.send_account_unsuspended, target.email, operator_contact()
)
background.add_task(email_mod.send_account_unsuspended, target.email, operator_contact())
return _serialize_user(target)
@@ -286,12 +296,20 @@ def admin_delete_user(
raise HTTPException(
status_code=400, detail="Delete your own account from Settings → Account."
)
if target.role == "admin" and not target.is_suspended and count_admins(db, active_only=True) <= 1:
if (
target.role == "admin"
and not target.is_suspended
and count_admins(db, active_only=True) <= 1
):
raise HTTPException(status_code=400, detail="Can't delete the last admin.")
# Record BEFORE the purge — the target row (and its email) is gone afterwards.
audit.record(
db, admin.id, AuditAction.USER_DELETE,
target_type="user", target_id=target.id, summary=target.email,
db,
admin.id,
AuditAction.USER_DELETE,
target_type="user",
target_id=target.id,
summary=target.email,
)
purge_user(db, target, background)
return {"deleted": user_id}
@@ -299,6 +317,7 @@ def admin_delete_user(
# --- Demo account: email whitelist + state reset -------------------------------------
def _serialize_demo(row: DemoWhitelist) -> dict:
return {
"id": row.id,
@@ -310,13 +329,9 @@ def _serialize_demo(row: DemoWhitelist) -> dict:
@router.get("/demo/whitelist")
def list_demo_whitelist(
_: User = Depends(admin_user), db: Session = Depends(get_db)
) -> list[dict]:
def list_demo_whitelist(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> list[dict]:
rows = (
db.execute(select(DemoWhitelist).order_by(DemoWhitelist.created_at.desc()))
.scalars()
.all()
db.execute(select(DemoWhitelist).order_by(DemoWhitelist.created_at.desc())).scalars().all()
)
return [_serialize_demo(r) for r in rows]
@@ -337,9 +352,7 @@ def add_demo_whitelist(
email = body.email.strip().lower()
if not valid_email(email):
raise HTTPException(status_code=400, detail="Enter a valid email address.")
row = db.execute(
select(DemoWhitelist).where(DemoWhitelist.email == email)
).scalar_one_or_none()
row = db.execute(select(DemoWhitelist).where(DemoWhitelist.email == email)).scalar_one_or_none()
if row is None:
row = DemoWhitelist(
email=email,
@@ -347,7 +360,9 @@ def add_demo_whitelist(
added_by=admin.email,
)
db.add(row)
audit.record(db, admin.id, AuditAction.DEMO_ADD, target_type="demo", target_id=email, summary=email)
audit.record(
db, admin.id, AuditAction.DEMO_ADD, target_type="demo", target_id=email, summary=email
)
db.commit()
return _serialize_demo(row)
@@ -360,29 +375,34 @@ def remove_demo_whitelist(
) -> dict:
row = db.get(DemoWhitelist, entry_id)
if row is not None:
audit.record(db, admin.id, AuditAction.DEMO_REMOVE, target_type="demo", target_id=row.email, summary=row.email)
audit.record(
db,
admin.id,
AuditAction.DEMO_REMOVE,
target_type="demo",
target_id=row.email,
summary=row.email,
)
db.delete(row)
db.commit()
return {"deleted": entry_id}
@router.post("/demo/reset")
def reset_demo(
admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
"""Manually clean up the shared demo sandbox back to a baseline. A scheduled job does the
same automatically (admin-tunable interval); this is the admin's on-demand button."""
demo = get_or_create_demo_user(db)
seeded = reset_demo_state(db, demo)
audit.record(db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded")
audit.record(
db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded"
)
db.commit()
return {"reset": True, "playlists_seeded": seeded}
@router.post("/purge-discovery")
def purge_discovery(
admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
def purge_discovery(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
"""Reclaim all un-kept discovery content NOW (ignoring the grace period): live-search result
videos nobody watched/saved/subscribed to, plus explored-but-unsubscribed channels. The
scheduled discovery-cleanup job does the same on its interval; this is the on-demand button."""
@@ -390,8 +410,11 @@ def purge_discovery(
result = {**purge_unkept_search(db, grace_days=0), **purge_unkept_explored(db)}
audit.record(
db, admin.id, AuditAction.DISCOVERY_PURGE,
summary=", ".join(f"{k}: {v}" for k, v in result.items()) or None, after=result,
db,
admin.id,
AuditAction.DISCOVERY_PURGE,
summary=", ".join(f"{k}: {v}" for k, v in result.items()) or None,
after=result,
)
db.commit()
return result
backend/app/routes/audit.py
+2 -3
View File
@@ -3,6 +3,7 @@
Admin-only. The client renders these rows in a DataTable (client-side sort/filter/paginate), so
this returns the most recent window of rows (capped) plus the true total, rather than paging
server-side — the log stays small in practice (state-changing runs + admin actions only)."""
from fastapi import APIRouter, Depends
from sqlalchemy import delete, func, select
from sqlalchemy.orm import Session
@@ -53,9 +54,7 @@ def list_audit(
actor_ids = {r.actor_id for r in rows if r.actor_id is not None}
actors: dict[int, str] = {}
if actor_ids:
actors = dict(
db.execute(select(User.id, User.email).where(User.id.in_(actor_ids))).all()
)
actors = dict(db.execute(select(User.id, User.email).where(User.id.in_(actor_ids))).all())
return {
"items": [_serialize(r, actors) for r in rows],
"total": total,
backend/app/routes/auth.py
+12 -38
View File
@@ -188,7 +188,6 @@ def _oauth_redirect_uri(request: Request) -> str:
return settings.oauth_redirect_url
# --- Request bodies (typed contracts; B-core-6) ---------------------------------------------
# Fields are `str | None` (not required) so the handlers keep their exact prior behaviour — a
# missing/null field reads as None and the existing `(... or "")` coercions still apply — while
@@ -277,9 +276,7 @@ async def callback(
email = (userinfo.get("email") or "").lower()
if link_uid is not None:
return _complete_link(
request, db, link_uid, link_explicit, sub, userinfo, token
)
return _complete_link(request, db, link_uid, link_explicit, sub, userinfo, token)
if not is_allowed(db, email):
log.warning("Login denied (not approved): %s", email or "<no email>")
@@ -337,9 +334,7 @@ async def callback(
# owns — that would hit the unique constraint and 500, wedging this user's sign-in (rare: a
# Google-side email change to a value that collides with a different local account).
if user.email != email:
clash = (
db.query(User).filter(User.email == email, User.id != user.id).one_or_none()
)
clash = db.query(User).filter(User.email == email, User.id != user.id).one_or_none()
if clash is None:
user.email = email
user.display_name = userinfo.get("name")
@@ -404,9 +399,7 @@ def _store_token(db: Session, user: User, token: dict) -> None:
# refresh token — a DB/backup/log leak otherwise hands out working Google API tokens.
tok.access_token = encrypt(token.get("access_token"))
expires_at = token.get("expires_at")
tok.expiry = (
datetime.fromtimestamp(expires_at, tz=timezone.utc) if expires_at else None
)
tok.expiry = datetime.fromtimestamp(expires_at, tz=timezone.utc) if expires_at else None
# Union (never narrow): a broader-or-equal exchange may still list only the newly-added scope.
tok.scopes = " ".join(sorted(old | new)) or BASE_SCOPES
db.add(tok)
@@ -436,9 +429,7 @@ def _complete_link(
user = db.get(User, link_uid)
if user is None or user.is_demo:
return RedirectResponse(url="/?link=error")
other = (
db.query(User).filter(User.google_sub == sub, User.id != user.id).one_or_none()
)
other = db.query(User).filter(User.google_sub == sub, User.id != user.id).one_or_none()
if other is not None:
return RedirectResponse(url="/?link=conflict")
if user.google_sub is not None and user.google_sub != sub:
@@ -624,10 +615,7 @@ def _register_account(email: str, password: str) -> None:
notice) for a genuinely new email; no-op for an already-registered one. Runs after the response
with its own DB session, so the request's timing never reveals whether the email already exists."""
with SessionLocal() as db:
if (
db.execute(select(User).where(User.email == email)).scalar_one_or_none()
is not None
):
if db.execute(select(User).where(User.email == email)).scalar_one_or_none() is not None:
return # already registered — nothing to do (the uniform "ok" was already returned)
# Without working SMTP we can't deliver a verification link, so email ownership can't gate
# sign-in. Admin approval stays the real gate (is_active=False); mark the account verified so
@@ -728,9 +716,7 @@ def password_login(
Once the password is proven correct, the owner gets a specific reason if their account is
still pending or suspended — that's not an enumeration leak (they already hold the password)."""
if not _login_limiter.allow(_client_ip(request)):
raise HTTPException(
status_code=429, detail="Too many attempts. Try again shortly."
)
raise HTTPException(status_code=429, detail="Too many attempts. Try again shortly.")
email = (body.email or "").strip().lower()
password = body.password or ""
user = db.execute(select(User).where(User.email == email)).scalar_one_or_none()
@@ -752,9 +738,7 @@ def password_login(
status_code=403, detail="Please verify your email first (check your inbox)."
)
if not user.is_active:
raise HTTPException(
status_code=403, detail="Your account is awaiting admin approval."
)
raise HTTPException(status_code=403, detail="Your account is awaiting admin approval.")
_establish_session(request, user)
log.info("Password login: %s (id=%s, role=%s)", email, user.id, user.role)
return {"ok": True}
@@ -803,9 +787,7 @@ def password_reset_confirm(body: TokenPasswordIn, db: Session = Depends(get_db))
)
user = _consume_token(db, token, "reset")
if user is None:
raise HTTPException(
status_code=400, detail="This reset link is invalid or has expired."
)
raise HTTPException(status_code=400, detail="This reset link is invalid or has expired.")
user.password_hash = hash_password(new_password)
# A valid reset link proves control of the mailbox, so completing a reset also verifies the
# email — this rescues an account that registered but never clicked the verification link (it
@@ -863,9 +845,7 @@ def logout_others(
are rejected, then re-stamp THIS cookie so the caller stays signed in. For "I left myself logged
in somewhere" / a suspected session compromise. The demo account (shared) is excluded."""
if user.is_demo:
raise HTTPException(
status_code=403, detail="Not available in the demo account."
)
raise HTTPException(status_code=403, detail="Not available in the demo account.")
# Commit the bump BEFORE re-stamping this cookie, so a failed commit can't strand the current
# session at a newer epoch than the DB (see set_password).
bump_session_epoch(user)
@@ -912,9 +892,7 @@ def set_password(
account) needs no current password — the session already proves identity. Changing an
existing one requires the current password, so a hijacked session can't silently rotate it."""
if user.is_demo:
raise HTTPException(
status_code=403, detail="Not available in the demo account."
)
raise HTTPException(status_code=403, detail="Not available in the demo account.")
new_password = body.password or ""
if len(new_password) < PASSWORD_MIN_LEN:
raise HTTPException(
@@ -922,12 +900,8 @@ def set_password(
detail=f"Password must be at least {PASSWORD_MIN_LEN} characters.",
)
if user.password_hash:
if not verify_password(
body.current_password or "", user.password_hash
):
raise HTTPException(
status_code=403, detail="Current password is incorrect."
)
if not verify_password(body.current_password or "", user.password_hash):
raise HTTPException(status_code=403, detail="Current password is incorrect.")
user.password_hash = hash_password(new_password)
# SA4: a password change logs out every OTHER session; re-stamp this cookie at the new epoch so
# the person making the change stays signed in here. Commit BEFORE re-stamping so a failed commit
backend/app/routes/channels.py
+55 -35
View File
@@ -1,6 +1,7 @@
"""Channel manager: the user's subscriptions with their personal overrides (priority,
hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side writes
(subscribe / unsubscribe) that require the optional write scope."""
import logging
from datetime import datetime, timezone
@@ -38,9 +39,7 @@ log = logging.getLogger("siftlode.api")
@router.get("")
def list_channels(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> list[dict]:
def list_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
rows = db.execute(
select(Subscription, Channel)
.join(Channel, Channel.id == Subscription.channel_id)
@@ -91,9 +90,7 @@ def list_channels(
# The user's personal tag links, grouped by channel.
tags_by_channel: dict[str, list[int]] = {}
for cid, tag_id in db.execute(
select(ChannelTag.channel_id, ChannelTag.tag_id).where(
ChannelTag.user_id == user.id
)
select(ChannelTag.channel_id, ChannelTag.tag_id).where(ChannelTag.user_id == user.id)
).all():
tags_by_channel.setdefault(cid, []).append(tag_id)
@@ -160,7 +157,10 @@ def discover_channels(
# to yourself. Lazy + cached; needs OAuth, so demo / token-less users just skip it.
if user.yt_channel_id is None and user.token is not None and not user.is_demo:
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt:
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
YouTubeClient(db, user) as yt,
):
own = yt.get_my_channel_id()
if own:
user.yt_channel_id = own
@@ -178,7 +178,10 @@ def discover_channels(
# Skip the quota-spending enrich for the demo account (it just sees the stub metadata).
if need and not user.is_demo:
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt:
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
YouTubeClient(db, user) as yt,
):
apply_channel_details(db, yt.get_channels(need))
db.commit()
# Match _discovery_rows' ORDER BY exactly: video-count DESC, then title (NULLs LAST,
@@ -285,7 +288,10 @@ def channel_detail(
raise HTTPException(status_code=404, detail="Unknown channel")
if channel.published_at is None and not user.is_demo:
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt:
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
YouTubeClient(db, user) as yt,
):
apply_channel_details(db, yt.get_channels([channel_id]))
db.commit()
channel = db.get(Channel, channel_id)
@@ -298,11 +304,14 @@ def channel_detail(
)
).scalar_one_or_none()
subscribed = sub is not None
explored = db.execute(
select(ExploredChannel.id).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id
)
).first() is not None
explored = (
db.execute(
select(ExploredChannel.id).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id
)
).first()
is not None
)
blocked = _is_blocked(db, user, channel_id)
# Stored total + the Normal/Shorts/Live split in one grouped pass (same filters as the manager
# list), so the channel page can show the per-type breakdown R14 moved off the table.
@@ -382,7 +391,10 @@ def explore_channel(
db.commit()
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE), YouTubeClient(db, user) as yt:
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE),
YouTubeClient(db, user) as yt,
):
# Enrich for the About panel + the uploads playlist id (needed to ingest videos).
if (
channel.details_synced_at is None
@@ -394,9 +406,7 @@ def explore_channel(
channel = db.get(Channel, channel_id)
result = explore_ingest_page(db, yt, channel, page_token)
except YouTubeError as exc:
raise HTTPException(
status_code=422, detail=f"YouTube couldn't load this channel: {exc}"
)
raise HTTPException(status_code=422, detail=f"YouTube couldn't load this channel: {exc}")
return {
"channel_id": channel_id,
"ingested": result["ingested"],
@@ -539,7 +549,10 @@ def subscribe(
).scalar_one_or_none()
if existing is not None:
raise HTTPException(status_code=409, detail="Already subscribed to this channel.")
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE), YouTubeClient(db, user) as yt:
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE),
YouTubeClient(db, user) as yt,
):
try:
yt_sub_id = yt.insert_subscription(channel_id)
except YouTubeError as exc:
@@ -553,7 +566,9 @@ def subscribe(
if "already exists" in msg or "duplicate" in msg:
yt_sub_id = ""
else:
raise HTTPException(status_code=422, detail=f"YouTube couldn't subscribe you: {exc}")
raise HTTPException(
status_code=422, detail=f"YouTube couldn't subscribe you: {exc}"
)
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
# render. Keep the client open (we're still inside the `with`); a metadata failure here
@@ -577,7 +592,9 @@ def subscribe(
if channel.from_explore:
channel.from_explore = False
db.execute(
update(Video).where(Video.channel_id == channel_id, Video.via_explore.is_(True)).values(via_explore=False)
update(Video)
.where(Video.channel_id == channel_id, Video.via_explore.is_(True))
.values(via_explore=False)
)
db.commit()
return {"subscribed": channel_id}
@@ -599,7 +616,10 @@ def unsubscribe(
detail="No YouTube subscription id on record — sync subscriptions first.",
)
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_UNSUBSCRIBE), YouTubeClient(db, user) as yt:
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_UNSUBSCRIBE),
YouTubeClient(db, user) as yt,
):
yt.delete_subscription(sub.yt_subscription_id)
except YouTubeError as exc:
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
@@ -611,16 +631,18 @@ def unsubscribe(
@router.get("/blocked/list")
def list_blocked(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> list[dict]:
def list_blocked(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
"""The user's blocked channels (videos from these are excluded from their search/feed/explore)."""
rows = db.execute(
select(Channel)
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
.where(BlockedChannel.user_id == user.id)
.order_by(func.lower(Channel.title))
).scalars().all()
rows = (
db.execute(
select(Channel)
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
.where(BlockedChannel.user_id == user.id)
.order_by(func.lower(Channel.title))
)
.scalars()
.all()
)
return [
{"id": c.id, "title": c.title, "handle": c.handle, "thumbnail_url": c.thumbnail_url}
for c in rows
@@ -643,8 +665,7 @@ def block_channel(
# Stop any active exploration of it so it can be cleaned up.
db.execute(
ExploredChannel.__table__.delete().where(
(ExploredChannel.user_id == user.id)
& (ExploredChannel.channel_id == channel_id)
(ExploredChannel.user_id == user.id) & (ExploredChannel.channel_id == channel_id)
)
)
db.commit()
@@ -659,8 +680,7 @@ def unblock_channel(
) -> dict:
db.execute(
BlockedChannel.__table__.delete().where(
(BlockedChannel.user_id == user.id)
& (BlockedChannel.channel_id == channel_id)
(BlockedChannel.user_id == user.id) & (BlockedChannel.channel_id == channel_id)
)
)
db.commit()
backend/app/routes/config.py
+11 -2
View File
@@ -1,5 +1,6 @@
"""Admin Configuration page API: read/edit the DB-overridable operational settings defined in
app.sysconfig. Secret values (e.g. SMTP password) are write-only — never returned."""
import re
from typing import Any
@@ -68,7 +69,11 @@ def set_config(
new = None if s.secret else sysconfig.get(db, key)
if s.secret or new != old:
audit.record(
db, admin.id, AuditAction.CONFIG_SET, target_type="config", target_id=key,
db,
admin.id,
AuditAction.CONFIG_SET,
target_type="config",
target_id=key,
summary=f"{key} = (secret updated)" if s.secret else f"{key} = {_redact(new)}",
before=None if s.secret else {"value": _redact(old)},
after=None if s.secret else {"value": _redact(new)},
@@ -89,7 +94,11 @@ def reset_config(
old = None if s.secret else sysconfig.get(db, key)
sysconfig.reset(db, key)
audit.record(
db, admin.id, AuditAction.CONFIG_RESET, target_type="config", target_id=key,
db,
admin.id,
AuditAction.CONFIG_RESET,
target_type="config",
target_id=key,
summary=f"{key} → default",
before=None if s.secret else {"value": _redact(old)},
)
backend/app/routes/downloads.py
+27 -30
View File
@@ -5,6 +5,7 @@ real identity). Admin routes add `admin_user`. Live job state is polled by the f
worker is a separate process); this module never downloads — it enqueues, manages job state,
serves finished files (range-aware, with the user's custom display name), and shares them.
"""
import re
from datetime import datetime, timedelta, timezone
from urllib.parse import parse_qs, urlparse
@@ -48,6 +49,7 @@ _MAX_LIST = 500
# --- source + serialization ----------------------------------------------------------------
def resolve_source(raw: str) -> tuple[str, str]:
"""Map a user-supplied id/URL to (source_kind, source_ref). YouTube is normalized to its
11-char id (so the cache dedups across watch/youtu.be/shorts URLs); anything else is kept
@@ -151,6 +153,7 @@ def _own_job(db: Session, user: User, job_id: int) -> DownloadJob:
# --- profiles ------------------------------------------------------------------------------
def _serialize_profile(p: DownloadProfile) -> dict:
return {
"id": p.id,
@@ -162,17 +165,14 @@ def _serialize_profile(p: DownloadProfile) -> dict:
@router.get("/profiles")
def list_profiles(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> list[dict]:
def list_profiles(user: User = Depends(require_human), db: Session = Depends(get_db)) -> list[dict]:
rows = (
db.execute(
select(DownloadProfile)
.where(
(DownloadProfile.is_builtin.is_(True))
| (DownloadProfile.user_id == user.id)
.where((DownloadProfile.is_builtin.is_(True)) | (DownloadProfile.user_id == user.id))
.order_by(
DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id
)
.order_by(DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id)
)
.scalars()
.all()
@@ -272,6 +272,7 @@ def _resolve_spec(
# --- enqueue + list + manage ---------------------------------------------------------------
class EnqueueDownloadIn(BaseModel):
source: str = ""
profile_id: int | None = None
@@ -287,9 +288,7 @@ def enqueue_download(
spec, profile_id = _resolve_spec(db, user, body.profile_id, body.spec)
display_name = (body.display_name or "").strip() or None
try:
job = service.enqueue(
db, user.id, source_kind, source_ref, spec, profile_id, display_name
)
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_detail(e))
return _serialize_job(db, job)
@@ -358,9 +357,7 @@ _INDEX_RANK = {"done": 4, "running": 3, "paused": 2, "queued": 1}
@router.get("/index")
def my_download_index(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
def my_download_index(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
"""source_ref -> best active status, for the feed's per-card "downloaded / queued" badge.
Small + cheap so the feed can poll it without loading the full job list."""
rows = db.execute(
@@ -515,6 +512,7 @@ def delete_download(
# --- file download (range-aware, custom display name) --------------------------------------
def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
job = db.get(DownloadJob, job_id)
if job is None:
@@ -558,6 +556,7 @@ def download_file(
# --- editor filmstrip (scrub track) --------------------------------------------------------
@router.get("/{job_id}/storyboard")
def storyboard_meta(
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
@@ -585,9 +584,7 @@ def storyboard_meta(
@router.get("/{job_id}/poster.jpg")
def poster_image(
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
):
def poster_image(job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)):
"""The locally-generated poster frame for a download whose source had no thumbnail."""
job = _accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
@@ -615,6 +612,7 @@ def storyboard_image(
# --- sharing -------------------------------------------------------------------------------
class ShareDownloadIn(BaseModel):
email: str = ""
@@ -647,9 +645,7 @@ def share_download(
)
).first()
if exists is None:
db.add(
DownloadShare(job_id=job_id, from_user_id=user.id, to_user_id=recipient.id)
)
db.add(DownloadShare(job_id=job_id, from_user_id=user.id, to_user_id=recipient.id))
from app.notifications import create_notification
create_notification(
@@ -708,6 +704,7 @@ def remove_shared_with_me(
# --- recipients (for the internal "share with a user" picker) ------------------------------
@router.get("/recipients")
def share_recipients(
user: User = Depends(require_human), db: Session = Depends(get_db)
@@ -728,6 +725,7 @@ def share_recipients(
# --- public watch links (share by link) ----------------------------------------------------
def _own_link(db: Session, user: User, link_id: int) -> DownloadLink:
link = db.get(DownloadLink, link_id)
if link is None:
@@ -753,7 +751,9 @@ def list_links(
_own_job(db, user, job_id)
rows = (
db.execute(
select(DownloadLink).where(DownloadLink.job_id == job_id).order_by(DownloadLink.id.desc())
select(DownloadLink)
.where(DownloadLink.job_id == job_id)
.order_by(DownloadLink.id.desc())
)
.scalars()
.all()
@@ -833,13 +833,16 @@ def revoke_link(
# --- admin ---------------------------------------------------------------------------------
@admin_router.get("")
def admin_list_downloads(
admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> list[dict]:
jobs = (
db.execute(
select(DownloadJob).order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()).limit(_MAX_LIST)
select(DownloadJob)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
.limit(_MAX_LIST)
)
.scalars()
.all()
@@ -847,9 +850,7 @@ def admin_list_downloads(
assets = _assets_for(db, jobs)
emails = {
u.id: u.email
for u in db.execute(
select(User).where(User.id.in_({j.user_id for j in jobs}))
).scalars()
for u in db.execute(select(User).where(User.id.in_({j.user_id for j in jobs}))).scalars()
}
out = []
for j in jobs:
@@ -860,9 +861,7 @@ def admin_list_downloads(
@admin_router.get("/storage")
def admin_storage(
admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
ready = db.execute(
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
MediaAsset.status == "ready"
@@ -876,9 +875,7 @@ def admin_storage(
).all()
emails = {
u.id: u.email
for u in db.execute(
select(User).where(User.id.in_([uid for uid, _ in per_user]))
).scalars()
for u in db.execute(select(User).where(User.id.in_([uid for uid, _ in per_user]))).scalars()
}
return {
"ready_files": ready[0],
backend/app/routes/feed.py
+7 -10
View File
@@ -228,9 +228,7 @@ def set_video_state(
# off); otherwise drop it. The in-app player fires this alongside a progress
# checkpoint that may DELETE the same row, so tolerate it vanishing under us.
row = db.execute(
select(VideoState).where(
VideoState.user_id == user.id, VideoState.video_id == video_id
)
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
).scalar_one_or_none()
if row is not None:
if row.position_seconds:
@@ -276,9 +274,7 @@ def clear_video_state(
watch position and watched_at), as if it had never been opened. Saved/playlist membership
lives elsewhere and is untouched."""
row = db.execute(
select(VideoState).where(
VideoState.user_id == user.id, VideoState.video_id == video_id
)
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
).scalar_one_or_none()
if row is not None:
db.delete(row)
@@ -321,9 +317,7 @@ def set_video_progress(
keep = position >= PROGRESS_MIN_SECONDS and not near_end
row = db.execute(
select(VideoState).where(
VideoState.user_id == user.id, VideoState.video_id == video_id
)
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
).scalar_one_or_none()
if not keep:
@@ -385,7 +379,10 @@ def get_video_detail(
raise HTTPException(status_code=404, detail="Unknown video")
try:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP), YouTubeClient(db, user) as yt:
with (
quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
YouTubeClient(db, user) as yt,
):
items = yt.get_videos([video_id])
except YouTubeError as exc:
raise HTTPException(status_code=422, detail=f"YouTube lookup failed: {exc}")
backend/app/routes/me.py
+1 -5
View File
@@ -100,11 +100,7 @@ def get_me(
pending_invites = 0
if user.role == "admin":
pending_invites = (
db.scalar(
select(func.count())
.select_from(Invite)
.where(Invite.status == "pending")
)
db.scalar(select(func.count()).select_from(Invite).where(Invite.status == "pending"))
or 0
)
return {
backend/app/routes/messages.py
+15 -20
View File
@@ -13,6 +13,7 @@ Every endpoint is gated by `require_human` (the shared demo account can't messag
(who messaged whom, when, read state) is NOT encrypted — standard for E2EE and needed for
routing and unread counts.
"""
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect
@@ -103,17 +104,13 @@ def _serialize_msg(m: Message) -> dict:
}
def ensure_welcome(db: Session, user: User) -> None:
"""Create the one-time system welcome message for a (human) user the first time it's needed.
Idempotent: guarded on whether any system message already exists for them."""
if user.is_demo:
return
exists = db.scalar(
select(Message.id)
.where(Message.recipient_id == user.id, Message.kind == "system")
.limit(1)
select(Message.id).where(Message.recipient_id == user.id, Message.kind == "system").limit(1)
)
if exists:
return
@@ -128,6 +125,7 @@ def ensure_welcome(db: Session, user: User) -> None:
# --- E2EE key directory / blob store -----------------------------------------
@router.get("/keys/me")
def my_key(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
"""My own key bundle. `configured` says whether secure messaging is set up; if so the
@@ -234,6 +232,7 @@ def public_key(
# --- directory + conversations -----------------------------------------------
@router.get("/users")
def list_recipients(
user: User = Depends(require_human), db: Session = Depends(get_db)
@@ -254,9 +253,7 @@ def list_recipients(
@router.get("/conversations")
def list_conversations(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
def list_conversations(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
"""One entry per conversation: each user partner (E2EE) plus the system "Siftlode" thread,
with the latest message + my unread count, newest first. Lazily creates the welcome on the
first open. User-message previews are ciphertext — the client decrypts them."""
@@ -298,8 +295,7 @@ def list_conversations(
partners = {}
if user_ids:
partners = {
u.id: u
for u in db.execute(select(User).where(User.id.in_(user_ids))).scalars().all()
u.id: u for u in db.execute(select(User).where(User.id.in_(user_ids))).scalars().all()
}
items = []
for m in latest:
@@ -311,7 +307,11 @@ def list_conversations(
else:
continue
items.append(
{"partner": partner, "last_message": _serialize_msg(m), "unread": int(unread.get(pid, 0))}
{
"partner": partner,
"last_message": _serialize_msg(m),
"unread": int(unread.get(pid, 0)),
}
)
items.sort(key=lambda x: x["last_message"]["created_at"] or "", reverse=True)
return {"items": items}
@@ -347,9 +347,7 @@ def get_thread(
raise HTTPException(status_code=404, detail="User not found")
partner = _serialize_user(p)
msgs = (
db.execute(select(Message).where(cond).order_by(Message.created_at.asc()))
.scalars()
.all()
db.execute(select(Message).where(cond).order_by(Message.created_at.asc())).scalars().all()
)
if mark_read:
now = datetime.now(timezone.utc)
@@ -390,9 +388,7 @@ def send_message(
raise HTTPException(status_code=404, detail="Recipient not available.")
if db.get(MessageKey, recipient.id) is None:
raise HTTPException(status_code=404, detail="That member hasn't set up messaging yet.")
m = Message(
kind="user", sender_id=user.id, recipient_id=recipient.id, ciphertext=ct, iv=iv
)
m = Message(kind="user", sender_id=user.id, recipient_id=recipient.id, ciphertext=ct, iv=iv)
db.add(m)
db.commit()
db.refresh(m)
@@ -410,9 +406,7 @@ def send_message(
@router.get("/unread_count")
def unread_count(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
def unread_count(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
n = db.scalar(
select(func.count())
.select_from(Message)
@@ -423,6 +417,7 @@ def unread_count(
# --- live delivery -----------------------------------------------------------
@router.websocket("/ws")
async def messages_ws(ws: WebSocket) -> None:
"""Live push channel: authenticated by the session cookie, registers the connection so
backend/app/routes/notifications.py
+5 -12
View File
@@ -4,6 +4,7 @@ with the client-side transient bell. Read-only listing plus read/dismiss/clear m
All endpoints use `current_user` (not `require_human`): notifications and their read/dismiss
state are personal UI state with no quota or YouTube involvement, so the shared demo account
manages its own inbox normally."""
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select, update
from sqlalchemy.orm import Session
@@ -41,9 +42,7 @@ def list_notifications(
where = [Notification.user_id == user.id]
if not include_dismissed:
where.append(Notification.dismissed.is_(False))
total = db.scalar(
select(func.count()).select_from(Notification).where(*where)
) or 0
total = db.scalar(select(func.count()).select_from(Notification).where(*where)) or 0
rows = (
db.execute(
select(Notification)
@@ -59,9 +58,7 @@ def list_notifications(
@router.get("/unread_count")
def unread_count(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
def unread_count(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
"""Cheap poll target for the nav badge (uses the (user_id, read) index)."""
count = db.scalar(
select(func.count())
@@ -96,9 +93,7 @@ def mark_read(
@router.post("/read_all")
def mark_all_read(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
def mark_all_read(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
db.execute(
update(Notification)
.where(Notification.user_id == user.id, Notification.read.is_(False))
@@ -124,9 +119,7 @@ def dismiss(
@router.post("/clear")
def clear_all(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
def clear_all(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
"""Dismiss every (still-visible) notification for this user."""
db.execute(
update(Notification)
backend/app/routes/playlists.py
+13 -16
View File
@@ -2,6 +2,7 @@
CRUD (create / rename / delete, add / remove / reorder items) plus YouTube two-way sync —
mirror the user's own YouTube playlists in (`/sync-youtube`) and push/revert local edits back
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy import func, select, and_
@@ -66,9 +67,7 @@ def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) ->
if exists is not None:
return False
db.add(
PlaylistItem(
playlist_id=pl.id, video_id=video_id, position=_next_item_position(db, pl.id)
)
PlaylistItem(playlist_id=pl.id, video_id=video_id, position=_next_item_position(db, pl.id))
)
if mark_dirty:
recompute_dirty(db, pl)
@@ -168,10 +167,14 @@ def list_playlists(
)
# Batch the cover thumbnails (first item per playlist by position) in ONE window query
# instead of a per-playlist SELECT inside _summary (the old N+1).
rn = func.row_number().over(
partition_by=PlaylistItem.playlist_id,
order_by=PlaylistItem.position,
).label("rn")
rn = (
func.row_number()
.over(
partition_by=PlaylistItem.playlist_id,
order_by=PlaylistItem.position,
)
.label("rn")
)
first_items = (
select(
PlaylistItem.playlist_id.label("pid"),
@@ -226,9 +229,7 @@ def create_playlist(
def _watch_later(db: Session, user: User) -> Playlist:
"""The user's built-in Watch later playlist, created on first use."""
pl = db.scalar(
select(Playlist).where(
Playlist.user_id == user.id, Playlist.kind == "watch_later"
)
select(Playlist).where(Playlist.user_id == user.id, Playlist.kind == "watch_later")
)
if pl is None:
pl = Playlist(
@@ -264,9 +265,7 @@ def remove_watch_later(
db: Session = Depends(get_db),
) -> dict:
pl = db.scalar(
select(Playlist).where(
Playlist.user_id == user.id, Playlist.kind == "watch_later"
)
select(Playlist).where(Playlist.user_id == user.id, Playlist.kind == "watch_later")
)
if pl is not None:
_remove_item(db, pl, video_id, mark_dirty=False)
@@ -460,9 +459,7 @@ def delete_playlist(
if not pl.yt_playlist_id:
raise HTTPException(status_code=400, detail="This playlist isn't on YouTube.")
if not has_write_scope(user):
raise HTTPException(
status_code=403, detail="Enable YouTube editing in Settings first."
)
raise HTTPException(status_code=403, detail="Enable YouTube editing in Settings first.")
try:
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
with YouTubeClient(db, user) as yt:
backend/app/routes/plex/_shared.py
+8 -4
View File
@@ -31,11 +31,15 @@ def _content_libs(db: Session) -> tuple[list[int], list[int]]:
"""The BROWSABLE content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
source of the browse scope. Excludes HU overlay libraries (`overlay_of` set): those are never
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows (E5 S5)."""
libs = db.execute(
select(PlexLibrary).where(
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
libs = (
db.execute(
select(PlexLibrary).where(
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
)
)
).scalars().all()
.scalars()
.all()
)
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
return movie_lib_ids, show_lib_ids
backend/app/routes/plex/admin.py
+8 -24
View File
@@ -25,9 +25,7 @@ router = APIRouter(prefix="/api/plex", tags=["plex"])
@router.post("/test")
def test_connection(
_: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
def test_connection(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
"""Admin: verify the configured Plex server URL + token, returning the server name and the
movie/show sections (for the config library-picker). Also probes ONE real media file through
the local mount, so a missing bind-mount / wrong path-map is caught here rather than only
@@ -59,9 +57,7 @@ def test_connection(
}
def _sample_plex_path(
db: Session, plex: PlexClient, sections: list[dict]
) -> str | None:
def _sample_plex_path(db: Session, plex: PlexClient, sections: list[dict]) -> str | None:
"""A real Plex-side media file path to validate the mount + path-map against. Prefers the
already-mirrored catalog (cheap); before the first sync, pulls one movie leaf from an enabled
section straight from Plex."""
@@ -113,13 +109,9 @@ def _link_dict(link: PlexLink | None) -> dict:
@router.get("/watch/link")
def watch_link_status(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
def watch_link_status(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
"""This user's Plex watch-sync link status (drives the Settings toggle)."""
link = db.execute(
select(PlexLink).where(PlexLink.user_id == user.id)
).scalars().first()
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
return _link_dict(link)
@@ -139,13 +131,9 @@ def watch_link_set(
if not sysconfig.get_bool(db, "plex_enabled"):
raise HTTPException(status_code=409, detail="The Plex module is disabled.")
enabled = body.enabled
link = db.execute(
select(PlexLink).where(PlexLink.user_id == user.id)
).scalars().first()
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
if link is None:
link = PlexLink(
user_id=user.id, uses_admin=True, linked_at=datetime.now(timezone.utc)
)
link = PlexLink(user_id=user.id, uses_admin=True, linked_at=datetime.now(timezone.utc))
db.add(link)
link.sync_enabled = enabled
db.commit()
@@ -157,14 +145,10 @@ def watch_link_set(
@router.post("/watch/import")
def watch_import_now(
user: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
def watch_import_now(user: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
"""Admin: (re-)run the "Plex is master" import for this account — Plex re-wins on the
intersection; Siftlode-only states are preserved. Idempotent."""
link = db.execute(
select(PlexLink).where(PlexLink.user_id == user.id)
).scalars().first()
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
if link is None or not link.sync_enabled:
raise HTTPException(status_code=409, detail="Enable Plex watch-sync first.")
result = plex_watch.import_owner_watch_state(db, link)
backend/app/routes/plex/browse.py
+18 -30
View File
@@ -50,16 +50,18 @@ router = APIRouter(prefix="/api/plex", tags=["plex"])
@router.get("/libraries")
def list_libraries(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
def list_libraries(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
"""The mirrored libraries the user can browse (drives the Plex scope selector). HU overlay
libraries are excluded — they're not browsable, they only feed the EN twin's Hungarian metadata."""
libs = db.execute(
select(PlexLibrary)
.where(PlexLibrary.overlay_of.is_(None))
.order_by(PlexLibrary.kind.desc(), PlexLibrary.title)
).scalars().all()
libs = (
db.execute(
select(PlexLibrary)
.where(PlexLibrary.overlay_of.is_(None))
.order_by(PlexLibrary.kind.desc(), PlexLibrary.title)
)
.scalars()
.all()
)
out = []
for lib in libs:
if lib.kind == "movie":
@@ -69,9 +71,7 @@ def list_libraries(
)
)
else:
count = db.scalar(
select(func.count(PlexShow.id)).where(PlexShow.library_id == lib.id)
)
count = db.scalar(select(func.count(PlexShow.id)).where(PlexShow.library_id == lib.id))
out.append(
{
"key": lib.plex_key,
@@ -116,9 +116,7 @@ def unified_library(
# `episode` scope targets episode rows only — the movie/show grid stays empty; the recheck for
# the specific title/people/summary fields is a strict narrowing on top of the GIN prefilter.
grid_field = field != "episode"
recheck = (
field_recheck if tsq is not None and field not in ("all", "episode") else None
)
recheck = field_recheck if tsq is not None and field not in ("all", "episode") else None
selects = []
if want_movies and grid_field:
@@ -201,9 +199,7 @@ def unified_library(
items = []
total = 0
if selects:
union = (
selects[0] if len(selects) == 1 else selects[0].union_all(*selects[1:])
)
union = selects[0] if len(selects) == 1 else selects[0].union_all(*selects[1:])
u = union.subquery()
total = db.scalar(select(func.count()).select_from(u)) or 0
@@ -219,9 +215,7 @@ def unified_library(
}.get(sort, u.c.added_at)
order = [(col.asc() if sort_dir == "asc" else col.desc()).nullslast()]
order.append(u.c.rk.desc())
rows = db.execute(
select(u).order_by(*order).offset(offset).limit(limit)
).all()
rows = db.execute(select(u).order_by(*order).offset(offset).limit(limit)).all()
for r in rows:
if r.kind == "movie":
@@ -268,9 +262,7 @@ def unified_library(
)
eps = db.execute(
select(PlexItem, ep_st, PlexShow.title.label("show_title"))
.outerjoin(
ep_st, and_(ep_st.item_id == PlexItem.id, ep_st.user_id == user.id)
)
.outerjoin(ep_st, and_(ep_st.item_id == PlexItem.id, ep_st.user_id == user.id))
.outerjoin(PlexShow, PlexShow.id == PlexItem.show_id)
.where(
PlexItem.kind == "episode",
@@ -389,9 +381,7 @@ def facets(
.where(func.jsonb_typeof(gcol) == "array")
.subquery()
)
for g, c in db.execute(
select(expanded.c.g, func.count()).group_by(expanded.c.g)
).all():
for g, c in db.execute(select(expanded.c.g, func.count()).group_by(expanded.c.g)).all():
genre_counts[g] = genre_counts.get(g, 0) + c
# Content ratings — exclude its own.
crq = (
@@ -482,12 +472,10 @@ def facets(
studio_counts.setdefault(s, 0)
return {
"genres": [
{"value": g, "count": c}
for g, c in sorted(genre_counts.items(), key=lambda kv: kv[0])
{"value": g, "count": c} for g, c in sorted(genre_counts.items(), key=lambda kv: kv[0])
],
"content_ratings": [
{"value": cr, "count": c}
for cr, c in sorted(cr_counts.items(), key=lambda kv: -kv[1])
{"value": cr, "count": c} for cr, c in sorted(cr_counts.items(), key=lambda kv: -kv[1])
],
"studios": [
{"value": s, "count": c}
backend/app/routes/plex/collections.py
+60 -37
View File
@@ -25,10 +25,16 @@ from app.routes.plex._shared import _content_libs
# The most-restrictive → least ordering used to pick a collection's effective content rating from its
# members (Plex shows one cert on the collection; we don't store it, so derive it).
_CERT_RANK = {
"G": 1, "TV-Y": 1, "TV-Y7": 1, "TV-G": 1,
"PG": 2, "TV-PG": 2,
"PG-13": 3, "TV-14": 3,
"R": 4, "TV-MA": 4,
"G": 1,
"TV-Y": 1,
"TV-Y7": 1,
"TV-G": 1,
"PG": 2,
"TV-PG": 2,
"PG-13": 3,
"TV-14": 3,
"R": 4,
"TV-MA": 4,
"NC-17": 5,
}
@@ -37,6 +43,7 @@ def _collection_cert(certs: list[str | None]) -> str | None:
ranked = [(c, _CERT_RANK.get(c, 0)) for c in certs if c]
return max(ranked, key=lambda x: x[1])[0] if ranked else None
router = APIRouter(prefix="/api/plex", tags=["plex"])
_STRIP_PROVIDERS = ("imdb", "tmdb", "tvdb", "trakt")
@@ -85,9 +92,11 @@ def collections(
the UNION across all enabled libraries (the unified-scope sidebar picker, which has no single
library selected)."""
if library is not None:
lib = db.execute(
select(PlexLibrary).where(PlexLibrary.plex_key == str(library))
).scalars().first()
lib = (
db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == str(library)))
.scalars()
.first()
)
if lib is None:
return {"collections": []}
query = select(PlexCollection).where(PlexCollection.library_id == lib.id)
@@ -101,18 +110,20 @@ def collections(
)
if not enabled_ids:
return {"collections": []}
query = select(PlexCollection).where(
PlexCollection.library_id.in_(enabled_ids)
)
query = select(PlexCollection).where(PlexCollection.library_id.in_(enabled_ids))
term = (q or "").strip()
if term:
query = query.where(PlexCollection.title.ilike(f"%{like_escape(term)}%"))
rows = db.execute(
query.order_by(
PlexCollection.child_count.desc().nullslast(),
func.lower(PlexCollection.title),
rows = (
db.execute(
query.order_by(
PlexCollection.child_count.desc().nullslast(),
func.lower(PlexCollection.title),
)
)
).scalars().all()
.scalars()
.all()
)
# A montage of the top titles per tile + the collection's total runtime. Art unions MOVIE and SHOW
# members (a collection lives in one library, so exactly one side has art) — a show collection would
# otherwise render blank; the collection's own poster is the next fallback, then the colour hash.
@@ -150,16 +161,24 @@ def collection_detail(
col = by_rating_key(db, PlexCollection, rating_key)
if col is None:
raise HTTPException(status_code=404, detail="Unknown collection")
movies = db.execute(
select(PlexItem)
.where(PlexItem.kind == "movie", PlexItem.collection_keys.contains([rating_key]))
.order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title))
).scalars().all()
shows = db.execute(
select(PlexShow)
.where(PlexShow.collection_keys.contains([rating_key]))
.order_by(PlexShow.year.asc().nullslast(), func.lower(PlexShow.title))
).scalars().all()
movies = (
db.execute(
select(PlexItem)
.where(PlexItem.kind == "movie", PlexItem.collection_keys.contains([rating_key]))
.order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title))
)
.scalars()
.all()
)
shows = (
db.execute(
select(PlexShow)
.where(PlexShow.collection_keys.contains([rating_key]))
.order_by(PlexShow.year.asc().nullslast(), func.lower(PlexShow.title))
)
.scalars()
.all()
)
mstate = rollup.item_state_map(db, user.id, [m.id for m in movies])
sstate = rollup.state_map(db, user.id, [s.id for s in shows])
# Merge the two type-homogeneous lists into ONE member list ordered by (year, title).
@@ -168,7 +187,9 @@ def collection_detail(
]
tagged.sort(key=lambda t: (t[0] is None, t[0] or 0, t[1]))
items = [
movie_card(o, mstate.get(o.id)) if kind == "movie" else show_card(o, sstate.get(o.id, "new"))
movie_card(o, mstate.get(o.id))
if kind == "movie"
else show_card(o, sstate.get(o.id, "new"))
for _, _, kind, o in tagged
]
years = [y for y, _, _, _ in tagged if y is not None]
@@ -198,12 +219,16 @@ def _reject_duplicate_name(
"""No two collections in a section may share a name: Plex CONFLATES same-named collections'
membership (both rating_keys resolve to one merged member list — a real footgun), so forbid it up
front on create + rename rather than mirror the mess."""
dup = db.execute(
select(PlexCollection).where(
PlexCollection.library_id == library_id,
func.lower(PlexCollection.title) == title.lower(),
dup = (
db.execute(
select(PlexCollection).where(
PlexCollection.library_id == library_id,
func.lower(PlexCollection.title) == title.lower(),
)
)
).scalars().first()
.scalars()
.first()
)
if dup is not None and dup.rating_key != exclude_rk:
raise HTTPException(status_code=409, detail="A collection with this name already exists")
@@ -244,13 +269,11 @@ def create_collection(
title = body.title.strip()
seed = body.item_rating_key
if not title or not seed:
raise HTTPException(
status_code=400, detail="title and item_rating_key are required"
)
raise HTTPException(status_code=400, detail="title and item_rating_key are required")
if library:
lib = db.execute(
select(PlexLibrary).where(PlexLibrary.plex_key == library)
).scalars().first()
lib = (
db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == library)).scalars().first()
)
else:
# The "New collection" flow only knows the seed title → derive its library.
seed_item = by_rating_key(db, PlexItem, seed) or by_rating_key(db, PlexShow, seed)
backend/app/routes/plex/detail.py
+47 -48
View File
@@ -62,9 +62,7 @@ def _show_enrichment(db: Session, rating_key: str) -> tuple[dict, list]:
with PlexClient(db) as plex:
with ThreadPoolExecutor(max_workers=2) as ex:
meta_f = ex.submit(plex.metadata, rating_key)
related_f = ex.submit(
plex.related, rating_key
) # swallows its own errors → []
related_f = ex.submit(plex.related, rating_key) # swallows its own errors → []
meta = meta_f.result() or {}
related_raw = related_f.result()
except (PlexError, PlexNotConfigured):
@@ -95,19 +93,27 @@ def show_detail(
sh = by_rating_key(db, PlexShow, rating_key)
if sh is None:
raise HTTPException(status_code=404, detail="Unknown Plex show")
seasons = db.execute(
select(PlexSeason)
.where(PlexSeason.show_id == sh.id)
.order_by(PlexSeason.season_number.asc().nullsfirst())
).scalars().all()
eps = db.execute(
select(PlexItem)
.where(PlexItem.show_id == sh.id, PlexItem.kind == "episode")
.order_by(
PlexItem.season_number.asc().nullsfirst(),
PlexItem.episode_number.asc().nullsfirst(),
seasons = (
db.execute(
select(PlexSeason)
.where(PlexSeason.show_id == sh.id)
.order_by(PlexSeason.season_number.asc().nullsfirst())
)
).scalars().all()
.scalars()
.all()
)
eps = (
db.execute(
select(PlexItem)
.where(PlexItem.show_id == sh.id, PlexItem.kind == "episode")
.order_by(
PlexItem.season_number.asc().nullsfirst(),
PlexItem.episode_number.asc().nullsfirst(),
)
)
.scalars()
.all()
)
states = {
s.item_id: s
for s in db.execute(
@@ -118,9 +124,7 @@ def show_detail(
).scalars()
}
by_season: dict[int | None, list] = {}
all_cards: list[
dict
] = [] # every episode, in season/episode order → show-level rollup
all_cards: list[dict] = [] # every episode, in season/episode order → show-level rollup
for e in eps:
card = episode_card(e, states.get(e.id))
by_season.setdefault(e.season_id, []).append(card)
@@ -188,8 +192,7 @@ def _season_block(se: PlexSeason, cards: list[dict]) -> dict:
return {
"id": se.rating_key,
"season_number": se.season_number,
"title": se.title
or (f"Season {se.season_number}" if se.season_number else "Season"),
"title": se.title or (f"Season {se.season_number}" if se.season_number else "Season"),
"thumb": f"/api/plex/image/{se.rating_key}",
"episode_count": roll["episode_count"],
"status": roll["status"],
@@ -199,9 +202,7 @@ def _season_block(se: PlexSeason, cards: list[dict]) -> dict:
}
def _relatedshow_cards(
db: Session, user_id: int, related: list[dict], exclude: int
) -> list[dict]:
def _relatedshow_cards(db: Session, user_id: int, related: list[dict], exclude: int) -> list[dict]:
"""Map Plex 'related' metadata to show cards for the shows we actually mirror (so they're openable),
excluding the current show. Order preserved; capped."""
rks = [str(m.get("ratingKey")) for m in related if m.get("ratingKey")]
@@ -210,9 +211,7 @@ def _relatedshow_cards(
shows = {
s.rating_key: s
for s in db.execute(
select(PlexShow).where(
PlexShow.rating_key.in_(rks[:60]), PlexShow.id != exclude
)
select(PlexShow).where(PlexShow.rating_key.in_(rks[:60]), PlexShow.id != exclude)
).scalars()
}
statuses = rollup.state_map(db, user_id, [s.id for s in shows.values()])
@@ -242,22 +241,24 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
).scalars()
}
# Smallest (most specific — e.g. a franchise) first; the big auto-lists (IMDb/TMDb Popular) last.
ordered = sorted(
(c for c in cols.values()), key=lambda c: (c.child_count or 0, c.title)
)
ordered = sorted((c for c in cols.values()), key=lambda c: (c.child_count or 0, c.title))
out = []
for col in ordered:
rk = col.rating_key
members = db.execute(
select(PlexItem)
.where(
PlexItem.library_id == it.library_id,
PlexItem.kind == "movie",
PlexItem.collection_keys.contains([rk]),
members = (
db.execute(
select(PlexItem)
.where(
PlexItem.library_id == it.library_id,
PlexItem.kind == "movie",
PlexItem.collection_keys.contains([rk]),
)
.order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title))
.limit(60)
)
.order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title))
.limit(60)
).scalars().all()
.scalars()
.all()
)
ids = [m.id for m in members]
states = {
s.item_id: s
@@ -295,9 +296,7 @@ def _episode_nav(db: Session, it: PlexItem) -> tuple[str | None, str | None]:
i = keys.index(it.rating_key)
except ValueError:
return None, None
return (keys[i - 1] if i > 0 else None), (
keys[i + 1] if i < len(keys) - 1 else None
)
return (keys[i - 1] if i > 0 else None), (keys[i + 1] if i < len(keys) - 1 else None)
@router.get("/item/{rating_key}")
@@ -311,11 +310,13 @@ def item_detail(
prev/next, and the per-user resume position + watch status."""
lang = _norm_lang(lang)
it = item_or_404(db, rating_key)
st = db.execute(
select(PlexState).where(
PlexState.user_id == user.id, PlexState.item_id == it.id
st = (
db.execute(
select(PlexState).where(PlexState.user_id == user.id, PlexState.item_id == it.id)
)
).scalars().first()
.scalars()
.first()
)
# Markers + audio/subtitle streams are file-specific, so they stay LIVE; the rich detail (cast/IMDb/
# tagline/genres) is served from the DB row for movies (persisted below / by the backfill) and only
@@ -430,9 +431,7 @@ def item_detail(
"subtitle_streams": subtitle_streams,
"status": st.status if st else "new",
"position_seconds": st.position_seconds if st else 0,
"show_title": (
(show.title_hu or show.title if hu else show.title) if show else None
),
"show_title": ((show.title_hu or show.title if hu else show.title) if show else None),
"season_number": it.season_number,
"episode_number": it.episode_number,
"prev_id": prev_id,
backend/app/routes/plex/discover.py
+103 -58
View File
@@ -71,7 +71,13 @@ def _show_cards(db: Session, uid: int, shows: list[PlexShow]) -> list[dict]:
def _recently_added(
db: Session, uid: int, movie_ids: list[int], show_ids: list[int], p: dict, lang: str, f: LibraryFilters
db: Session,
uid: int,
movie_ids: list[int],
show_ids: list[int],
p: dict,
lang: str,
f: LibraryFilters,
) -> list[dict]:
"""Newest movies + shows, interleaved by `added_at` desc into one rail."""
movies = (
@@ -80,11 +86,15 @@ def _recently_added(
select(PlexItem).where(
PlexItem.kind == "movie", PlexItem.library_id.in_(movie_ids)
),
p, lang, f,
p,
lang,
f,
)
.order_by(PlexItem.added_at.desc().nullslast())
.limit(RAIL_LIMIT)
).scalars().all()
)
.scalars()
.all()
if movie_ids
else []
)
@@ -95,16 +105,16 @@ def _recently_added(
)
.order_by(PlexShow.added_at.desc().nullslast())
.limit(RAIL_LIMIT)
).scalars().all()
)
.scalars()
.all()
if show_ids
else []
)
# Merge the two type-homogeneous lists by added_at, then serialize each side in one batch so the
# per-card state lookups stay batched (no N+1). The `(is not None, value)` key sorts a missing
# timestamp last under reverse.
tagged = [(m.added_at, "movie", m) for m in movies] + [
(s.added_at, "show", s) for s in shows
]
tagged = [(m.added_at, "movie", m) for m in movies] + [(s.added_at, "show", s) for s in shows]
tagged.sort(key=lambda t: (t[0] is not None, t[0]), reverse=True)
top = tagged[:RAIL_LIMIT]
# Key the serialized cards back by rating_key (globally unique in Plex — a movie and a show never
@@ -113,14 +123,17 @@ def _recently_added(
top_shows = [o for _, kind, o in top if kind == "show"]
mcards = {m.rating_key: c for m, c in zip(top_movies, _movie_cards(db, uid, top_movies))}
scards = {s.rating_key: c for s, c in zip(top_shows, _show_cards(db, uid, top_shows))}
return [
mcards[o.rating_key] if kind == "movie" else scards[o.rating_key]
for _, kind, o in top
]
return [mcards[o.rating_key] if kind == "movie" else scards[o.rating_key] for _, kind, o in top]
def _recently_watched(
db: Session, uid: int, movie_ids: list[int], show_ids: list[int], p: dict, lang: str, f: LibraryFilters
db: Session,
uid: int,
movie_ids: list[int],
show_ids: list[int],
p: dict,
lang: str,
f: LibraryFilters,
) -> list[dict]:
"""Recently watched rail: the titles the user last FULLY watched, newest first. A movie is its own
card; a show contributes the specific EPISODE watched (carrying its show title). Movies honour the
@@ -162,7 +175,13 @@ def _recently_watched(
def _continue_watching(
db: Session, uid: int, movie_ids: list[int], show_ids: list[int], p: dict, lang: str, f: LibraryFilters
db: Session,
uid: int,
movie_ids: list[int],
show_ids: list[int],
p: dict,
lang: str,
f: LibraryFilters,
) -> list[dict]:
"""Resume rail: in-progress movies + partially-watched shows, most-recently-progressed first.
A movie's resume time is its own `progress_updated_at`; a show's is the max across its episodes."""
@@ -238,9 +257,7 @@ def _genre_tiles(
if model is PlexItem:
base = base.where(model.kind == "movie")
expanded = base.subquery()
for g, c in db.execute(
select(expanded.c.g, func.count()).group_by(expanded.c.g)
).all():
for g, c in db.execute(select(expanded.c.g, func.count()).group_by(expanded.c.g)).all():
counts[g] = counts.get(g, 0) + c
top = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[:GENRE_TILES]
# A 2×2 poster collage per tile: the top movies in that genre (color hash is the client fallback).
@@ -256,7 +273,13 @@ def _genre_tiles(
def _because_you_watched(
db: Session, uid: int, movie_ids: list[int], show_ids: list[int], p: dict, lang: str, f: LibraryFilters
db: Session,
uid: int,
movie_ids: list[int],
show_ids: list[int],
p: dict,
lang: str,
f: LibraryFilters,
) -> dict | None:
"""Recommendations anchored on the user's most-recently-watched, TMDB-enriched movie. Only the
recs that resolve to OUR library (by guid) become cards — the rest aren't actionable here. Returns
@@ -264,21 +287,25 @@ def _because_you_watched(
The metadata filters narrow the resolved recs (not the anchor pick) so the rail matches the grid."""
if not movie_ids:
return None
anchor = db.execute(
select(PlexItem)
.join(
PlexState,
and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid),
anchor = (
db.execute(
select(PlexItem)
.join(
PlexState,
and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid),
)
.where(
PlexItem.kind == "movie",
PlexItem.library_id.in_(movie_ids),
PlexState.status == "watched",
PlexItem.recommendations.isnot(None),
)
.order_by(PlexState.watched_at.desc().nullslast())
.limit(1)
)
.where(
PlexItem.kind == "movie",
PlexItem.library_id.in_(movie_ids),
PlexState.status == "watched",
PlexItem.recommendations.isnot(None),
)
.order_by(PlexState.watched_at.desc().nullslast())
.limit(1)
).scalars().first()
.scalars()
.first()
)
if anchor is None:
return None
resolved = resolve_recs(db, anchor.recommendations)
@@ -294,12 +321,14 @@ def _because_you_watched(
m.rating_key: m
for m in db.execute(
_movie_meta(
select(PlexItem).where(
PlexItem.kind == "movie", PlexItem.rating_key.in_(rks)
),
p, lang, f,
select(PlexItem).where(PlexItem.kind == "movie", PlexItem.rating_key.in_(rks)),
p,
lang,
f,
)
).scalars().all()
)
.scalars()
.all()
}
shows = {
s.rating_key: s
@@ -307,12 +336,16 @@ def _because_you_watched(
apply_meta_filters(
select(PlexShow).where(PlexShow.rating_key.in_(rks)), PlexShow, p, lang
)
).scalars().all()
)
.scalars()
.all()
}
ordered_movies = [movies[rk] for rk in rks if rk in movies]
ordered_shows = [shows[rk] for rk in rks if rk in shows]
# rating_key is globally unique in Plex, so it keys the serialized cards back safely (no id()).
mcards = {m.rating_key: c for m, c in zip(ordered_movies, _movie_cards(db, uid, ordered_movies))}
mcards = {
m.rating_key: c for m, c in zip(ordered_movies, _movie_cards(db, uid, ordered_movies))
}
scards = {s.rating_key: c for s, c in zip(ordered_shows, _show_cards(db, uid, ordered_shows))}
items: list[dict] = []
for rk in rks:
@@ -379,18 +412,24 @@ def surprise(
)
.exists()
)
movies = db.execute(
_movie_meta(
select(PlexItem).where(
PlexItem.kind == "movie",
PlexItem.library_id.in_(movie_ids),
~watched_movie,
),
p, lang, f,
movies = (
db.execute(
_movie_meta(
select(PlexItem).where(
PlexItem.kind == "movie",
PlexItem.library_id.in_(movie_ids),
~watched_movie,
),
p,
lang,
f,
)
.order_by(func.random())
.limit(per)
)
.order_by(func.random())
.limit(per)
).scalars().all()
.scalars()
.all()
)
shows = []
if show_ids:
# Exclude fully-watched shows (LEFT JOIN so a never-started show — no rollup row — still counts
@@ -399,16 +438,22 @@ def surprise(
not_fully_watched = or_(
prog.c.total.is_(None), prog.c.total == 0, prog.c.watched < prog.c.total
)
shows = db.execute(
apply_meta_filters(
select(PlexShow)
.outerjoin(prog, prog.c.show_id == PlexShow.id)
.where(PlexShow.library_id.in_(show_ids), not_fully_watched),
PlexShow, p, lang,
shows = (
db.execute(
apply_meta_filters(
select(PlexShow)
.outerjoin(prog, prog.c.show_id == PlexShow.id)
.where(PlexShow.library_id.in_(show_ids), not_fully_watched),
PlexShow,
p,
lang,
)
.order_by(func.random())
.limit(per)
)
.order_by(func.random())
.limit(per)
).scalars().all()
.scalars()
.all()
)
mcards = _movie_cards(db, user.id, movies)
scards = _show_cards(db, user.id, shows)
# Interleave movie/show cards, then trim to `limit`.
backend/app/routes/plex/people.py
+32 -26
View File
@@ -76,9 +76,7 @@ def people_cards(
want_shows = scope in ("show", "both") and bool(show_lib_ids)
photos = {
p.name: p.photo_thumb
for p in db.execute(
select(PlexPerson).where(PlexPerson.name.in_(wanted))
).scalars()
for p in db.execute(select(PlexPerson).where(PlexPerson.name.in_(wanted))).scalars()
}
out = []
for nm in wanted:
@@ -191,36 +189,44 @@ def person_detail(
movies: list[dict] = []
if movie_lib_ids:
items = db.execute(
select(PlexItem)
.where(
PlexItem.kind == "movie",
PlexItem.library_id.in_(movie_lib_ids),
or_(
PlexItem.cast_names.contains([name]),
PlexItem.directors.contains([name]),
),
items = (
db.execute(
select(PlexItem)
.where(
PlexItem.kind == "movie",
PlexItem.library_id.in_(movie_lib_ids),
or_(
PlexItem.cast_names.contains([name]),
PlexItem.directors.contains([name]),
),
)
.order_by(PlexItem.year.desc().nullslast(), func.lower(PlexItem.title))
.limit(_PERSON_SECTION_MAX)
)
.order_by(PlexItem.year.desc().nullslast(), func.lower(PlexItem.title))
.limit(_PERSON_SECTION_MAX)
).scalars().all()
.scalars()
.all()
)
states = rollup.item_state_map(db, user.id, [it.id for it in items])
movies = [movie_card(it, states.get(it.id)) for it in items]
shows: list[dict] = []
if show_lib_ids:
show_rows = db.execute(
select(PlexShow)
.where(
PlexShow.library_id.in_(show_lib_ids),
or_(
PlexShow.cast_names.contains([name]),
PlexShow.directors.contains([name]),
),
show_rows = (
db.execute(
select(PlexShow)
.where(
PlexShow.library_id.in_(show_lib_ids),
or_(
PlexShow.cast_names.contains([name]),
PlexShow.directors.contains([name]),
),
)
.order_by(PlexShow.year.desc().nullslast(), func.lower(PlexShow.title))
.limit(_PERSON_SECTION_MAX)
)
.order_by(PlexShow.year.desc().nullslast(), func.lower(PlexShow.title))
.limit(_PERSON_SECTION_MAX)
).scalars().all()
.scalars()
.all()
)
statuses = rollup.state_map(db, user.id, [s.id for s in show_rows])
shows = [show_card(sh, statuses.get(sh.id, "new")) for sh in show_rows]
backend/app/routes/plex/playback.py
+39 -45
View File
@@ -76,9 +76,7 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response
key = imgcache.key_for(db, str(rating_key), v)
if not key:
raise HTTPException(status_code=404, detail="No image")
plex = PlexClient(
db
) # reads base URL + token now; the fetch below needs no DB
plex = PlexClient(db) # reads base URL + token now; the fetch below needs no DB
except PlexNotConfigured as e:
raise HTTPException(status_code=400, detail=str(e))
try:
@@ -91,9 +89,7 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response
@router.get("/subtitle/{rating_key}/{sub_ord}")
def subtitle(
rating_key: str, sub_ord: int, request: Request, offset: float = 0.0
) -> Response:
def subtitle(rating_key: str, sub_ord: int, request: Request, offset: float = 0.0) -> Response:
"""Serve the ord-th subtitle track as WebVTT for the browser's <track> — so choosing a subtitle
overlays it directly (no HLS re-mux / session restart). Handles EXTERNAL sidecar subs (fetched
from Plex via the stream `key` — the old `-map 0:s` broke on these since they're not in the file)
@@ -109,12 +105,10 @@ def subtitle(
with SessionLocal() as db:
it = item_or_404(db, rating_key)
with PlexClient(db) as plex:
part = ((plex.metadata(it.rating_key) or {}).get("Media") or [{}])[
0
].get("Part") or [{}]
subs = [
s for s in (part[0].get("Stream") or []) if s.get("streamType") == 3
]
part = ((plex.metadata(it.rating_key) or {}).get("Media") or [{}])[0].get(
"Part"
) or [{}]
subs = [s for s in (part[0].get("Stream") or []) if s.get("streamType") == 3]
if sub_ord < 0 or sub_ord >= len(subs):
raise HTTPException(status_code=404, detail="No such subtitle")
s = subs[sub_ord]
@@ -136,22 +130,14 @@ def subtitle(
vtt = subtitles.ffmpeg_to_vtt(tmp, None)
finally:
os.unlink(tmp)
elif (
s.get("index") is not None
): # embedded → extract from the local file
elif s.get("index") is not None: # embedded → extract from the local file
src = plex_paths.local_media_path(db, it.file_path)
if src is None:
raise HTTPException(
status_code=404, detail="Media file not found on disk"
)
emb_ord = sum(
1 for x in subs[:sub_ord] if x.get("index") is not None
)
raise HTTPException(status_code=404, detail="Media file not found on disk")
emb_ord = sum(1 for x in subs[:sub_ord] if x.get("index") is not None)
vtt = subtitles.ffmpeg_to_vtt(str(src), emb_ord)
else:
raise HTTPException(
status_code=404, detail="Subtitle not fetchable"
)
raise HTTPException(status_code=404, detail="Subtitle not fetchable")
except PlexNotConfigured as e:
raise HTTPException(status_code=400, detail=str(e))
except PlexError as e:
@@ -209,11 +195,13 @@ def _push_watch(
return
# Capture the row's freshest timestamp NOW so the background push can tell if a newer local edit
# lands before it settles synced_to_plex (which would otherwise strand that newer value).
st = db.execute(
select(PlexState).where(
PlexState.user_id == user_id, PlexState.item_id == item_id
st = (
db.execute(
select(PlexState).where(PlexState.user_id == user_id, PlexState.item_id == item_id)
)
).scalars().first()
.scalars()
.first()
)
background.add_task(
plex_watch.push_state_to_plex,
user_id,
@@ -252,11 +240,13 @@ def item_progress(
duration = int(body.duration_seconds)
final = body.final
now = datetime.now(timezone.utc)
st = db.execute(
select(PlexState).where(
PlexState.user_id == user.id, PlexState.item_id == it.id
st = (
db.execute(
select(PlexState).where(PlexState.user_id == user.id, PlexState.item_id == it.id)
)
).scalars().first()
.scalars()
.first()
)
was_watched = st is not None and st.status == "watched"
near_end = duration > 0 and position > duration - _FINISH_MARGIN_S
@@ -326,11 +316,13 @@ def item_state(
status = body.status
if status not in ("new", "watched", "hidden"):
raise HTTPException(status_code=400, detail="Invalid status")
st = db.execute(
select(PlexState).where(
PlexState.user_id == user.id, PlexState.item_id == it.id
st = (
db.execute(
select(PlexState).where(PlexState.user_id == user.id, PlexState.item_id == it.id)
)
).scalars().first()
.scalars()
.first()
)
prev_status = st.status if st is not None else "new"
if status == "new":
if st is not None:
@@ -417,9 +409,11 @@ def show_state(
if sh is None:
raise HTTPException(status_code=404, detail="Unknown Plex show")
watched = body.watched
eps = db.execute(
select(PlexItem).where(PlexItem.show_id == sh.id, PlexItem.kind == "episode")
).scalars().all()
eps = (
db.execute(select(PlexItem).where(PlexItem.show_id == sh.id, PlexItem.kind == "episode"))
.scalars()
.all()
)
changed = _bulk_state(background, db, user.id, eps, watched)
return {"changed": changed, "watched": watched}
@@ -437,9 +431,11 @@ def season_state(
if se is None:
raise HTTPException(status_code=404, detail="Unknown Plex season")
watched = body.watched
eps = db.execute(
select(PlexItem).where(PlexItem.season_id == se.id, PlexItem.kind == "episode")
).scalars().all()
eps = (
db.execute(select(PlexItem).where(PlexItem.season_id == se.id, PlexItem.kind == "episode"))
.scalars()
.all()
)
changed = _bulk_state(background, db, user.id, eps, watched)
return {"changed": changed, "watched": watched}
@@ -504,9 +500,7 @@ def stream_file(
# A clean, safe download name from the title + the real file extension.
ext = src.suffix or (f".{it.container}" if it.container else "")
base = (
"".join(
c for c in (it.title or "video") if c.isalnum() or c in " -_."
).strip()
"".join(c for c in (it.title or "video") if c.isalnum() or c in " -_.").strip()
or "video"
)
return FileResponse(str(src), filename=f"{base}{ext}")
backend/app/routes/plex/playlists.py
+17 -31
View File
@@ -59,16 +59,12 @@ def playlists(
group it already holds) + a top-level `group_size`, for the 'add a whole season/show' dialog."""
contains_id = None
if contains:
contains_id = db.scalar(
select(PlexItem.id).where(PlexItem.rating_key == str(contains))
)
contains_id = db.scalar(select(PlexItem.id).where(PlexItem.rating_key == str(contains)))
group_ids: set[int] | None = None
if contains_group is not None:
rks = [s for s in (contains_group or "").split(",") if s]
group_ids = set(
db.execute(
select(PlexItem.id).where(PlexItem.rating_key.in_(rks or [""]))
).scalars()
db.execute(select(PlexItem.id).where(PlexItem.rating_key.in_(rks or [""]))).scalars()
)
out = []
for pl in db.execute(
@@ -78,9 +74,7 @@ def playlists(
).scalars():
count = (
db.scalar(
select(func.count(PlexPlaylistItem.id)).where(
PlexPlaylistItem.playlist_id == pl.id
)
select(func.count(PlexPlaylistItem.id)).where(PlexPlaylistItem.playlist_id == pl.id)
)
or 0
)
@@ -197,9 +191,7 @@ def rename_playlist(
db.commit()
count = (
db.scalar(
select(func.count(PlexPlaylistItem.id)).where(
PlexPlaylistItem.playlist_id == pl.id
)
select(func.count(PlexPlaylistItem.id)).where(PlexPlaylistItem.playlist_id == pl.id)
)
or 0
)
@@ -232,20 +224,22 @@ def playlist_add_item(
it = by_rating_key(db, PlexItem, body.item_rating_key)
if it is None:
raise HTTPException(status_code=404, detail="Unknown item")
exists = db.execute(
select(PlexPlaylistItem).where(
PlexPlaylistItem.playlist_id == pid, PlexPlaylistItem.item_id == it.id
exists = (
db.execute(
select(PlexPlaylistItem).where(
PlexPlaylistItem.playlist_id == pid, PlexPlaylistItem.item_id == it.id
)
)
).scalars().first()
.scalars()
.first()
)
if exists is None:
maxpos = db.scalar(
select(func.coalesce(func.max(PlexPlaylistItem.position), -1)).where(
PlexPlaylistItem.playlist_id == pid
)
)
db.add(
PlexPlaylistItem(playlist_id=pid, item_id=it.id, position=int(maxpos) + 1)
)
db.add(PlexPlaylistItem(playlist_id=pid, item_id=it.id, position=int(maxpos) + 1))
pl.updated_at = datetime.now(timezone.utc)
db.commit()
return {"ok": True}
@@ -282,9 +276,7 @@ def _items_by_rks(db: Session, rks: list) -> list[PlexItem]:
return []
by_rk = {
it.rating_key: it
for it in db.execute(
select(PlexItem).where(PlexItem.rating_key.in_(order))
).scalars()
for it in db.execute(select(PlexItem).where(PlexItem.rating_key.in_(order))).scalars()
}
seen: set[str] = set()
out: list[PlexItem] = []
@@ -332,9 +324,7 @@ def playlist_add_bulk(
db.commit()
total = (
db.scalar(
select(func.count(PlexPlaylistItem.id)).where(
PlexPlaylistItem.playlist_id == pid
)
select(func.count(PlexPlaylistItem.id)).where(PlexPlaylistItem.playlist_id == pid)
)
or 0
)
@@ -365,9 +355,7 @@ def playlist_remove_bulk(
db.commit()
total = (
db.scalar(
select(func.count(PlexPlaylistItem.id)).where(
PlexPlaylistItem.playlist_id == pid
)
select(func.count(PlexPlaylistItem.id)).where(PlexPlaylistItem.playlist_id == pid)
)
or 0
)
@@ -395,9 +383,7 @@ def reorder_playlist(
if iid is not None:
db.execute(
update(PlexPlaylistItem)
.where(
PlexPlaylistItem.playlist_id == pid, PlexPlaylistItem.item_id == iid
)
.where(PlexPlaylistItem.playlist_id == pid, PlexPlaylistItem.item_id == iid)
.values(position=pos)
)
pl.updated_at = datetime.now(timezone.utc)
backend/app/routes/public.py
+1 -3
View File
@@ -27,9 +27,7 @@ _unlock_limit = RateLimiter(max_events=10, window_seconds=300)
def _link_or_404(db: Session, token: str) -> DownloadLink:
link = db.execute(
select(DownloadLink).where(DownloadLink.token == token)
).scalar_one_or_none()
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
if link is None or linksmod.is_expired(link):
raise HTTPException(status_code=404, detail="This link is no longer available.")
return link
backend/app/routes/quota.py
+3 -9
View File
@@ -3,6 +3,7 @@
The aggregate daily budget lives in ApiQuotaUsage; here we read the per-spend audit log
(QuotaEvent) to attribute units to the user who triggered them (NULL actor = system).
"""
from datetime import datetime, timedelta, timezone
from fastapi import APIRouter, Depends
@@ -18,18 +19,11 @@ router = APIRouter(prefix="/api/quota", tags=["quota"])
def _sum(db: Session, *conds) -> int:
return int(
db.scalar(
select(func.coalesce(func.sum(QuotaEvent.units), 0)).where(*conds)
)
or 0
)
return int(db.scalar(select(func.coalesce(func.sum(QuotaEvent.units), 0)).where(*conds)) or 0)
@router.get("/my-usage")
def my_usage(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
now = datetime.now(timezone.utc)
mine = QuotaEvent.user_id == user.id
by_action = {
backend/app/routes/saved_views.py
+3 -11
View File
@@ -86,9 +86,7 @@ def create_view(
db.commit()
except IntegrityError:
db.rollback()
raise HTTPException(
status_code=409, detail=f"You already have a view named “{name}”."
)
raise HTTPException(status_code=409, detail=f"You already have a view named “{name}”.")
return _serialize(view)
@@ -140,9 +138,7 @@ def update_view(
db.commit()
except IntegrityError:
db.rollback()
raise HTTPException(
status_code=409, detail=f"You already have a view named “{view.name}”."
)
raise HTTPException(status_code=409, detail=f"You already have a view named “{view.name}”.")
return _serialize(view)
@@ -169,11 +165,7 @@ def reorder_views(
db: Session = Depends(get_db),
) -> dict:
ids = body.ids
rows = (
db.execute(select(SavedView).where(SavedView.user_id == user.id))
.scalars()
.all()
)
rows = db.execute(select(SavedView).where(SavedView.user_id == user.id)).scalars().all()
by_id = {v.id: v for v in rows}
pos = 0
for vid in ids:
backend/app/routes/scheduler.py
+17 -9
View File
@@ -1,6 +1,7 @@
"""Admin Scheduler dashboard: a live view of what the background scheduler is doing —
per-job run activity (from the in-process scheduler), the work still queued (DB-derived),
and the shared quota picture."""
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy import and_, func, or_, select
@@ -49,8 +50,13 @@ def set_job_interval(
)
applied = apply_interval(db, job_id, minutes)
audit.record(
db, admin.id, AuditAction.SCHEDULER_INTERVAL, target_type="job", target_id=job_id,
summary=f"{job_id} → {applied} min/run", after={"interval_minutes": applied},
db,
admin.id,
AuditAction.SCHEDULER_INTERVAL,
target_type="job",
target_id=job_id,
summary=f"{job_id} → {applied} min/run",
after={"interval_minutes": applied},
)
db.commit()
return {"id": job_id, "interval_minutes": applied}
@@ -97,15 +103,19 @@ def set_maintenance_settings(
except (TypeError, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc) or "Invalid value")
new = sysconfig.get_int(db, "maintenance_revalidate_batch")
audit.record(db, admin.id, AuditAction.MAINTENANCE_TUNE, summary=f"revalidate batch = {new}", after={"revalidate_batch": new})
audit.record(
db,
admin.id,
AuditAction.MAINTENANCE_TUNE,
summary=f"revalidate batch = {new}",
after={"revalidate_batch": new},
)
db.commit()
return {"revalidate_batch": new}
@router.get("")
def get_scheduler(
_: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
def get_scheduler(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
snap = scheduler_snapshot()
# Work still queued (shared catalog, so these are instance-wide, not per-user).
@@ -118,9 +128,7 @@ def get_scheduler(
.exists()
)
deep_channels = (
db.execute(
select(Channel).where(Channel.backfill_done.is_(False), deep_requested)
)
db.execute(select(Channel).where(Channel.backfill_done.is_(False), deep_requested))
.scalars()
.all()
)
backend/app/routes/search.py
+12 -19
View File
@@ -12,6 +12,7 @@ Ingested results are flagged (videos.via_search / channels.from_search) so the L
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
dropped from the search response up front)."""
import logging
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
@@ -69,11 +70,7 @@ def _classify_shorts(db: Session, videos: list[Video]) -> None:
# so DON'T finalise shorts_probed off a null duration; leave it for the scheduler
# (which only classifies enriched rows) to retry. Otherwise a real Short would leak.
continue
if (
v.live_status != "none"
or v.duration_seconds is None
or v.duration_seconds > probe_max
):
if v.live_status != "none" or v.duration_seconds is None or v.duration_seconds > probe_max:
v.is_short = False
v.shorts_probed = True
else:
@@ -165,7 +162,9 @@ def _ingest_candidates(
detail_by_id = {it["id"]: it for it in yt.get_videos(ids)}
except YouTubeError:
detail_by_id = {}
enriched_at = datetime.now(timezone.utc) # when THIS enrich ran (distinct from the stub placeholder)
enriched_at = datetime.now(
timezone.utc
) # when THIS enrich ran (distinct from the stub placeholder)
for v in videos:
item = detail_by_id.get(v.id)
if item is None:
@@ -180,9 +179,7 @@ def _ingest_candidates(
keep = {
v.id
for v in videos
if v.enriched_at is not None
and not v.is_short
and v.live_status not in LIVE_OR_UPCOMING
if v.enriched_at is not None and not v.is_short and v.live_status not in LIVE_OR_UPCOMING
}
return [vid for vid in ids if vid in keep]
@@ -256,9 +253,7 @@ def search_youtube(
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
blocked = set(
db.scalars(
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
).all()
db.scalars(select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)).all()
)
collected: list[str] = []
@@ -279,7 +274,9 @@ def search_youtube(
# dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY),
# which hides the real message and burns a second quota-costed search. Upstream failure
# on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1)
raise HTTPException(status_code=422, detail=f"YouTube search failed: {exc}") from exc
raise HTTPException(
status_code=422, detail=f"YouTube search failed: {exc}"
) from exc
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
candidates = [
@@ -365,16 +362,12 @@ def clear_search(
if not ids:
return {"removed": 0}
db.execute(
delete(SearchFind).where(
SearchFind.user_id == user.id, SearchFind.video_id.in_(ids)
)
delete(SearchFind).where(SearchFind.user_id == user.id, SearchFind.video_id.in_(ids))
)
db.commit()
has_state = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
in_playlist = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
subscribed = (
select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
)
subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
other_find = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists()
removed = db.execute(
delete(Video).where(
backend/app/routes/setup.py
+1
View File
@@ -5,6 +5,7 @@ mutating step goes through `require_setup`: it rejects once the instance is conf
the one-time setup token (printed to the container logs at first boot, carried by the wizard as the
X-Setup-Token header).
"""
from typing import Any
from fastapi import APIRouter, Depends, Header, HTTPException
backend/app/routes/sync.py
+6 -18
View File
@@ -15,9 +15,7 @@ router = APIRouter(prefix="/api/sync", tags=["sync"])
@router.post("/subscriptions")
def sync_subscriptions(
user: User = Depends(require_human), db: Session = Depends(get_db)
) -> dict:
def sync_subscriptions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
if not has_read_scope(user):
# No YouTube read grant yet — the onboarding wizard hasn't been completed.
raise HTTPException(
@@ -32,13 +30,9 @@ def sync_subscriptions(
@router.get("/status")
def sync_status(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
def sync_status(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
sub_count = db.scalar(
select(func.count())
.select_from(Subscription)
.where(Subscription.user_id == user.id)
select(func.count()).select_from(Subscription).where(Subscription.user_id == user.id)
)
return {
"subscriptions": sub_count,
@@ -58,13 +52,9 @@ def sync_status(
@router.get("/my-status")
def my_status(
user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
def my_status(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
"""Per-user sync progress: how far the channels *this* user subscribes to have synced."""
sub_join = and_(
Subscription.channel_id == Channel.id, Subscription.user_id == user.id
)
sub_join = and_(Subscription.channel_id == Channel.id, Subscription.user_id == user.id)
total, details_synced, recent_synced, deep_done = db.execute(
select(
func.count(Channel.id),
@@ -134,9 +124,7 @@ def deep_all(
) -> dict:
"""Opt every one of the user's channels into (or out of) full-history backfill."""
result = db.execute(
update(Subscription)
.where(Subscription.user_id == user.id)
.values(deep_requested=on)
update(Subscription).where(Subscription.user_id == user.id).values(deep_requested=on)
)
db.commit()
return {"updated": result.rowcount, "deep_requested": on}
backend/app/scheduler.py
+19 -8
View File
@@ -1,5 +1,6 @@
"""Background scheduler: free RSS detection, enrichment of new videos, and quota-aware
recent-first / deep backfill. Runs inside the API process (single worker)."""
import contextvars
import logging
import threading
@@ -154,7 +155,11 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
return
try:
audit.record(
db, actor_id, AuditAction.JOB_RUN, target_type="job", target_id=job_id,
db,
actor_id,
AuditAction.JOB_RUN,
target_type="job",
target_id=job_id,
summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""),
after={"status": status, "summary": summary},
)
@@ -173,7 +178,9 @@ def _job(name: str, fn) -> None:
return
db = SessionLocal()
actor_id = _manual_actor.get() # set only for a manual "run now" trigger
_record(name, last_started=_now_iso(), last_error=None, progress=None) # running already claimed
_record(
name, last_started=_now_iso(), last_error=None, progress=None
) # running already claimed
def sink(current, total, phase):
_record(name, progress={"current": current, "total": total, "phase": phase})
@@ -181,8 +188,13 @@ def _job(name: str, fn) -> None:
try:
if is_sync_paused(db):
logger.info("job %s skipped (sync paused)", name)
_record(name, status="skipped", last_finished=_now_iso(),
last_result="paused", progress=None)
_record(
name,
status="skipped",
last_finished=_now_iso(),
last_result="paused",
progress=None,
)
if actor_id is not None:
_notify_done(db, actor_id, name, "skipped", "sync paused")
_audit_run(db, actor_id, name, "skipped", "sync paused", None)
@@ -191,8 +203,7 @@ def _job(name: str, fn) -> None:
result = fn(db)
summary = _summarize(result)
logger.info("job %s -> %s", name, result)
_record(name, status="ok", last_finished=_now_iso(),
last_result=summary, progress=None)
_record(name, status="ok", last_finished=_now_iso(), last_result=summary, progress=None)
if actor_id is not None:
_notify_done(db, actor_id, name, "ok", summary)
_audit_run(db, actor_id, name, "ok", summary, result)
@@ -200,8 +211,7 @@ def _job(name: str, fn) -> None:
db.rollback()
logger.exception("job %s failed", name)
err = str(exc) or exc.__class__.__name__
_record(name, status="error", last_finished=_now_iso(),
last_error=err, progress=None)
_record(name, status="error", last_finished=_now_iso(), last_error=err, progress=None)
if actor_id is not None:
_notify_done(db, actor_id, name, "error", err)
_audit_run(db, actor_id, name, "error", err, None)
@@ -365,6 +375,7 @@ def _audit_gc_job() -> None:
# forever). Pure DB, no quota.
def work(db):
from app import sysconfig
return {"reaped": audit.gc(db, sysconfig.get_int(db, "audit_retention_days"))}
_job("audit_gc", work)
backend/app/security.py
+1
View File
@@ -28,6 +28,7 @@ def verify_password(password: str, hashed: str | None) -> bool:
# Any failure (mismatch, malformed hash) is a non-match — never raise to the caller.
return False
_fernet: Fernet | None = (
Fernet(settings.token_encryption_key.encode()) if settings.token_encryption_key else None
)
backend/app/state.py
+2
View File
@@ -1,4 +1,5 @@
"""Global admin-controlled app state (e.g. pausing background sync, first-run setup)."""
import logging
import secrets
@@ -39,6 +40,7 @@ def set_sync_paused(db: Session, paused: bool) -> None:
# --- First-run install wizard ------------------------------------------------------------
def is_configured(db: Session) -> bool:
return _row(db).configured
backend/app/sync/autotag.py
+2 -3
View File
@@ -5,6 +5,7 @@ detection over a sample of recent video titles. Topics are mapped from YouTube's
topicDetails categories and the channel's dominant video category. System tags are
regenerated freely; user tags are never touched here.
"""
import logging
import re
@@ -221,9 +222,7 @@ def apply_channel_autotags(db: Session, channel: Channel) -> None:
desired[tag.id] = confidence
for topic in compute_channel_topics(db, channel):
tag = ensure_system_tag(
db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR)
)
tag = ensure_system_tag(db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR))
desired.setdefault(tag.id, None)
existing = {
backend/app/sync/explore.py
+6 -17
View File
@@ -7,6 +7,7 @@ stay private to users who explored (or subscribe to) the channel — the feed qu
— so one user's curiosity never floods everyone's catalog. Subscribing is the "keep" signal;
otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a
grace period (mirrors the maintenance job's grace-then-delete safety)."""
import logging
from datetime import timedelta
@@ -35,11 +36,7 @@ def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
if not video_ids:
return 0
videos = (
db.execute(
select(Video).where(
Video.id.in_(video_ids), Video.enriched_at.is_(None)
)
)
db.execute(select(Video).where(Video.id.in_(video_ids), Video.enriched_at.is_(None)))
.scalars()
.all()
)
@@ -101,17 +98,11 @@ def purge_unkept_explored(db: Session) -> dict:
).rowcount
db.commit()
sub_exists = (
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
)
exp_exists = (
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
)
sub_exists = select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
exp_exists = select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
orphan_ids = (
db.execute(
select(Channel.id).where(
Channel.from_explore.is_(True), ~sub_exists, ~exp_exists
)
select(Channel.id).where(Channel.from_explore.is_(True), ~sub_exists, ~exp_exists)
)
.scalars()
.all()
@@ -158,9 +149,7 @@ def purge_unkept_search(db: Session, grace_days: int | None = None) -> dict:
cutoff = _now() - timedelta(days=grace)
state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
sub_exists = (
select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
)
sub_exists = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
deleted = db.execute(
delete(Video).where(
Video.via_search.is_(True),
backend/app/sync/maintenance.py
+5 -6
View File
@@ -19,6 +19,7 @@ flag an `upcoming` premiere that blew past its scheduled start and never went li
Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect the same
backfill reserve as the other jobs so interactive syncs never starve.
"""
import logging
from datetime import timedelta
@@ -132,11 +133,7 @@ def _notify_removed(db: Session, impact: dict[int, list[dict]]) -> int:
def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
"""Phase 1: recover or hard-delete videos already flagged unavailable."""
flagged = (
db.execute(select(Video).where(Video.unavailable_since.is_not(None)))
.scalars()
.all()
)
flagged = db.execute(select(Video).where(Video.unavailable_since.is_not(None))).scalars().all()
if not flagged:
return {"checked": 0, "recovered": 0, "deleted": 0, "notified": 0}
progress.report(0, len(flagged), "recheck")
@@ -181,7 +178,9 @@ def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
batch = sysconfig.get_int(db, "maintenance_revalidate_batch") # admin override, else config default
batch = sysconfig.get_int(
db, "maintenance_revalidate_batch"
) # admin override, else config default
flagged_new = 0
checked = 0
now = _now()
backend/app/sync/playlists.py
+4 -9
View File
@@ -5,6 +5,7 @@ edits out to YouTube (`plan_push` / `push_playlist`), behind the optional write
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
ingested (with a stub channel) so the mirror is faithful."""
import hashlib
import logging
from collections.abc import Callable
@@ -59,9 +60,7 @@ def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None
the API won't return are simply left out."""
if not video_ids:
return
have = set(
db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all()
)
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
if not missing:
return
@@ -109,9 +108,7 @@ def _replace_items_from_youtube(
ordered ids."""
_ensure_videos(db, yt, ids)
present = (
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all())
if ids
else set()
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
)
ordered: list[str] = []
seen: set[str] = set()
@@ -139,9 +136,7 @@ def sync_user_playlists(db: Session, user: User) -> dict:
existing = {
pl.yt_playlist_id: pl
for pl in db.execute(
select(Playlist).where(
Playlist.user_id == user.id, Playlist.source == "youtube"
)
select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube")
).scalars()
}
# YouTube playlists that we already own as local (exported) playlists must not be
backend/app/sync/runner.py
+5 -4
View File
@@ -4,6 +4,7 @@ Channel/video data is shared across users, so background work acts through a "se
user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configured for
public reads.
"""
import logging
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -173,9 +174,7 @@ def run_recent_backfill(
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
if channels is None:
channels = (
db.execute(select(Channel).where(Channel.recent_synced_at.is_(None)))
.scalars()
.all()
db.execute(select(Channel).where(Channel.recent_synced_at.is_(None))).scalars().all()
)
if max_channels:
channels = channels[:max_channels]
@@ -325,7 +324,9 @@ def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
total_units = sum(units(c) for c in pending)
daily_budget = max(
1, sysconfig.get_int(db, "quota_daily_budget") - sysconfig.get_int(db, "backfill_quota_reserve")
1,
sysconfig.get_int(db, "quota_daily_budget")
- sysconfig.get_int(db, "backfill_quota_reserve"),
)
eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
return {
backend/app/sync/subscriptions.py
+5 -7
View File
@@ -1,4 +1,5 @@
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
import logging
from datetime import datetime, timezone
@@ -57,9 +58,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None:
channel.title = snippet.get("title") or channel.title
channel.handle = snippet.get("customUrl")
channel.description = normalize_description(snippet.get("description"))
channel.thumbnail_url = (
best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
)
channel.thumbnail_url = best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
channel.default_language = snippet.get("defaultLanguage")
channel.country = snippet.get("country")
channel.uploads_playlist_id = content.get("relatedPlaylists", {}).get("uploads")
@@ -125,16 +124,15 @@ def import_subscriptions(db: Session, user: User) -> dict:
# (see `should_prune`).
removed = 0
existing = (
db.execute(select(Subscription).where(Subscription.user_id == user.id))
.scalars()
.all()
db.execute(select(Subscription).where(Subscription.user_id == user.id)).scalars().all()
)
prune_skipped = not should_prune(len(fetched), len(existing))
if prune_skipped:
log.warning(
"Subscription import (user %s): YouTube returned 0 subscriptions while %d exist "
"locally — skipping the prune (treating it as a transient empty response)",
user.id, len(existing),
user.id,
len(existing),
)
else:
for sub_row in existing:
backend/app/sync/videos.py
+11 -19
View File
@@ -1,6 +1,7 @@
"""Video ingestion: free RSS detection, recent-first (and deep) backfill from the
uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts
and livestream classification)."""
import re
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
@@ -16,9 +17,7 @@ from app.titles import normalize_title
from app.youtube.client import YouTubeClient, best_thumbnail
from app.youtube.shorts import make_client, probe_is_short
_DURATION_RE = re.compile(
r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
)
_DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
def _now() -> datetime:
@@ -177,9 +176,7 @@ def backfill_channel_deep(
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
rows = [
s
for s in (
stub_from_playlist_item(it, channel.id) for it in data.get("items", [])
)
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
if s is not None
]
total += insert_stubs(db, rows)
@@ -201,11 +198,7 @@ def reconcile_full_history(db: Session) -> int:
deep-requested, so the demand-driven deep job never ran). Without this such a channel
sits in 'needs full history' forever despite having every video. Idempotent, no API
quota; returns the number of channels flipped."""
stored = (
select(func.count(Video.id))
.where(Video.channel_id == Channel.id)
.scalar_subquery()
)
stored = select(func.count(Video.id)).where(Video.channel_id == Channel.id).scalar_subquery()
result = db.execute(
update(Channel)
.where(
@@ -263,9 +256,7 @@ def apply_video_details(video: Video, item: dict) -> None:
tags = snippet.get("tags")
video.keywords = " ".join(tags) if tags else None
video.topic_categories = topics.get("topicCategories")
video.default_language = snippet.get("defaultLanguage") or snippet.get(
"defaultAudioLanguage"
)
video.default_language = snippet.get("defaultLanguage") or snippet.get("defaultAudioLanguage")
video.live_status = _live_status(snippet, live)
# status part: used by the maintenance/validation job to judge playability.
if status:
@@ -309,9 +300,7 @@ def _mark_enriched(video: Video) -> None:
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
limit = limit or sysconfig.get_int(db, "enrich_batch_size")
videos = (
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit))
.scalars()
.all()
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit)).scalars().all()
)
return _apply_video_batch(db, yt, list(videos), _mark_enriched)
@@ -377,12 +366,14 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
# 2) Probe the remaining candidates.
candidates = (
db.execute(
select(Video).where(
select(Video)
.where(
Video.shorts_probed.is_(False),
Video.enriched_at.is_not(None),
Video.duration_seconds <= probe_max,
Video.live_status == "none",
).limit(limit)
)
.limit(limit)
)
.scalars()
.all()
@@ -393,6 +384,7 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
probed = 0
shorts = 0
with make_client() as client:
def work(video: Video):
return video, probe_is_short(client, video.id)
backend/app/sysconfig.py
+69 -13
View File
@@ -7,6 +7,7 @@ bounds, and whether it's a secret. The resolver returns the DB override if one i
stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secrets can't be
stored in the DB and stay env-only).
"""
import logging
from dataclasses import dataclass
@@ -46,15 +47,35 @@ SPECS: tuple[ConfigSpec, ...] = (
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
# --- Quota (shared YouTube Data API daily budget) ---
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000),
ConfigSpec("search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000),
ConfigSpec(
"backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000
),
ConfigSpec(
"search_daily_limit_per_user",
"int",
"quota",
"search_daily_limit_per_user",
min=0,
max=100_000,
),
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
ConfigSpec("search_source", "str", "quota", "search_source"),
# --- Backfill (recent-first first pass per channel) ---
ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000),
ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500),
ConfigSpec(
"backfill_recent_max_videos",
"int",
"backfill",
"backfill_recent_max_videos",
min=1,
max=100_000,
),
ConfigSpec(
"backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500
),
# --- Shorts probe ---
ConfigSpec("shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600),
ConfigSpec(
"shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600
),
ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000),
# --- Batch sizes ---
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
@@ -62,7 +83,14 @@ SPECS: tuple[ConfigSpec, ...] = (
# --- Maintenance (rolling re-validation of already-known videos) ---
# Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin
# knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard.
ConfigSpec("maintenance_revalidate_batch", "int", "maintenance", "maintenance_revalidate_batch", min=100, max=100_000),
ConfigSpec(
"maintenance_revalidate_batch",
"int",
"maintenance",
"maintenance_revalidate_batch",
min=100,
max=100_000,
),
# --- Discovery cleanup (search + explore ephemeral content) ---
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
@@ -80,13 +108,40 @@ SPECS: tuple[ConfigSpec, ...] = (
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0),
ConfigSpec("download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0),
ConfigSpec("download_default_max_concurrent", "int", "downloads", "download_default_max_concurrent", min=1, max=20),
ConfigSpec("download_default_max_jobs", "int", "downloads", "download_default_max_jobs", min=1, max=100_000),
ConfigSpec("download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650),
ConfigSpec("download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650),
ConfigSpec(
"download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0
),
ConfigSpec(
"download_default_max_concurrent",
"int",
"downloads",
"download_default_max_concurrent",
min=1,
max=20,
),
ConfigSpec(
"download_default_max_jobs",
"int",
"downloads",
"download_default_max_jobs",
min=1,
max=100_000,
),
ConfigSpec(
"download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650
),
ConfigSpec(
"download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650
),
ConfigSpec("download_layout", "str", "downloads", "download_layout"),
ConfigSpec("download_worker_concurrency", "int", "downloads", "download_worker_concurrency", min=1, max=16),
ConfigSpec(
"download_worker_concurrency",
"int",
"downloads",
"download_worker_concurrency",
min=1,
max=16,
),
ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"),
# --- Audit log ---
# Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever).
@@ -155,7 +210,8 @@ def _raw_override(db: Session, key: str) -> str | None:
_decrypt_warned.add(key)
log.warning(
"config %r could not be decrypted (TOKEN_ENCRYPTION_KEY changed or missing?) — falling "
"back to the env/config default; the key must not change once secrets are stored", key,
"back to the env/config default; the key must not change once secrets are stored",
key,
)
return None
backend/app/titles.py
+80 -7
View File
@@ -19,6 +19,7 @@ Rules (user-approved "option b"):
an otherwise normal title is left alone.
4. Collapse repeated punctuation (!!! → !) and whitespace.
"""
import re
import unicodedata
@@ -47,15 +48,87 @@ _REPEAT_PUNCT = re.compile(r"([!?.,\-])\1{1,}")
# Short function words → lowercased when the title is de-shouted (proper title-case style).
_SHORT_LOWER = {
"to", "the", "and", "of", "a", "an", "in", "on", "at", "for", "or", "vs", "the",
"és", "az", "egy", "meg", "nem", "hogy", "de", "ha", "der", "die", "das", "und", "von",
"to",
"the",
"and",
"of",
"a",
"an",
"in",
"on",
"at",
"for",
"or",
"vs",
"the",
"és",
"az",
"egy",
"meg",
"nem",
"hogy",
"de",
"ha",
"der",
"die",
"das",
"und",
"von",
}
# Kept as-is even when everything else is de-shouted (common acronyms; compared lowercased).
_ACRONYMS = {
"ps", "ai", "pc", "tv", "hd", "4k", "8k", "uk", "eu", "us", "usa", "gta", "rpg", "fps",
"diy", "vr", "ar", "id", "ok", "faq", "nasa", "fbi", "cia", "ceo", "amd", "pdf", "usb",
"gps", "api", "dj", "mc", "suv", "gpu", "cpu", "ssd", "hdmi", "led", "ufo", "dna", "nba",
"nfl", "asmr", "pov", "diy", "wtf", "lol", "rtx", "gtx", "ios", "mmo", "vip", "hp",
"ps",
"ai",
"pc",
"tv",
"hd",
"4k",
"8k",
"uk",
"eu",
"us",
"usa",
"gta",
"rpg",
"fps",
"diy",
"vr",
"ar",
"id",
"ok",
"faq",
"nasa",
"fbi",
"cia",
"ceo",
"amd",
"pdf",
"usb",
"gps",
"api",
"dj",
"mc",
"suv",
"gpu",
"cpu",
"ssd",
"hdmi",
"led",
"ufo",
"dna",
"nba",
"nfl",
"asmr",
"pov",
"diy",
"wtf",
"lol",
"rtx",
"gtx",
"ios",
"mmo",
"vip",
"hp",
}
@@ -98,7 +171,7 @@ def normalize_title(raw: str | None) -> str | None:
if shouting: # ensure the first letter is capitalized (a leading function word got lowered)
for i, ch in enumerate(t):
if ch.isalpha():
t = t[:i] + ch.upper() + t[i + 1:]
t = t[:i] + ch.upper() + t[i + 1 :]
break
# Nothing survived cleaning: fall back to the prefix+hashtag-stripped text (so neither the
# stats prefix nor the SEO hashtags are resurrected). Only use the untouched raw when no prefix
backend/app/userscope.py
+1
View File
@@ -5,6 +5,7 @@ the messaging directory AND the download share-by-email / recipient-picker endpo
place so the two forms (SQL clauses for a query, Python check for a loaded row) can't drift apart,
and so a route module doesn't have to import another route module just to reuse the rule.
"""
from app.models import User
backend/app/utils.py
+2
View File
@@ -1,4 +1,5 @@
"""Small cross-cutting helpers shared across modules (kept dependency-light on purpose)."""
import re
from datetime import date, datetime, timezone
@@ -10,6 +11,7 @@ def iso(dt: date | None) -> str | None:
too (datetime is a subclass) — e.g. MediaAsset.upload_date is a plain `date`."""
return dt.isoformat() if dt is not None else None
# Single source of truth for "looks like an email" — used by every endpoint that accepts an
# address (auth register/reset/demo, admin invites + demo whitelist, the setup wizard). Keep
# the strictness in one place so routes don't drift between this and a looser "@"-in check.
backend/app/worker.py
+35 -12
View File
@@ -15,6 +15,7 @@ Design:
yt-dlp download if it's no longer `running`.
* Crash-safe: on startup, orphaned `running` jobs are requeued and `downloading` assets reset.
"""
import logging
import os
import shutil
@@ -53,6 +54,7 @@ def _handle_signal(signum, frame):
# --- claim + recovery -----------------------------------------------------------------------
def _wait_for_schema(timeout: float = 300.0) -> bool:
"""Block until the download tables exist before doing any DB work. Returns False if they never
appeared (or we were asked to stop while waiting).
@@ -74,7 +76,8 @@ def _wait_for_schema(timeout: float = 300.0) -> bool:
log.error(
"download schema still absent after %ss — the API's migrations have not landed; "
"exiting so the container restarts: %s",
timeout, str(exc)[:120],
timeout,
str(exc)[:120],
)
return False
log.info("waiting for the database schema (API migrations)…")
@@ -135,9 +138,7 @@ def _claim_job() -> int | None:
),
{"dmc": default_mc},
).fetchall()
chosen = next(
(jid for jid, uid in rows if not quota.at_concurrency_limit(db, uid)), None
)
chosen = next((jid for jid, uid in rows if not quota.at_concurrency_limit(db, uid)), None)
if chosen is None:
db.commit() # release the row locks; nothing eligible right now
return None
@@ -308,7 +309,12 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
except Exception: # noqa: BLE001 — a watchdog error must NEVER leave the child running unsupervised
log.exception("supervisor loop failed for job %s; stopping the child", job_id)
_fail_after_kill(
job_id, asset_id, proc, grace_s, "The download was stopped after a worker error.", "unknown"
job_id,
asset_id,
proc,
grace_s,
"The download was stopped after a worker error.",
"unknown",
)
return
@@ -324,24 +330,37 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
if status == "running":
_cleanup_staging(asset_id)
_set_job(
job_id, status="error", error_code="unknown", phase=None,
job_id,
status="error",
error_code="unknown",
phase=None,
error=f"The download process exited unexpectedly (code {proc.returncode}).",
)
log.warning("job %s: child exited %s without finishing", job_id, proc.returncode)
log.warning(
"job %s: child exited %s without finishing", job_id, proc.returncode
)
except Exception: # noqa: BLE001 — boot recovery is the backstop for a stuck 'running' job
log.exception("post-exit reconciliation failed for job %s", job_id)
return
if reason == "stalled":
_fail_after_kill(
job_id, asset_id, proc, grace_s,
f"The download stalled (no progress for {int(stall_s)}s) and was stopped.", "network",
job_id,
asset_id,
proc,
grace_s,
f"The download stalled (no progress for {int(stall_s)}s) and was stopped.",
"network",
)
return
if reason == "timeout":
_fail_after_kill(
job_id, asset_id, proc, grace_s,
f"The download exceeded the {int(wall_s)}s time limit and was stopped.", "network",
job_id,
asset_id,
proc,
grace_s,
f"The download exceeded the {int(wall_s)}s time limit and was stopped.",
"network",
)
return
# cancelled / abandon / shutdown: stop + free the asset, but leave the job's own DB status.
@@ -349,7 +368,9 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
if reason == "shutdown":
_requeue_if_running(job_id, None) # resume on the next boot
elif reason == "abandon":
log.warning("job %s: killed a child wedged past the wall-clock; DB status left intact", job_id)
log.warning(
"job %s: killed a child wedged past the wall-clock; DB status left intact", job_id
)
else: # cancelled
log.info("job %s: child stopped (route pause/cancel)", job_id)
@@ -363,6 +384,7 @@ def _cleanup_staging(asset_id: int) -> None:
# --- processing -----------------------------------------------------------------------------
def _process_job(job_id: int) -> None:
"""Dispatch a claimed job. The trivial outcomes (cache hit, dead/contended asset) are handled
inline and cheaply; the actual heavy work (yt-dlp download / ffmpeg edit) is claimed here and
@@ -423,6 +445,7 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
# --- loop -----------------------------------------------------------------------------------
def _worker_loop(idx: int) -> None:
while not _stop.is_set():
try:
backend/app/youtube/client.py
+7 -12
View File
@@ -5,6 +5,7 @@ Public reads (channels/videos/playlistItems) use the configured API key when ava
so they don't depend on a specific user's token; subscriptions.list?mine=true always
uses OAuth.
"""
import logging
from collections.abc import Iterator
from datetime import datetime, timedelta, timezone
@@ -251,9 +252,7 @@ class YouTubeClient:
resp.status_code,
resp.text[:200],
)
raise YouTubeError(
f"DELETE subscriptions -> {resp.status_code}: {resp.text[:300]}"
)
raise YouTubeError(f"DELETE subscriptions -> {resp.status_code}: {resp.text[:300]}")
def insert_subscription(self, channel_id: str) -> str:
"""Subscribe to a channel on YouTube. Requires the write scope and the user's
@@ -264,11 +263,7 @@ class YouTubeClient:
"POST",
"subscriptions",
params={"part": "snippet"},
json={
"snippet": {
"resourceId": {"kind": "youtube#channel", "channelId": channel_id}
}
},
json={"snippet": {"resourceId": {"kind": "youtube#channel", "channelId": channel_id}}},
)
return data.get("id", "")
@@ -284,9 +279,7 @@ class YouTubeClient:
# --- write endpoints (OAuth only, never the API key; each costs 50 quota units) ---
def _write(self, method: str, path: str, *, params: dict, json: dict | None = None) -> dict:
headers = {"Authorization": f"Bearer {self._access_token()}"}
resp = self._send(
method, f"{API_BASE}/{path}", params=params, json=json, headers=headers
)
resp = self._send(method, f"{API_BASE}/{path}", params=params, json=json, headers=headers)
quota.record_usage(50)
if resp.status_code not in (200, 204):
log.warning("YouTube %s %s -> %s: %s", method, path, resp.status_code, resp.text[:200])
@@ -327,7 +320,9 @@ class YouTubeClient:
)
return data["id"]
def add_playlist_item(self, playlist_id: str, video_id: str, position: int | None = None) -> str:
def add_playlist_item(
self, playlist_id: str, video_id: str, position: int | None = None
) -> str:
"""Append (or insert at `position`) a video into a playlist; returns the new
playlistItem id. 50 units."""
snippet = {
backend/app/youtube/rss.py
+1
View File
@@ -1,5 +1,6 @@
"""Free per-channel RSS feed reader (no API quota). Returns the channel's most recent
~15 uploads as lightweight stubs for fast fresh-video detection."""
from datetime import datetime, timezone
import feedparser
backend/app/youtube/search_scrape.py
+3 -11
View File
@@ -16,6 +16,7 @@ only difference is the cursor is an InnerTube *continuation token*, not a Data A
``published_at`` is left ``None`` (the scrape only exposes a relative time like "3 years ago");
the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"``
and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path)."""
import re
import httpx
@@ -88,12 +89,7 @@ def _video_item(vr: dict) -> dict | None:
channel_id = channel_title = None
if runs:
channel_title = runs[0].get("text")
channel_id = (
runs[0]
.get("navigationEndpoint", {})
.get("browseEndpoint", {})
.get("browseId")
)
channel_id = runs[0].get("navigationEndpoint", {}).get("browseEndpoint", {}).get("browseId")
thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or []
return {
"id": vid,
@@ -129,11 +125,7 @@ def _find_continuation(obj) -> str | None:
if isinstance(obj, dict):
cir = obj.get("continuationItemRenderer")
if isinstance(cir, dict):
tok = (
cir.get("continuationEndpoint", {})
.get("continuationCommand", {})
.get("token")
)
tok = cir.get("continuationEndpoint", {}).get("continuationCommand", {}).get("token")
if tok:
return tok
for v in obj.values():
backend/app/youtube/shorts.py
+2 -3
View File
@@ -2,6 +2,7 @@
A real Short returns HTTP 200 at that URL; a regular video redirects to /watch.
This uses no API quota."""
import httpx
SHORTS_URL = "https://www.youtube.com/shorts/{video_id}"
@@ -16,9 +17,7 @@ _HEADERS = {
def probe_is_short(client: httpx.Client, video_id: str) -> bool | None:
"""True if a Short, False if a regular video, None if undetermined (retry later)."""
try:
resp = client.get(
SHORTS_URL.format(video_id=video_id), follow_redirects=False
)
resp = client.get(SHORTS_URL.format(video_id=video_id), follow_redirects=False)
except httpx.HTTPError:
return None
if resp.status_code == 200: