Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9b87838e3a | ||
|
|
f014416e31 | ||
|
|
3369587412 | ||
|
|
9af4e8f4ff | ||
|
|
a0b725d779 | ||
|
|
bc33b2ed16 | ||
|
|
5280949667 | ||
|
|
4892c38466 | ||
|
|
9ae6c47d6c | ||
|
|
b7a6451dba | ||
|
|
11e32163b8 | ||
|
|
257dc5741d |
@@ -0,0 +1,46 @@
|
||||
"""download_credentials (S6: per-user, per-host yt-dlp cookie/auth store)
|
||||
|
||||
Each user stores their OWN login cookies (Netscape cookies.txt) for an auth-walled download host
|
||||
(Facebook, Instagram, …), Fernet-encrypted at rest. A download of a yt-dlp-supported, auth-walled
|
||||
link on that host is retried with the acting user's cookie. Unique per (user_id, host_key) so a
|
||||
user holds at most one credential per host. Empty for every existing install; forward-only in
|
||||
practice (a downgrade just drops the table + its secrets).
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0074_download_credentials"
|
||||
down_revision: Union[str, None] = "0073_saved_view_source"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"download_credentials",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("host_key", sa.String(length=64), nullable=False),
|
||||
sa.Column("cookies_enc", sa.Text(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.UniqueConstraint("user_id", "host_key", name="uq_download_credential"),
|
||||
)
|
||||
op.create_index("ix_download_credentials_user_id", "download_credentials", ["user_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_download_credentials_user_id", table_name="download_credentials")
|
||||
op.drop_table("download_credentials")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""playlist synced_video_ids (S7: 3-way merge baseline for two-way YouTube sync)
|
||||
|
||||
The one-directional "local wins" push is replaced by a 3-way merge that classifies each item
|
||||
against the last-synced state. That needs the raw ordered id set as a baseline, not just the
|
||||
`synced_fingerprint` hash. Add a nullable `synced_video_ids` (JSON list) column; NULL on every
|
||||
existing row means "no baseline yet" — the first merge then unions losslessly and records one.
|
||||
Forward-only in practice (a downgrade just drops the column).
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0075_playlist_sync_baseline"
|
||||
down_revision: Union[str, None] = "0074_download_credentials"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("playlists", sa.Column("synced_video_ids", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("playlists", "synced_video_ids")
|
||||
@@ -44,6 +44,9 @@ class AuditAction:
|
||||
SYNC_RESUME = "sync_resume"
|
||||
# Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic)
|
||||
JOB_RUN = "job_run"
|
||||
# Per-user download cookie/auth credential set or removed (the value is NEVER recorded)
|
||||
DOWNLOAD_CREDENTIAL_SET = "download_credential_set"
|
||||
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
||||
AUDIT_CLEAR = "audit_clear"
|
||||
|
||||
|
||||
@@ -186,6 +186,21 @@ class Settings(BaseSettings):
|
||||
download_supervisor_poll_seconds: int = 3
|
||||
# Grace between SIGTERM and SIGKILL when stopping a child (matches run_ffmpeg's escalation).
|
||||
download_kill_grace_seconds: int = 5
|
||||
# --- worker-child resource de-prioritization ---
|
||||
# The download/edit child (yt-dlp + ffmpeg) shares ONE LAN box with the API, Postgres, and Plex
|
||||
# playback. Without a priority handicap a CPU re-encode pegs every core for ~30s and starves web
|
||||
# requests + playback until it finishes. The child lowers its OWN priority at startup — inherited
|
||||
# by every ffmpeg/yt-dlp descendant — so any foreground work always wins. Each knob is best-effort
|
||||
# and independently disable-able with 0.
|
||||
download_child_nice: int = 10 # CPU nice increment applied to the child; 0 = don't renice
|
||||
download_child_ionice_class: int = 3 # best-effort IO class (3 = idle); 0 = leave IO untouched
|
||||
# Cap the browser-safe re-encode's encoder threads so a transcode can't claim every core. 0 =
|
||||
# reserve one core for foreground work (max(1, cpu_count - 1)); >0 pins exactly that many.
|
||||
download_ffmpeg_threads: int = 0
|
||||
# Throttle the retention GC's whole-tree orphan file sweep so its rglob can't monopolize disk IO
|
||||
# in the API process: sleep `pause_ms` after every `batch` files touched. pause_ms=0 disables.
|
||||
download_gc_sweep_pause_ms: int = 20
|
||||
download_gc_sweep_batch: int = 400
|
||||
# Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the
|
||||
# least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit).
|
||||
download_total_max_bytes: int = 0
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Per-user, per-host download cookies — the single source of truth for "does the acting user's
|
||||
cookie apply to this download, and how does that keep the result private".
|
||||
|
||||
Two call sites depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
in prose):
|
||||
* `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user.
|
||||
* the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it
|
||||
is downloading into is that private, salted identity — so a cookie can never poison a shared
|
||||
asset (the load-bearing cross-user privacy fix).
|
||||
|
||||
Both go through `cookie_salt` / `authed_format_sig` / `host_key_for_url` here, so they can't drift.
|
||||
|
||||
Security controls enforced here:
|
||||
* **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated
|
||||
registry via the netloc-scoped `_host_is`, so a cookie is only ever offered to the exact site it
|
||||
belongs to; an arbitrary `?url=` (or a look-alike host) resolves to no host_key and gets no
|
||||
cookie.
|
||||
* **secret handling** — cookies are Fernet-encrypted at rest; the blob is only ever decrypted
|
||||
in-worker into a 0600 file that is unlinked in a `finally`, and is never returned by a route,
|
||||
logged, or written to a job's error/snapshot.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential
|
||||
from app.security import decrypt_optional, encrypt
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.credentials")
|
||||
|
||||
|
||||
class HostSpec:
|
||||
"""A supported cookie host: a short canonical `key` (stored on the credential + folded into the
|
||||
cache salt), a human `label` for the UI, and the netloc `domains` it matches."""
|
||||
|
||||
__slots__ = ("key", "label", "domains")
|
||||
|
||||
def __init__(self, key: str, label: str, domains: tuple[str, ...]):
|
||||
self.key = key
|
||||
self.label = label
|
||||
self.domains = domains
|
||||
|
||||
|
||||
# Curated registry (user's call: seed the common auth-walled hosts now). One `key` can cover several
|
||||
# domains of the same site. Add a host by adding a row — the FE dropdown, host-scope guard, and salt
|
||||
# all derive from this, so there is nothing else to wire up.
|
||||
SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
|
||||
HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
|
||||
HostSpec("instagram", "Instagram", ("instagram.com",)),
|
||||
HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
|
||||
HostSpec("tiktok", "TikTok", ("tiktok.com",)),
|
||||
HostSpec("reddit", "Reddit", ("reddit.com",)),
|
||||
)
|
||||
|
||||
_BY_KEY: dict[str, HostSpec] = {h.key: h for h in SUPPORTED_HOSTS}
|
||||
|
||||
|
||||
def is_supported_host(host_key: str) -> bool:
|
||||
return host_key in _BY_KEY
|
||||
|
||||
|
||||
def host_label(host_key: str) -> str:
|
||||
spec = _BY_KEY.get(host_key)
|
||||
return spec.label if spec else host_key
|
||||
|
||||
|
||||
def host_key_for_url(url: str | None) -> str | None:
|
||||
"""The supported host_key whose domains own `url`'s netloc, or None. Netloc-scoped (via
|
||||
`_host_is`), so a look-alike host ('myfacebook.com', 'facebook.com.evil') never matches. When
|
||||
several specs could match, the most-specific (longest matched domain) wins so the result is
|
||||
deterministic across the enqueue and worker call sites."""
|
||||
if not url:
|
||||
return None
|
||||
from urllib.parse import urlparse
|
||||
|
||||
host = urlparse(url).netloc.lower()
|
||||
if not host:
|
||||
return None
|
||||
best_key: str | None = None
|
||||
best_len = -1
|
||||
for spec in SUPPORTED_HOSTS:
|
||||
for dom in spec.domains:
|
||||
if _host_is(host, dom) and len(dom) > best_len:
|
||||
best_key, best_len = spec.key, len(dom)
|
||||
return best_key
|
||||
|
||||
|
||||
def cookie_salt(user_id: int, host_key: str) -> str:
|
||||
"""The cache-signature salt for a cookie-authed download. Keyed by user id + host so the authed
|
||||
asset is private to that user (never dedup-shared) and stable across that user's own re-adds."""
|
||||
return f"cookie:{user_id}:{host_key}"
|
||||
|
||||
|
||||
def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str:
|
||||
"""The salted `format_sig` an authed download for (user, host) resolves to — the private asset
|
||||
identity. Both enqueue (to create it) and the worker (to recognise it) compute it HERE."""
|
||||
return formats.format_sig(spec, salt=cookie_salt(user_id, host_key))
|
||||
|
||||
|
||||
def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||
"""At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a
|
||||
normal shared download). Returns the salt string so the caller passes it straight to
|
||||
`format_sig(spec, salt=...)`."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return None
|
||||
if get_credential(db, user_id, host_key) is None:
|
||||
return None
|
||||
return cookie_salt(user_id, host_key)
|
||||
|
||||
|
||||
# --- storage (CRUD) ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None:
|
||||
return db.execute(
|
||||
select(DownloadCredential).where(
|
||||
DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
class InvalidCookies(ValueError):
|
||||
"""The pasted cookies aren't a usable Netscape cookies.txt — surfaced as a 400 by the route."""
|
||||
|
||||
|
||||
def validate_cookies(raw: str) -> str:
|
||||
"""Sanity-check a pasted cookies.txt (Netscape format) and return the cleaned text. yt-dlp reads
|
||||
the Netscape format: comment/`#` lines plus TAB-separated cookie lines (domain, flag, path,
|
||||
secure, expiry, name, value). We don't fully parse — just reject obvious non-cookie paste (empty,
|
||||
or no tab-separated data line) so a mistaken paste fails HERE with a clear message rather than
|
||||
silently producing a cookie file yt-dlp ignores."""
|
||||
text = (raw or "").strip()
|
||||
if not text:
|
||||
raise InvalidCookies("empty")
|
||||
has_data_line = any(
|
||||
line.strip() and not line.lstrip().startswith("#") and "\t" in line
|
||||
for line in text.splitlines()
|
||||
)
|
||||
if not has_data_line:
|
||||
raise InvalidCookies("format")
|
||||
# Netscape cookie files must end with a newline for some parsers; normalise.
|
||||
return text if text.endswith("\n") else text + "\n"
|
||||
|
||||
|
||||
def upsert_credential(db: Session, user_id: int, host_key: str, raw: str) -> DownloadCredential:
|
||||
"""Create or replace this user's cookie blob for `host_key`. Caller has validated host_key is
|
||||
supported and `raw` via `validate_cookies`. Does NOT commit (the route commits with the audit
|
||||
row so the trail can't drift). The blob is encrypted here; the plaintext never leaves this call."""
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
enc = encrypt(raw)
|
||||
if cred is None:
|
||||
cred = DownloadCredential(user_id=user_id, host_key=host_key, cookies_enc=enc)
|
||||
db.add(cred)
|
||||
else:
|
||||
cred.cookies_enc = enc
|
||||
return cred
|
||||
|
||||
|
||||
def delete_credential(db: Session, user_id: int, host_key: str) -> bool:
|
||||
"""Delete this user's cookie for `host_key`. Returns whether a row existed. Does NOT commit."""
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return False
|
||||
db.delete(cred)
|
||||
return True
|
||||
|
||||
|
||||
def status_list(db: Session, user_id: int) -> list[dict]:
|
||||
"""The supported hosts with this user's configured/updated state — the write-only Settings view.
|
||||
Never includes the blob."""
|
||||
have = {
|
||||
c.host_key: c
|
||||
for c in db.execute(
|
||||
select(DownloadCredential).where(DownloadCredential.user_id == user_id)
|
||||
).scalars()
|
||||
}
|
||||
out = []
|
||||
for spec in SUPPORTED_HOSTS:
|
||||
cred = have.get(spec.key)
|
||||
out.append(
|
||||
{
|
||||
"host_key": spec.key,
|
||||
"label": spec.label,
|
||||
"configured": cred is not None,
|
||||
"updated_at": cred.updated_at.isoformat() if cred is not None else None,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# --- worker: resolve + write the cookie file --------------------------------------------------
|
||||
|
||||
|
||||
def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset_format_sig: str,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
"""In the worker child: the cookies.txt to hand yt-dlp for this download, or None.
|
||||
|
||||
The double gate that prevents cross-user leakage:
|
||||
1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND
|
||||
2. the asset being downloaded into must BE this user's private, salted identity
|
||||
(`asset_format_sig == authed_format_sig(...)`).
|
||||
|
||||
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||
InvalidToken) also yields None; the download then fails honestly on the login wall (S3's
|
||||
`login_required`)."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return None
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return None
|
||||
if asset_format_sig != authed_format_sig(spec, user_id, host_key):
|
||||
return None # shared asset — never attach a cookie to a non-private identity
|
||||
raw = decrypt_optional(cred.cookies_enc)
|
||||
if not raw:
|
||||
log.warning(
|
||||
"download cookie for user %s host %s could not be decrypted (rotated key?) — "
|
||||
"proceeding without it",
|
||||
user_id,
|
||||
host_key,
|
||||
)
|
||||
return None
|
||||
return _write_cookiefile(raw, staging)
|
||||
|
||||
|
||||
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
||||
"""Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path.
|
||||
|
||||
Deliberately NOT in the staging root: the download retry loop unlinks every file it finds there,
|
||||
and `_produced_file` picks the largest file there as the output — either would corrupt a
|
||||
cookies.txt in the root. A `.auth` SUBDIR survives both (they skip directories) and is still
|
||||
removed by the parent worker's staging cleanup on a hard kill. The caller unlinks it in a
|
||||
`finally` on the normal/exception paths."""
|
||||
auth_dir = staging / ".auth"
|
||||
auth_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
path = auth_dir / "cookies.txt"
|
||||
# Open with 0600 from the start (don't create world-readable then chmod).
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
try:
|
||||
os.write(fd, raw.encode("utf-8"))
|
||||
finally:
|
||||
os.close(fd)
|
||||
return path
|
||||
@@ -25,6 +25,7 @@ import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -32,9 +33,21 @@ import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
log = logging.getLogger("siftlode.edit")
|
||||
|
||||
|
||||
def encode_thread_cap() -> int:
|
||||
"""How many encoder threads a browser-safe re-encode may use. `download_ffmpeg_threads` when the
|
||||
admin pinned one (>0), else reserve a core for foreground work (max(1, cpu_count - 1)). Shared by
|
||||
the ffmpeg `-threads` flag and the child's `OMP_NUM_THREADS` so both honour the same budget."""
|
||||
n = settings.download_ffmpeg_threads
|
||||
if n > 0:
|
||||
return n
|
||||
return max(1, (os.cpu_count() or 2) - 1)
|
||||
|
||||
|
||||
class EditAborted(Exception):
|
||||
"""Raised by run_ffmpeg when the job was paused/canceled mid-encode."""
|
||||
|
||||
@@ -510,11 +523,12 @@ def browser_transcode_flags(
|
||||
|
||||
|
||||
def build_browser_safe_cmd(
|
||||
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool
|
||||
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool, threads: int | None = None
|
||||
) -> list[str]:
|
||||
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
||||
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
||||
pipe:1` streams machine-readable progress for run_ffmpeg."""
|
||||
pipe:1` streams machine-readable progress for run_ffmpeg. `threads` caps the encoder so a
|
||||
re-encode can't claim every core on the shared box (see the worker-child de-prioritization)."""
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
@@ -534,6 +548,8 @@ def build_browser_safe_cmd(
|
||||
else ["-c:v", "copy"]
|
||||
)
|
||||
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
||||
if threads and threads > 0:
|
||||
cmd += ["-threads", str(threads)]
|
||||
cmd += ["-movflags", "+faststart", str(dest)]
|
||||
return cmd
|
||||
|
||||
|
||||
@@ -24,6 +24,16 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
# Some extractors (e.g. Reddit) gate content behind a logged-in account: "Account
|
||||
# authentication is required." — an auth wall we deliberately don't cross, not a transient error.
|
||||
("authentication is required", "login_required"),
|
||||
# Closed-group / private-post auth walls (Facebook groups, Instagram private, …). yt-dlp phrases
|
||||
# these many ways; the common thread is "you must be signed in" or its own suggestion to supply
|
||||
# cookies. Treating the cookie suggestion as an auth signal is what makes a login-walled link read
|
||||
# honestly (and stay non-retryable) when the acting user has no usable cookie for that host.
|
||||
("log in to", "login_required"),
|
||||
("login required", "login_required"),
|
||||
("you must be logged in", "login_required"),
|
||||
("only available for registered users", "login_required"),
|
||||
("use --cookies", "login_required"),
|
||||
("--cookies-from-browser", "login_required"),
|
||||
# Geo-restricted.
|
||||
("in your country", "geo_blocked"),
|
||||
("available in your location", "geo_blocked"),
|
||||
@@ -42,6 +52,10 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
("account associated with this video has been terminated", "unavailable"),
|
||||
("this video is not available", "unavailable"),
|
||||
("this video has been removed", "unavailable"),
|
||||
# yt-dlp doesn't support this site at all ("Unsupported URL: …") — distinct from an auth wall on a
|
||||
# supported site. Non-retryable: no cookie or retry can make an unsupported host downloadable.
|
||||
("unsupported url", "unsupported_host"),
|
||||
("no video could be found in this url", "unsupported_host"),
|
||||
# DRM.
|
||||
("drm protected", "drm"),
|
||||
("drm-protected", "drm"),
|
||||
|
||||
@@ -98,12 +98,17 @@ def build_ydl_opts(
|
||||
outtmpl: str,
|
||||
progress_hook,
|
||||
extra: dict | None = None,
|
||||
cookiefile: str | None = None,
|
||||
) -> dict:
|
||||
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
||||
|
||||
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
|
||||
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||
generic options below. Everything here is source-agnostic."""
|
||||
generic options below. Everything here is source-agnostic.
|
||||
|
||||
`cookiefile` is the acting user's per-host cookies.txt path (S6), attached ONLY by the worker
|
||||
after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra`
|
||||
merge so a source adapter can never clobber it."""
|
||||
s = normalize(spec)
|
||||
h = s["max_height"]
|
||||
opts: dict = {
|
||||
@@ -132,6 +137,11 @@ def build_ydl_opts(
|
||||
if extra:
|
||||
opts.update(extra)
|
||||
|
||||
# The acting user's per-host cookie (S6). AFTER the adapter merge on purpose: an adapter must not
|
||||
# be able to override the authenticated session the worker resolved for this specific user.
|
||||
if cookiefile:
|
||||
opts["cookiefile"] = cookiefile
|
||||
|
||||
if s["mode"] == "a":
|
||||
opts["format"] = "bestaudio/best"
|
||||
opts["postprocessors"].append(
|
||||
|
||||
@@ -86,11 +86,25 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
|
||||
cutoff = time.time() - _ORPHAN_MIN_AGE_S
|
||||
removed = 0
|
||||
# Throttle the whole-tree walk: this runs in the API process, and an unpaced rglob+stat over a
|
||||
# large download root monopolizes disk IO long enough to make web requests + playback stutter.
|
||||
# Yield the IO channel after every `batch` files touched so foreground work interleaves.
|
||||
pause_s = max(0, settings.download_gc_sweep_pause_ms) / 1000.0
|
||||
batch = max(1, settings.download_gc_sweep_batch)
|
||||
touched = 0
|
||||
|
||||
def _throttle() -> None:
|
||||
nonlocal touched
|
||||
touched += 1
|
||||
if pause_s and touched % batch == 0:
|
||||
time.sleep(pause_s)
|
||||
|
||||
for entry in root.iterdir():
|
||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||
continue
|
||||
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
||||
for f in files:
|
||||
_throttle()
|
||||
try:
|
||||
if f.resolve() in keep or f.stat().st_mtime > cutoff:
|
||||
continue
|
||||
|
||||
@@ -83,6 +83,10 @@ def public_meta(
|
||||
from app.downloads import service
|
||||
|
||||
source_url = service.reference_url(job, asset)
|
||||
# Audio-only assets (mp3 rips, extracted audio) carry a codec but no video stream. WebKit (all
|
||||
# iOS browsers) refuses to play them inside a <video> element, so the watch page must render a
|
||||
# native <audio> instead — flag it here from the recorded codecs (a real video always has vcodec).
|
||||
is_audio = asset.vcodec is None and asset.acodec is not None
|
||||
return {
|
||||
"title": (job and job.display_name) or asset.title,
|
||||
"uploader": (job and job.display_uploader) or asset.uploader,
|
||||
@@ -92,6 +96,7 @@ def public_meta(
|
||||
"duration_s": asset.duration_s,
|
||||
"width": asset.width,
|
||||
"height": asset.height,
|
||||
"audio": is_audio,
|
||||
"allow_download": link.allow_download,
|
||||
"file_url": file_url,
|
||||
"poster_url": poster_url,
|
||||
|
||||
@@ -8,13 +8,14 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import adapters, formats, service, storage
|
||||
from app.downloads import adapters, credentials, formats, service, storage
|
||||
from app.downloads.jobdb import (
|
||||
_edit_staging_dir,
|
||||
_job_status,
|
||||
@@ -304,10 +305,13 @@ def _media_info(info: dict) -> dict:
|
||||
return {**info, **chosen}
|
||||
|
||||
|
||||
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter, cookiefile=None):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
|
||||
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated."""
|
||||
client_sets = adapter.client_sets()
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
last_exc: Exception | None = None
|
||||
@@ -317,6 +321,7 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
extra=adapter.ydl_opts_extra(clients),
|
||||
cookiefile=cookiefile,
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||
@@ -381,7 +386,9 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
# re-encode still shows real progress instead of a frozen 0%.
|
||||
total_s = int(info["duration"]) if info.get("duration") else editmod.probe_duration(produced)
|
||||
editmod.run_ffmpeg(
|
||||
editmod.build_browser_safe_cmd(produced, dest, reenc_v, reenc_a),
|
||||
editmod.build_browser_safe_cmd(
|
||||
produced, dest, reenc_v, reenc_a, threads=editmod.encode_thread_cap()
|
||||
),
|
||||
total_s=total_s,
|
||||
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
@@ -395,8 +402,11 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
return dest
|
||||
|
||||
|
||||
def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None:
|
||||
def _download(
|
||||
job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict, user_id: int
|
||||
) -> None:
|
||||
staging = _staging_dir(asset_id)
|
||||
cookiefile: Path | None = None
|
||||
try:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
@@ -405,7 +415,27 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
_set_job(job_id, phase="downloading", progress=0)
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
||||
url = service.source_url(source_kind, source_ref)
|
||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
|
||||
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
with SessionLocal() as cdb:
|
||||
asset_sig = cdb.execute(
|
||||
select(MediaAsset.format_sig).where(MediaAsset.id == asset_id)
|
||||
).scalar_one_or_none()
|
||||
if asset_sig is not None:
|
||||
cookiefile = credentials.resolve_cookiefile(
|
||||
cdb, user_id, url, asset_sig, spec, staging
|
||||
)
|
||||
info = _media_info(
|
||||
_extract_with_fallback(
|
||||
job_id,
|
||||
asset_id,
|
||||
url,
|
||||
spec,
|
||||
staging,
|
||||
adapter,
|
||||
cookiefile=str(cookiefile) if cookiefile else None,
|
||||
)
|
||||
)
|
||||
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||
@@ -515,6 +545,10 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
db.commit()
|
||||
log.warning("job %s failed: %s", job_id, msg)
|
||||
finally:
|
||||
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
||||
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
||||
if cookiefile is not None:
|
||||
cookiefile.unlink(missing_ok=True)
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
|
||||
@@ -12,16 +12,48 @@ progress hook), then exits 0. An unexpected crash exits non-zero and the parent
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.models import DownloadJob
|
||||
|
||||
log = logging.getLogger("siftlode.worker.runner")
|
||||
|
||||
|
||||
def _deprioritize() -> None:
|
||||
"""Lower THIS child's CPU + IO priority so its yt-dlp/ffmpeg work — every descendant inherits the
|
||||
handicap — can never starve the co-located API, Postgres, or Plex playback. Run once at startup,
|
||||
before any heavy work. Every step is best-effort: a failed renice must never fail a download."""
|
||||
nice_inc = settings.download_child_nice
|
||||
if nice_inc > 0:
|
||||
try:
|
||||
os.nice(nice_inc) # relative increment; the child starts at the worker's nice (0)
|
||||
except OSError as exc:
|
||||
log.info("could not renice download child: %s", str(exc)[:120])
|
||||
cls = settings.download_child_ionice_class
|
||||
if cls > 0 and shutil.which("ionice"): # util-linux; absent on some minimal images
|
||||
try:
|
||||
subprocess.run(
|
||||
["ionice", "-c", str(cls), "-p", str(os.getpid())],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
log.info("could not ionice download child: %s", str(exc)[:120])
|
||||
# Cap OpenMP-parallel ffmpeg filters to the same budget as the encoder threads, so a filter
|
||||
# graph can't fan out across every core behind ffmpeg's own `-threads` cap.
|
||||
os.environ.setdefault("OMP_NUM_THREADS", str(editmod.encode_thread_cap()))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
_deprioritize()
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
@@ -35,6 +67,7 @@ def main() -> None:
|
||||
job_kind = job.job_kind
|
||||
spec = dict(job.profile_snapshot or {})
|
||||
source_kind, source_ref = job.source_kind, job.source_ref
|
||||
user_id = job.user_id
|
||||
if asset_id is None:
|
||||
return
|
||||
|
||||
@@ -45,7 +78,7 @@ def main() -> None:
|
||||
if job_kind == "edit":
|
||||
_process_edit(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec)
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -15,6 +15,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import adapters
|
||||
from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
@@ -155,7 +156,12 @@ def enqueue(
|
||||
Raises quota.QuotaExceeded if the user is at their job-count or footprint cap."""
|
||||
quota.check_enqueue(db, user_id)
|
||||
spec = formats.normalize(spec)
|
||||
sig = formats.format_sig(spec)
|
||||
# If this user has a stored cookie for the download's host, salt the cache signature with their
|
||||
# id so the cookie-authed result is a PRIVATE asset — never dedup-shared with, or cache-served
|
||||
# to, another user (the load-bearing privacy fix; the worker attaches the cookie to exactly this
|
||||
# salted identity). A normal download (no matching cookie) gets salt=None and shares as before.
|
||||
salt = credentials.salt_for_enqueue(db, user_id, source_url(source_kind, source_ref))
|
||||
sig = formats.format_sig(spec, salt=salt)
|
||||
asset = get_or_create_asset(db, source_kind, source_ref, sig)
|
||||
# Show a real title + thumbnail the moment it's queued (feed downloads are catalog videos).
|
||||
populate_from_catalog(db, asset)
|
||||
|
||||
@@ -551,6 +551,10 @@ class Playlist(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# SHA-256 of name + ordered video ids as last synced from / pushed to YouTube; the
|
||||
# baseline `dirty` is derived against. NULL = no baseline yet (treated as dirty).
|
||||
synced_fingerprint: Mapped[str | None] = mapped_column(String)
|
||||
# JSON list of the ordered video ids as of the last sync — the 3-way-merge baseline the
|
||||
# two-way YouTube sync classifies each item against (local-add vs YT-add vs real delete).
|
||||
# NULL = no baseline yet; the first merge then does a lossless union (never deletes).
|
||||
synced_video_ids: Mapped[str | None] = mapped_column(Text)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
items: Mapped[list["PlaylistItem"]] = relationship(
|
||||
@@ -867,6 +871,35 @@ class DownloadLink(Base, TimestampMixin):
|
||||
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
|
||||
|
||||
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
|
||||
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
|
||||
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
|
||||
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
|
||||
the routes are write-only (host + updated_at only). See `downloads.credentials`.
|
||||
|
||||
Two load-bearing security controls live around this row, NOT on it:
|
||||
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
|
||||
user id, so the authed asset is private to that user and can never be cache-served to
|
||||
another user (`downloads.credentials.cookie_salt`).
|
||||
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
|
||||
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
|
||||
can't exfiltrate the user's session to a look-alike host.
|
||||
|
||||
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
|
||||
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
|
||||
|
||||
__tablename__ = "download_credentials"
|
||||
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
host_key: Mapped[str] = mapped_column(String(64))
|
||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
|
||||
|
||||
class MessageKey(Base, TimestampMixin):
|
||||
"""A user's end-to-end-encryption key material for direct messaging (phase 2).
|
||||
|
||||
|
||||
@@ -439,15 +439,89 @@ def _repush_dirty(db: Session, plex: PlexClient, link: PlexLink) -> int:
|
||||
return n
|
||||
|
||||
|
||||
# How many stale in-progress movies we probe per incremental run (one metadata() call each). The set
|
||||
# is normally tiny (movies started but no longer on Plex's Continue-Watching); the daily full reconcile
|
||||
# is the backstop for anything beyond the cap. Most-recently-touched are probed first.
|
||||
_MOVIE_PROBE_CAP = 100
|
||||
|
||||
|
||||
def _reconcile_inprogress_movies(
|
||||
db: Session, plex: PlexClient, link: PlexLink, on_deck_ids: set[int]
|
||||
) -> dict[str, int]:
|
||||
"""Close the movie "stuck in progress" gap that the history + onDeck feeds structurally miss.
|
||||
|
||||
When a movie is finished on Plex (often by scrubbing through the end credits) Plex sets
|
||||
``viewCount>0``, clears ``viewOffset`` and DROPS the movie from Continue-Watching. So onDeck no
|
||||
longer lists it (and onDeck can only ever ADD an in-progress row, never resolve one to watched —
|
||||
see `_pull_apply`), and a scrub-to-end frequently emits no `/status/sessions/history` session row,
|
||||
so the history feed misses it too. The stale local ``status='new', position>0`` row then survives
|
||||
until the *daily* full reconcile. Movies are hit hardest: a movie is a single leaf with no
|
||||
episode-rollup to mask the stuck row, and once watched it leaves onDeck entirely.
|
||||
|
||||
Fix: for each local in-progress MOVIE that Plex's onDeck no longer lists, read the item's current
|
||||
per-(token-)account ``viewCount``/``viewOffset`` with one cheap `metadata()` call and apply it via
|
||||
the shared `_pull_apply` — flipping to watched when Plex says so. Returns ``{"watched", "in_progress",
|
||||
"probed"}`` counts. Episodes are intentionally left to onDeck/history (they promote the next episode
|
||||
and aren't affected). Note: like the rest of this job, `metadata()` reads the TOKEN account (the
|
||||
owner) — a HTPC playing under a managed Plex user (accountID>1) is out of scope here as elsewhere."""
|
||||
conds = [
|
||||
PlexState.user_id == link.user_id,
|
||||
PlexState.status.notin_(("watched", "hidden")),
|
||||
PlexState.position_seconds > _PROGRESS_MIN_S,
|
||||
PlexItem.kind == "movie",
|
||||
]
|
||||
if on_deck_ids:
|
||||
conds.append(PlexState.item_id.notin_(on_deck_ids))
|
||||
rows = db.execute(
|
||||
select(PlexState, PlexItem.rating_key)
|
||||
.join(PlexItem, PlexItem.id == PlexState.item_id)
|
||||
.where(*conds)
|
||||
.order_by(PlexState.progress_updated_at.desc().nullslast())
|
||||
.limit(_MOVIE_PROBE_CAP + 1)
|
||||
).all()
|
||||
if len(rows) > _MOVIE_PROBE_CAP:
|
||||
rows = rows[:_MOVIE_PROBE_CAP]
|
||||
log.warning(
|
||||
"in-progress movie probe hit the %d cap (user %s); reconcile catches the tail",
|
||||
_MOVIE_PROBE_CAP,
|
||||
link.user_id,
|
||||
)
|
||||
out = {"watched": 0, "in_progress": 0, "probed": 0}
|
||||
for st, rk in rows:
|
||||
try:
|
||||
meta = plex.metadata(rk)
|
||||
except PlexError as e:
|
||||
log.warning(
|
||||
"movie watch-probe failed (user %s, item %s): %s", link.user_id, st.item_id, e
|
||||
)
|
||||
continue
|
||||
out["probed"] += 1
|
||||
if meta is None:
|
||||
continue
|
||||
res = _plex_watch_to_state(
|
||||
meta.get("viewCount"), meta.get("viewOffset"), meta.get("lastViewedAt")
|
||||
)
|
||||
if res is None:
|
||||
# Plex holds no watch signal now (viewCount 0, no offset) — don't clear it here; an un-watch
|
||||
# is the daily reconcile's job, and a Siftlode-only in-progress row must be preserved.
|
||||
continue
|
||||
status, pos, watched_at, prog_at = res
|
||||
ch = _pull_apply(db, link.user_id, st.item_id, status, pos, watched_at or prog_at)
|
||||
if ch:
|
||||
out[ch] += 1
|
||||
return out
|
||||
|
||||
|
||||
def run_plex_watch_sync(db: Session) -> dict:
|
||||
"""Scheduler job (default 15 min) — the cheap incremental pass. Pulls Plex-side changes since the
|
||||
high-water mark (watch history + on-deck, filtered to the owner account) into Siftlode, resolving a
|
||||
divergence to Plex (the source of truth — see `_pull_apply`), then re-pushes any still-unsynced
|
||||
divergence to Plex (the source of truth — see `_pull_apply`), reconciles stale in-progress movies
|
||||
Plex has since finished (see `_reconcile_inprogress_movies`), then re-pushes any still-unsynced
|
||||
local states. Does NOT propagate un-watches (the history feed only carries positive views) — that's
|
||||
`run_plex_watch_reconcile`'s job."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
return {"skipped": "disabled"}
|
||||
total = {"links": 0, "watched": 0, "in_progress": 0, "repushed": 0}
|
||||
total = {"links": 0, "watched": 0, "in_progress": 0, "repushed": 0, "movie_probes": 0}
|
||||
for link in _active_sync_links(db):
|
||||
total["links"] += 1
|
||||
try:
|
||||
@@ -467,10 +541,14 @@ def run_plex_watch_sync(db: Session) -> dict:
|
||||
if ch:
|
||||
total[ch] += 1
|
||||
|
||||
# onDeck resolves resume-position changes AND tells us which items Plex still considers
|
||||
# in-progress (needed by the movie reconcile below to know which local rows are stale).
|
||||
on_deck_ids: set[int] = set()
|
||||
for row in plex.on_deck():
|
||||
iid = rk_to_id.get(str(row.get("ratingKey") or ""))
|
||||
if iid is None:
|
||||
continue
|
||||
on_deck_ids.add(iid)
|
||||
off = int(row.get("viewOffset") or 0) // 1000
|
||||
if off < _PROGRESS_MIN_S:
|
||||
continue
|
||||
@@ -480,6 +558,11 @@ def run_plex_watch_sync(db: Session) -> dict:
|
||||
if ch:
|
||||
total[ch] += 1
|
||||
|
||||
mv = _reconcile_inprogress_movies(db, plex, link, on_deck_ids)
|
||||
total["watched"] += mv["watched"]
|
||||
total["in_progress"] += mv["in_progress"]
|
||||
total["movie_probes"] += mv["probed"]
|
||||
|
||||
total["repushed"] += _repush_dirty(db, plex, link)
|
||||
link.last_watch_sync_at = now
|
||||
db.commit()
|
||||
|
||||
@@ -16,10 +16,11 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import audit, sysconfig
|
||||
from app.auth import admin_user, require_human
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import quota, service, storage
|
||||
@@ -247,6 +248,65 @@ def delete_profile(
|
||||
return {"deleted": profile_id}
|
||||
|
||||
|
||||
# --- per-user download auth (cookies) ------------------------------------------------------
|
||||
# Write-only surface: the stored blob is NEVER returned — only the supported hosts + whether each
|
||||
# is configured + when. See downloads.credentials for the host-scope + cache-privacy guarantees.
|
||||
|
||||
|
||||
@router.get("/credentials")
|
||||
def list_credentials(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
return {"hosts": dlcreds.status_list(db, user.id)}
|
||||
|
||||
|
||||
class SetCredentialIn(BaseModel):
|
||||
cookies: str
|
||||
|
||||
|
||||
@router.put("/credentials/{host_key}")
|
||||
def set_credential(
|
||||
host_key: str,
|
||||
body: SetCredentialIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Store (encrypted) this user's cookies.txt for one supported host. Write-only: the value is
|
||||
never echoed back, logged, or audited — only the fact that it changed."""
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise HTTPException(status_code=404, detail="Unknown host")
|
||||
try:
|
||||
cleaned = dlcreds.validate_cookies(body.cookies)
|
||||
except dlcreds.InvalidCookies as e:
|
||||
raise HTTPException(status_code=400, detail={"code": "cookies", "reason": str(e)})
|
||||
dlcreds.upsert_credential(db, user.id, host_key, cleaned)
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_CREDENTIAL_SET,
|
||||
target_type="download_credential",
|
||||
target_id=host_key,
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "hosts": dlcreds.status_list(db, user.id)}
|
||||
|
||||
|
||||
@router.delete("/credentials/{host_key}")
|
||||
def delete_credential(
|
||||
host_key: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise HTTPException(status_code=404, detail="Unknown host")
|
||||
if dlcreds.delete_credential(db, user.id, host_key):
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_CREDENTIAL_DELETE,
|
||||
target_type="download_credential",
|
||||
target_id=host_key,
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "hosts": dlcreds.status_list(db, user.id)}
|
||||
|
||||
|
||||
def _resolve_spec(
|
||||
db: Session, user: User, profile_id: int | None, spec: dict | None
|
||||
) -> tuple[dict, int | None]:
|
||||
|
||||
@@ -14,8 +14,8 @@ from app.db import get_db
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
||||
from app.sync.playlists import (
|
||||
plan_push,
|
||||
push_playlist,
|
||||
merge_playlist,
|
||||
plan_merge,
|
||||
recompute_dirty,
|
||||
repull_playlist,
|
||||
sync_user_playlists,
|
||||
@@ -329,13 +329,13 @@ def push_plan(
|
||||
user: User = Depends(require_write_scope),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Dry run: what 'Sync to YouTube' would do + the quota estimate, so the UI can show a
|
||||
confirm. For a linked playlist this reads the live YouTube state (cheap, 1 unit/page).
|
||||
Write scope is required (require_write_scope) — syncing is a YouTube write."""
|
||||
"""Dry run: what a two-way 'Sync to YouTube' merge would do (what each side gains/loses) +
|
||||
the quota estimate, so the UI can show a confirm. For a linked playlist this reads the live
|
||||
YouTube state (cheap, 1 unit/page). Write scope is required — syncing is a YouTube write."""
|
||||
pl = _pushable(db, user, playlist_id)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
plan = plan_push(db, user, pl)
|
||||
plan = plan_merge(db, user, pl)
|
||||
except YouTubeError:
|
||||
raise HTTPException(status_code=422, detail="Couldn't read the playlist on YouTube.")
|
||||
plan["remaining_today"] = quota.remaining_today(db)
|
||||
@@ -349,14 +349,14 @@ def push(
|
||||
user: User = Depends(require_write_scope),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Push a local playlist to YouTube (create it on first push, else reconcile to match
|
||||
local — local wins). Refuses if the estimated quota exceeds what's left today, so a
|
||||
big playlist can't strand half-pushed. Write scope required (require_write_scope)."""
|
||||
"""Two-way sync a local playlist with YouTube (create it on first push, else 3-way merge so
|
||||
neither side loses an item). Refuses if the estimated quota exceeds what's left today, so a
|
||||
big sync can't strand half-applied. Write scope required (require_write_scope)."""
|
||||
pl = _pushable(db, user, playlist_id)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||
# plan and the push, halving read-quota and closing the second read→write TOCTOU
|
||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||
live = None
|
||||
if pl.yt_playlist_id:
|
||||
@@ -365,13 +365,13 @@ def push(
|
||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||
)
|
||||
plan = plan_push(db, user, pl, live=live)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="Not enough YouTube quota left today for this sync.",
|
||||
)
|
||||
return push_playlist(db, user, pl, live=live)
|
||||
return merge_playlist(db, user, pl, live=live)
|
||||
except YouTubeError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=422, detail="YouTube playlist sync failed.")
|
||||
|
||||
+240
-71
@@ -1,14 +1,16 @@
|
||||
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
|
||||
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: push local
|
||||
edits out to YouTube (`plan_push` / `push_playlist`), behind the optional write scope.
|
||||
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
|
||||
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope.
|
||||
|
||||
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
||||
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
||||
ingested (with a stub channel) so the mirror is faithful."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import TypeVar
|
||||
|
||||
@@ -42,6 +44,57 @@ def current_fingerprint(db: Session, pl: Playlist) -> str:
|
||||
return fingerprint(pl.name, _local_video_ids(db, pl))
|
||||
|
||||
|
||||
def _dump_ids(ids: list[str]) -> str:
|
||||
"""Serialize the ordered id list stored as the 3-way-merge baseline (`synced_video_ids`)."""
|
||||
return json.dumps(ids)
|
||||
|
||||
|
||||
def _load_baseline(pl: Playlist) -> tuple[list[str] | None, set[str]]:
|
||||
"""Parse `synced_video_ids` into (ordered id set, un-ingestable id set). Two on-disk forms:
|
||||
a plain JSON list (legacy / no un-ingestable ids) or `{"ids": [...], "unavailable": [...]}`.
|
||||
`unavailable` = YouTube-only videos that exist on YouTube but the Data API won't return
|
||||
(private/deleted/region-locked), so they never became local rows — tracked so the merge can
|
||||
skip them instead of forever re-classifying them as a YouTube-add (they'd never converge)."""
|
||||
if not pl.synced_video_ids:
|
||||
return None, set()
|
||||
try:
|
||||
val = json.loads(pl.synced_video_ids)
|
||||
except (ValueError, TypeError):
|
||||
return None, set()
|
||||
if isinstance(val, list):
|
||||
return [str(v) for v in val], set()
|
||||
if isinstance(val, dict):
|
||||
raw_ids = val.get("ids")
|
||||
ids = [str(v) for v in raw_ids] if isinstance(raw_ids, list) else None
|
||||
raw_un = val.get("unavailable")
|
||||
un = {str(v) for v in raw_un} if isinstance(raw_un, list) else set()
|
||||
return ids, un
|
||||
return None, set()
|
||||
|
||||
|
||||
def parse_baseline(pl: Playlist) -> list[str] | None:
|
||||
"""The last-synced ordered id set, or None when no baseline has been recorded yet (a fresh
|
||||
export, or a link created before S7). None makes the first merge union losslessly."""
|
||||
return _load_baseline(pl)[0]
|
||||
|
||||
|
||||
def parse_unavailable(pl: Playlist) -> set[str]:
|
||||
"""The last-synced set of un-ingestable-but-live YouTube ids (see `_load_baseline`)."""
|
||||
return _load_baseline(pl)[1]
|
||||
|
||||
|
||||
def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None = None) -> None:
|
||||
"""Record `ids` as both the hash (drives `dirty`, over the local ids only) and the raw ordered
|
||||
set (drives merge). `unavailable` = un-ingestable-but-live YouTube ids to carry forward so the
|
||||
next merge skips them; stored only when non-empty (keeps the common case a plain list)."""
|
||||
pl.synced_fingerprint = fingerprint(pl.name, ids)
|
||||
if unavailable:
|
||||
pl.synced_video_ids = json.dumps({"ids": ids, "unavailable": list(unavailable)})
|
||||
else:
|
||||
pl.synced_video_ids = _dump_ids(ids)
|
||||
pl.dirty = False
|
||||
|
||||
|
||||
def recompute_dirty(db: Session, pl: Playlist) -> None:
|
||||
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
|
||||
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
|
||||
@@ -116,12 +169,18 @@ def _replace_items_from_youtube(
|
||||
if v in present and v not in seen:
|
||||
seen.add(v)
|
||||
ordered.append(v)
|
||||
_set_local_items(db, pl, ordered)
|
||||
# After a pull/reset local == YouTube, so this ordered set IS the new merge baseline.
|
||||
_set_baseline(pl, ordered)
|
||||
return ordered
|
||||
|
||||
|
||||
def _set_local_items(db: Session, pl: Playlist, ordered: list[str]) -> None:
|
||||
"""Authoritatively replace `pl`'s local items with `ordered` (positions 0..n). The caller
|
||||
is responsible for having ensured every id exists in the catalog."""
|
||||
db.execute(delete(PlaylistItem).where(PlaylistItem.playlist_id == pl.id))
|
||||
for pos, vid in enumerate(ordered):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=pos))
|
||||
pl.synced_fingerprint = fingerprint(pl.name, ordered)
|
||||
pl.dirty = False
|
||||
return ordered
|
||||
|
||||
|
||||
def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
@@ -216,22 +275,91 @@ def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
||||
return moves
|
||||
|
||||
|
||||
def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
"""Compute (without writing) what pushing this local playlist to YouTube would do:
|
||||
create vs update, how many inserts/deletes/reorders, the quota estimate, and whether
|
||||
YouTube has diverged (items present there that local will remove). Costs read units
|
||||
(1/page) for a linked playlist; nothing for a fresh export. `live` = a pre-fetched
|
||||
(current_items, snippet) tuple so the caller can share one read with push_playlist."""
|
||||
desired = _local_video_ids(db, pl)
|
||||
@dataclass
|
||||
class MergePlan:
|
||||
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
|
||||
loses an item, plus the merged order. `final` is the id set both sides end up with."""
|
||||
|
||||
final: list[str] # merged order (local order, YouTube-only adds appended)
|
||||
add_to_yt: list[str] # local-adds to push out to YouTube
|
||||
add_to_local: list[str] # YouTube-adds to pull into Siftlode
|
||||
remove_from_yt: list[str] # items deleted locally that are still on YouTube
|
||||
remove_from_local: list[str] # items deleted on YouTube that are still local
|
||||
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
|
||||
baseline_known: bool
|
||||
|
||||
|
||||
def merge_plan(
|
||||
baseline: list[str] | None,
|
||||
local: list[str],
|
||||
youtube: list[str],
|
||||
unavailable: set[str] | None = None,
|
||||
) -> MergePlan:
|
||||
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
|
||||
- in local but not youtube, not deleted → add to YouTube (a local-add)
|
||||
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
|
||||
- in baseline but gone from local → a genuine local delete → propagate to YouTube
|
||||
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
|
||||
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
|
||||
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
|
||||
additions appended in their YouTube order.
|
||||
`unavailable` = known un-ingestable-but-live YouTube ids: dropped from the YouTube side up
|
||||
front so they're never re-classified as a YouTube-add (they can't ingest) nor as a local
|
||||
delete (they were never local) — otherwise such a video would make the playlist never
|
||||
converge. They stay in place on YouTube, untouched by this merge."""
|
||||
un = set(unavailable or ())
|
||||
L = list(dict.fromkeys(local))
|
||||
Y = [v for v in dict.fromkeys(youtube) if v not in un]
|
||||
Lset, Yset = set(L), set(Y)
|
||||
if baseline is None:
|
||||
add_to_yt = [v for v in L if v not in Yset]
|
||||
add_to_local = [v for v in Y if v not in Lset]
|
||||
remove_from_yt: list[str] = []
|
||||
remove_from_local: list[str] = []
|
||||
final = L + [v for v in Y if v not in Lset]
|
||||
else:
|
||||
B = set(baseline)
|
||||
deleted_local = B - Lset # was synced, removed on our side
|
||||
deleted_yt = B - Yset # was synced, removed on YouTube's side
|
||||
gone = deleted_local | deleted_yt
|
||||
remove_from_yt = [v for v in Y if v in deleted_local]
|
||||
remove_from_local = [v for v in L if v in deleted_yt]
|
||||
add_to_yt = [v for v in L if v not in Yset and v not in gone]
|
||||
add_to_local = [v for v in Y if v not in Lset and v not in gone]
|
||||
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
|
||||
final_set = set(final)
|
||||
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
|
||||
# the reorder pass then moves items into `final`.
|
||||
after_membership = [v for v in Y if v in final_set] + add_to_yt
|
||||
reorders = _reorder_moves(after_membership, final)
|
||||
return MergePlan(
|
||||
final=final,
|
||||
add_to_yt=add_to_yt,
|
||||
add_to_local=add_to_local,
|
||||
remove_from_yt=remove_from_yt,
|
||||
remove_from_local=remove_from_local,
|
||||
reorders=reorders,
|
||||
baseline_known=baseline is not None,
|
||||
)
|
||||
|
||||
|
||||
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
|
||||
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
|
||||
writes only — local edits are free). Reads live YouTube state (1 unit/page) for a linked
|
||||
playlist; nothing for a fresh export. `live` = a pre-fetched (current_items, snippet) tuple
|
||||
so the caller can share one read with merge_playlist."""
|
||||
local = _local_video_ids(db, pl)
|
||||
if not pl.yt_playlist_id:
|
||||
ops = 1 + len(desired) # create the playlist + one insert per video
|
||||
ops = 1 + len(local) # create the playlist + one insert per video
|
||||
return {
|
||||
"action": "create",
|
||||
"to_insert": len(desired),
|
||||
"to_delete": 0,
|
||||
"add_yt": len(local),
|
||||
"add_local": 0,
|
||||
"remove_yt": 0,
|
||||
"remove_local": 0,
|
||||
"to_reorder": 0,
|
||||
"rename": False,
|
||||
"yt_extra": 0,
|
||||
"units_estimate": ops * WRITE_UNIT_COST,
|
||||
}
|
||||
if live is not None:
|
||||
@@ -241,23 +369,17 @@ def plan_push(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
|
||||
cur_vids = [it["video_id"] for it in current]
|
||||
cur_set = set(cur_vids)
|
||||
desired_set = set(desired)
|
||||
to_insert = [v for v in desired if v not in cur_set]
|
||||
to_delete = [v for v in cur_vids if v not in desired_set]
|
||||
# After membership reconciliation, the YT order is current-minus-deletes plus inserts
|
||||
# appended; the reorder pass then moves items into the desired order.
|
||||
after_membership = [v for v in cur_vids if v in desired_set] + to_insert
|
||||
reorders = _reorder_moves(after_membership, desired)
|
||||
ops = len(to_insert) + len(to_delete) + reorders + (1 if rename else 0)
|
||||
yt_ids = [it["video_id"] for it in current]
|
||||
mp = merge_plan(parse_baseline(pl), local, yt_ids, parse_unavailable(pl))
|
||||
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
|
||||
return {
|
||||
"action": "update",
|
||||
"to_insert": len(to_insert),
|
||||
"to_delete": len(to_delete),
|
||||
"to_reorder": reorders,
|
||||
"add_yt": len(mp.add_to_yt),
|
||||
"add_local": len(mp.add_to_local),
|
||||
"remove_yt": len(mp.remove_from_yt),
|
||||
"remove_local": len(mp.remove_from_local),
|
||||
"to_reorder": mp.reorders,
|
||||
"rename": rename,
|
||||
"yt_extra": len(to_delete),
|
||||
"units_estimate": ops * WRITE_UNIT_COST,
|
||||
}
|
||||
|
||||
@@ -327,13 +449,16 @@ class _Breaker:
|
||||
return out
|
||||
|
||||
|
||||
def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
"""Push a local playlist to YouTube: create it if needed, then reconcile membership
|
||||
and order so YouTube matches local (local wins). Marks the playlist clean on success.
|
||||
def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
"""Two-way sync a local playlist with YouTube: create it on first push, else 3-way MERGE
|
||||
both sides so an item added (or deleted) on either side survives — no side wins wholesale.
|
||||
Local edits are applied directly; YouTube edits go through the failure breaker. On full
|
||||
success the merged set becomes the new baseline and the playlist is marked clean; on any
|
||||
YouTube failure the baseline is left untouched so the next sync retries and converges.
|
||||
Caller must have checked the write scope and the quota budget. `live` = a pre-fetched
|
||||
(current_items, snippet) tuple (from plan_push's read) to avoid a second live page-through."""
|
||||
desired = _local_video_ids(db, pl)
|
||||
inserted = deleted = reordered = created = 0
|
||||
(current_items, snippet) tuple (from plan_merge's read) to avoid a second page-through."""
|
||||
local = _local_video_ids(db, pl)
|
||||
inserted = deleted = reordered = created = pulled = removed_local = 0
|
||||
failures: list[str] = []
|
||||
breaker = _Breaker()
|
||||
with YouTubeClient(db, user) as yt:
|
||||
@@ -341,8 +466,8 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
pl.yt_playlist_id = yt.create_playlist(pl.name)
|
||||
db.commit()
|
||||
created = 1
|
||||
# Brand-new playlist: just append every video in order.
|
||||
for vid in desired:
|
||||
# Brand-new playlist: just append every local video in order.
|
||||
for vid in local:
|
||||
added = breaker.run(
|
||||
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
||||
lambda v=vid: failures.append(v),
|
||||
@@ -350,60 +475,95 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
if not isinstance(added, _Failed):
|
||||
inserted += 1
|
||||
if not failures:
|
||||
pl.synced_fingerprint = fingerprint(pl.name, desired)
|
||||
pl.dirty = False
|
||||
_set_baseline(pl, local)
|
||||
db.commit()
|
||||
return {
|
||||
"created": created,
|
||||
"inserted": inserted,
|
||||
"deleted": deleted,
|
||||
"reordered": reordered,
|
||||
"pulled": pulled,
|
||||
"removed_local": removed_local,
|
||||
"failures": failures,
|
||||
"aborted": breaker.tripped,
|
||||
"yt_playlist_id": pl.yt_playlist_id,
|
||||
}
|
||||
|
||||
# Existing link: diff against the live YouTube state (reuse plan_push's read if given).
|
||||
# Existing link: 3-way merge against the live YouTube state (reuse plan_merge's read).
|
||||
if live is not None:
|
||||
current, snippet = live
|
||||
else:
|
||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
||||
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
||||
yt_ids = [it["video_id"] for it in current]
|
||||
# Re-attempt ingestion of ids previously marked un-ingestable that are still on YouTube: a
|
||||
# video that has since become available (e.g. private→public) should re-enter the merge, not
|
||||
# stay skipped forever. Those that still won't ingest remain filtered (`effective_unavailable`).
|
||||
prev_unavailable = parse_unavailable(pl)
|
||||
still_listed = [v for v in yt_ids if v in prev_unavailable]
|
||||
if still_listed:
|
||||
_ensure_videos(db, yt, still_listed)
|
||||
healed = set(
|
||||
db.execute(select(Video.id).where(Video.id.in_(still_listed))).scalars().all()
|
||||
)
|
||||
effective_unavailable = prev_unavailable - healed
|
||||
else:
|
||||
effective_unavailable = prev_unavailable
|
||||
mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable)
|
||||
|
||||
# --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
|
||||
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
||||
# YouTube write trips the breaker.
|
||||
_ensure_videos(db, yt, mp.add_to_local)
|
||||
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
|
||||
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
|
||||
present = (
|
||||
set(db.execute(select(Video.id).where(Video.id.in_(mp.final))).scalars().all())
|
||||
if mp.final
|
||||
else set()
|
||||
)
|
||||
final_local = [v for v in mp.final if v in present]
|
||||
pulled = len([v for v in mp.add_to_local if v in present])
|
||||
removed_local = len(mp.remove_from_local)
|
||||
_set_local_items(db, pl, final_local)
|
||||
db.commit()
|
||||
|
||||
# --- YouTube side (metered, breaker-guarded).
|
||||
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
||||
try:
|
||||
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
||||
except YouTubeError:
|
||||
failures.append("title")
|
||||
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
||||
cur_vids = [it["video_id"] for it in current]
|
||||
desired_set = set(desired)
|
||||
# Delete extras (local is authoritative).
|
||||
for it in current:
|
||||
if it["video_id"] not in desired_set:
|
||||
dropped = breaker.run(
|
||||
lambda i=it: yt.delete_playlist_item(i["item_id"]),
|
||||
lambda i=it: failures.append(i["video_id"]),
|
||||
)
|
||||
if not isinstance(dropped, _Failed):
|
||||
deleted += 1
|
||||
# Insert missing (append; the reorder pass below fixes positions).
|
||||
cur_set = set(cur_vids)
|
||||
for vid in desired:
|
||||
if vid not in cur_set:
|
||||
item_id = breaker.run(
|
||||
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
||||
lambda v=vid: failures.append(v),
|
||||
)
|
||||
if not isinstance(item_id, _Failed):
|
||||
item_id_by_vid[vid] = item_id
|
||||
inserted += 1
|
||||
# Reorder to match `desired` via insertion-sort moves over a local model.
|
||||
model = [v for v in cur_vids if v in desired_set] + [
|
||||
v for v in desired if v not in cur_set and v in item_id_by_vid
|
||||
# Propagate genuine local deletes.
|
||||
for vid in mp.remove_from_yt:
|
||||
item_id = item_id_by_vid.get(vid)
|
||||
if item_id is None:
|
||||
continue
|
||||
dropped = breaker.run(
|
||||
lambda i=item_id: yt.delete_playlist_item(i),
|
||||
lambda v=vid: failures.append(v),
|
||||
)
|
||||
if not isinstance(dropped, _Failed):
|
||||
deleted += 1
|
||||
item_id_by_vid.pop(vid, None)
|
||||
# Insert local-adds (append; the reorder pass below fixes positions).
|
||||
for vid in mp.add_to_yt:
|
||||
item_id = breaker.run(
|
||||
lambda v=vid: yt.add_playlist_item(pl.yt_playlist_id, v),
|
||||
lambda v=vid: failures.append(v),
|
||||
)
|
||||
if not isinstance(item_id, _Failed):
|
||||
item_id_by_vid[vid] = item_id
|
||||
inserted += 1
|
||||
# Reorder YouTube to match `final` via insertion-sort moves over a local model.
|
||||
final_set = set(mp.final)
|
||||
model = [v for v in yt_ids if v in final_set and v in item_id_by_vid] + [
|
||||
v for v in mp.add_to_yt if v in item_id_by_vid
|
||||
]
|
||||
for i, want in enumerate(desired):
|
||||
for i, want in enumerate(mp.final):
|
||||
if want not in item_id_by_vid:
|
||||
continue # failed insert; skip
|
||||
continue # deleted-from-YT or failed insert; skip
|
||||
if i < len(model) and model[i] == want:
|
||||
continue
|
||||
moved = breaker.run(
|
||||
@@ -419,14 +579,23 @@ def push_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
model.pop(j)
|
||||
model.insert(i, want)
|
||||
if not failures:
|
||||
pl.synced_fingerprint = fingerprint(pl.name, desired)
|
||||
pl.dirty = False
|
||||
# Carry forward un-ingestable-but-live YouTube ids so the next merge skips them
|
||||
# (see merge_plan): this run's failed pulls, plus prior ones still on YouTube, minus
|
||||
# any that have since left YouTube. Recomputed each success → self-healing.
|
||||
yt_id_set = set(yt_ids)
|
||||
unavailable = sorted(
|
||||
(effective_unavailable | {v for v in mp.add_to_local if v not in present})
|
||||
& yt_id_set
|
||||
)
|
||||
_set_baseline(pl, final_local, unavailable)
|
||||
db.commit()
|
||||
return {
|
||||
"created": created,
|
||||
"inserted": inserted,
|
||||
"deleted": deleted,
|
||||
"reordered": reordered,
|
||||
"pulled": pulled,
|
||||
"removed_local": removed_local,
|
||||
"failures": failures,
|
||||
"aborted": breaker.tripped,
|
||||
"yt_playlist_id": pl.yt_playlist_id,
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
"""S6 — per-user, per-host download cookies: host-scope, cache privacy, encryption, write-only API.
|
||||
|
||||
The two load-bearing security properties are covered as behaviour, not prose:
|
||||
* **cache privacy** — a cookie-authed download is a PRIVATE, salted asset; a second user without a
|
||||
cookie enqueuing the same URL gets the shared (unsalted) asset and can never be cache-served the
|
||||
authed one, and the worker attaches a cookie ONLY to that private identity.
|
||||
* **host scope (SEC-12)** — a cookie is resolved ONLY for a URL whose netloc the credential owns; a
|
||||
look-alike or unrelated host resolves to no host_key and gets no cookie.
|
||||
|
||||
Pure-logic parts run with no DB; the CRUD / enqueue-salting / API parts use the `db`/`as_user` lane.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from app import security
|
||||
from app.downloads import credentials, formats, service
|
||||
from app.models import DownloadCredential, MediaAsset, User
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fernet(monkeypatch):
|
||||
# A throwaway key so encrypt/decrypt work without TOKEN_ENCRYPTION_KEY in the env (mirrors
|
||||
# test_security). Autouse: every credential write in this module encrypts.
|
||||
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key()))
|
||||
|
||||
|
||||
COOKIES = "# Netscape HTTP Cookie File\n.facebook.com\tTRUE\t/\tTRUE\t0\tc_user\t123\n"
|
||||
|
||||
|
||||
# --- host scope (pure logic) ------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHostKeyForUrl:
|
||||
def test_matches_a_supported_host(self):
|
||||
assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook"
|
||||
assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook"
|
||||
assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter"
|
||||
|
||||
def test_look_alike_host_never_matches(self):
|
||||
assert credentials.host_key_for_url("https://myfacebook.com/x") is None
|
||||
assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None
|
||||
|
||||
def test_unrelated_or_empty_url_has_no_host_key(self):
|
||||
assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None
|
||||
assert credentials.host_key_for_url("") is None
|
||||
assert credentials.host_key_for_url(None) is None
|
||||
|
||||
|
||||
class TestValidateCookies:
|
||||
def test_accepts_a_netscape_file(self):
|
||||
assert credentials.validate_cookies(COOKIES).endswith("\n")
|
||||
|
||||
def test_rejects_empty(self):
|
||||
with pytest.raises(credentials.InvalidCookies):
|
||||
credentials.validate_cookies(" ")
|
||||
|
||||
def test_rejects_non_cookie_paste(self):
|
||||
# Comments only, no tab-separated data line -> not a usable cookies.txt.
|
||||
with pytest.raises(credentials.InvalidCookies):
|
||||
credentials.validate_cookies("just some pasted text\n# a comment")
|
||||
|
||||
|
||||
def test_authed_sig_differs_per_user_and_from_the_shared_sig():
|
||||
spec = formats.normalize({})
|
||||
shared = formats.format_sig(spec)
|
||||
a = credentials.authed_format_sig(spec, 1, "facebook")
|
||||
b = credentials.authed_format_sig(spec, 2, "facebook")
|
||||
assert a != shared and b != shared and a != b
|
||||
|
||||
|
||||
def test_write_cookiefile_is_private_and_survives_the_retry_wipe(tmp_path):
|
||||
staging = tmp_path / "staging"
|
||||
staging.mkdir()
|
||||
path = credentials._write_cookiefile(COOKIES, staging)
|
||||
assert path.read_text() == COOKIES
|
||||
# 0600 (owner-only) — POSIX perms only (the worker is Linux-only; Windows has no mode bits).
|
||||
if os.name == "posix":
|
||||
assert (path.stat().st_mode & 0o777) == 0o600
|
||||
# It lives in a SUBDIR so the download retry loop (which unlinks files in the staging ROOT) and
|
||||
# _produced_file (which scans the root) never see it.
|
||||
assert path.parent == staging / ".auth"
|
||||
assert not any(p.is_file() for p in staging.iterdir())
|
||||
|
||||
|
||||
# --- CRUD + encryption at rest (DB) -----------------------------------------------------------
|
||||
|
||||
|
||||
def _user(db, email="cred@example.com") -> User:
|
||||
u = User(email=email)
|
||||
db.add(u)
|
||||
db.flush()
|
||||
return u
|
||||
|
||||
|
||||
class TestCrud:
|
||||
def test_upsert_stores_ciphertext_not_plaintext(self, db):
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
row = db.get(DownloadCredential, 1)
|
||||
assert row.cookies_enc != COOKIES # encrypted at rest
|
||||
assert COOKIES not in row.cookies_enc
|
||||
assert security.decrypt(row.cookies_enc) == COOKIES
|
||||
|
||||
def test_upsert_replaces_in_place(self, db):
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES.replace("123", "999"))
|
||||
db.commit()
|
||||
rows = db.query(DownloadCredential).filter_by(user_id=u.id, host_key="facebook").all()
|
||||
assert len(rows) == 1
|
||||
assert "999" in security.decrypt(rows[0].cookies_enc)
|
||||
|
||||
def test_status_list_never_leaks_the_blob(self, db):
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
status = credentials.status_list(db, u.id)
|
||||
fb = next(h for h in status if h["host_key"] == "facebook")
|
||||
assert fb["configured"] is True and fb["updated_at"] is not None
|
||||
ig = next(h for h in status if h["host_key"] == "instagram")
|
||||
assert ig["configured"] is False and ig["updated_at"] is None
|
||||
# No field anywhere carries the cookie value.
|
||||
assert all(COOKIES not in str(v) for h in status for v in h.values())
|
||||
|
||||
def test_delete(self, db):
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
assert credentials.delete_credential(db, u.id, "facebook") is True
|
||||
db.commit()
|
||||
assert credentials.get_credential(db, u.id, "facebook") is None
|
||||
assert credentials.delete_credential(db, u.id, "facebook") is False
|
||||
|
||||
|
||||
# --- the crux: enqueue salts privately, no cross-user cache serve (DB) -------------------------
|
||||
|
||||
FB_URL = "https://www.facebook.com/watch?v=100"
|
||||
|
||||
|
||||
class TestCachePrivacy:
|
||||
def test_enqueue_with_a_cookie_creates_a_private_salted_asset(self, db):
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
job = service.enqueue(db, u.id, "url", FB_URL, {})
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
spec = formats.normalize({})
|
||||
assert asset.format_sig == credentials.authed_format_sig(spec, u.id, "facebook")
|
||||
assert asset.format_sig != formats.format_sig(spec)
|
||||
|
||||
def test_enqueue_without_a_cookie_uses_the_shared_sig(self, db):
|
||||
u = _user(db)
|
||||
job = service.enqueue(db, u.id, "url", FB_URL, {})
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
assert asset.format_sig == formats.format_sig(formats.normalize({}))
|
||||
|
||||
def test_two_users_never_share_the_authed_asset(self, db):
|
||||
a = _user(db, "a@example.com")
|
||||
b = _user(db, "b@example.com")
|
||||
credentials.upsert_credential(db, a.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
ja = service.enqueue(db, a.id, "url", FB_URL, {}) # A: authed, private
|
||||
jb = service.enqueue(db, b.id, "url", FB_URL, {}) # B: no cookie, shared
|
||||
assert ja.asset_id != jb.asset_id
|
||||
# B's asset is the shared identity — A's private group video can never be served to B.
|
||||
assert db.get(MediaAsset, jb.asset_id).format_sig == formats.format_sig(
|
||||
formats.normalize({})
|
||||
)
|
||||
|
||||
|
||||
# --- the worker gate: attach only to the private identity, host-scoped (DB) --------------------
|
||||
|
||||
|
||||
class TestResolveCookiefile:
|
||||
def _asset_sig(self, sig: str) -> MediaAsset:
|
||||
return MediaAsset(
|
||||
source_kind="url", source_ref=FB_URL, format_sig=sig, status="downloading"
|
||||
)
|
||||
|
||||
def test_attaches_to_the_private_salted_asset(self, db, tmp_path):
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
spec = formats.normalize({})
|
||||
salted = credentials.authed_format_sig(spec, u.id, "facebook")
|
||||
path = credentials.resolve_cookiefile(db, u.id, FB_URL, salted, spec, tmp_path)
|
||||
assert path is not None and Path(path).read_text() == COOKIES
|
||||
|
||||
def test_never_attaches_to_a_shared_asset(self, db, tmp_path):
|
||||
# Cookie exists, but the asset is the UNSALTED shared identity -> attaching would poison the
|
||||
# global cache with a private-group video. Must return None.
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
spec = formats.normalize({})
|
||||
shared = formats.format_sig(spec)
|
||||
assert credentials.resolve_cookiefile(db, u.id, FB_URL, shared, spec, tmp_path) is None
|
||||
|
||||
def test_non_matching_host_gets_no_cookie(self, db, tmp_path):
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
spec = formats.normalize({})
|
||||
# A YouTube URL has no supported host_key -> no cookie regardless of what's stored.
|
||||
yt = "https://www.youtube.com/watch?v=abc"
|
||||
salted = credentials.authed_format_sig(spec, u.id, "facebook")
|
||||
assert credentials.resolve_cookiefile(db, u.id, yt, salted, spec, tmp_path) is None
|
||||
|
||||
def test_no_credential_gets_no_cookie(self, db, tmp_path):
|
||||
u = _user(db)
|
||||
spec = formats.normalize({})
|
||||
salted = credentials.authed_format_sig(spec, u.id, "facebook")
|
||||
assert credentials.resolve_cookiefile(db, u.id, FB_URL, salted, spec, tmp_path) is None
|
||||
|
||||
|
||||
# --- API: write-only, never echoes the blob (DB) ----------------------------------------------
|
||||
|
||||
|
||||
class TestApi:
|
||||
def test_put_get_delete_roundtrip_never_returns_the_blob(self, db, as_user):
|
||||
u = _user(db)
|
||||
db.commit()
|
||||
client = as_user(u)
|
||||
|
||||
r = client.put("/api/downloads/credentials/facebook", json={"cookies": COOKIES})
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["ok"] is True
|
||||
assert COOKIES not in r.text # the response must never carry the value
|
||||
fb = next(h for h in body["hosts"] if h["host_key"] == "facebook")
|
||||
assert fb["configured"] is True
|
||||
|
||||
r = client.get("/api/downloads/credentials")
|
||||
assert r.status_code == 200 and COOKIES not in r.text
|
||||
|
||||
r = client.delete("/api/downloads/credentials/facebook")
|
||||
assert r.status_code == 200
|
||||
assert credentials.get_credential(db, u.id, "facebook") is None
|
||||
|
||||
def test_unknown_host_is_404(self, db, as_user):
|
||||
u = _user(db)
|
||||
db.commit()
|
||||
r = as_user(u).put("/api/downloads/credentials/nope", json={"cookies": COOKIES})
|
||||
assert r.status_code == 404
|
||||
|
||||
def test_bad_cookies_are_rejected(self, db, as_user):
|
||||
u = _user(db)
|
||||
db.commit()
|
||||
r = as_user(u).put("/api/downloads/credentials/facebook", json={"cookies": " "})
|
||||
assert r.status_code == 400
|
||||
assert r.json()["detail"]["code"] == "cookies"
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Worker-child resource de-prioritization (S4): the encoder thread cap, the `-threads` flag on the
|
||||
browser-safe re-encode, and the child's startup renice/ionice/OMP handicap. These keep a CPU-heavy
|
||||
transcode from pegging every core on the box it shares with the API, Postgres, and Plex playback."""
|
||||
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import runner
|
||||
|
||||
|
||||
class TestEncodeThreadCap:
|
||||
def test_pinned_setting_wins(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_ffmpeg_threads", 3)
|
||||
assert editmod.encode_thread_cap() == 3
|
||||
|
||||
def test_zero_reserves_one_core(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_ffmpeg_threads", 0)
|
||||
monkeypatch.setattr(editmod.os, "cpu_count", lambda: 8)
|
||||
assert editmod.encode_thread_cap() == 7
|
||||
|
||||
def test_single_core_box_never_caps_below_one(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_ffmpeg_threads", 0)
|
||||
monkeypatch.setattr(editmod.os, "cpu_count", lambda: 1)
|
||||
assert editmod.encode_thread_cap() == 1
|
||||
|
||||
def test_unknown_cpu_count_falls_back(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_ffmpeg_threads", 0)
|
||||
monkeypatch.setattr(editmod.os, "cpu_count", lambda: None)
|
||||
assert editmod.encode_thread_cap() == 1 # max(1, 2 - 1)
|
||||
|
||||
|
||||
class TestBrowserSafeThreads:
|
||||
def test_threads_flag_added_when_capped(self):
|
||||
cmd = editmod.build_browser_safe_cmd(
|
||||
"/in.mkv", "/out.mp4", reencode_video=True, reencode_audio=False, threads=2
|
||||
)
|
||||
assert "-threads" in cmd
|
||||
assert cmd[cmd.index("-threads") + 1] == "2"
|
||||
# The cap precedes the output file, so it binds the encoder (not input decoding).
|
||||
assert cmd.index("-threads") < cmd.index("/out.mp4")
|
||||
|
||||
@pytest.mark.parametrize("threads", [0, None])
|
||||
def test_no_threads_flag_when_uncapped(self, threads):
|
||||
cmd = editmod.build_browser_safe_cmd(
|
||||
"/in.mkv", "/out.mp4", reencode_video=True, reencode_audio=False, threads=threads
|
||||
)
|
||||
assert "-threads" not in cmd
|
||||
|
||||
|
||||
class TestDeprioritize:
|
||||
def test_renices_by_configured_increment(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_child_nice", 12)
|
||||
monkeypatch.setattr(settings, "download_child_ionice_class", 0) # skip ionice branch
|
||||
calls = []
|
||||
monkeypatch.setattr(runner.os, "nice", lambda n: calls.append(n))
|
||||
monkeypatch.setattr(runner.os, "environ", {})
|
||||
runner._deprioritize()
|
||||
assert calls == [12]
|
||||
|
||||
def test_nice_zero_does_not_renice(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_child_nice", 0)
|
||||
monkeypatch.setattr(settings, "download_child_ionice_class", 0)
|
||||
calls = []
|
||||
monkeypatch.setattr(runner.os, "nice", lambda n: calls.append(n))
|
||||
monkeypatch.setattr(runner.os, "environ", {})
|
||||
runner._deprioritize()
|
||||
assert calls == []
|
||||
|
||||
def test_sets_omp_budget_to_thread_cap(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_child_nice", 0)
|
||||
monkeypatch.setattr(settings, "download_child_ionice_class", 0)
|
||||
monkeypatch.setattr(settings, "download_ffmpeg_threads", 4)
|
||||
env: dict = {}
|
||||
monkeypatch.setattr(runner.os, "environ", env)
|
||||
runner._deprioritize()
|
||||
assert env["OMP_NUM_THREADS"] == "4"
|
||||
|
||||
def test_ionice_invoked_when_binary_present(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_child_nice", 0)
|
||||
monkeypatch.setattr(settings, "download_child_ionice_class", 3)
|
||||
monkeypatch.setattr(runner.os, "environ", {})
|
||||
monkeypatch.setattr(runner.shutil, "which", lambda name: "/usr/bin/ionice")
|
||||
ran = []
|
||||
monkeypatch.setattr(
|
||||
runner.subprocess,
|
||||
"run",
|
||||
lambda cmd, **kw: ran.append(cmd) or subprocess.CompletedProcess(cmd, 0),
|
||||
)
|
||||
runner._deprioritize()
|
||||
assert ran and ran[0][:3] == ["ionice", "-c", "3"]
|
||||
|
||||
def test_ionice_skipped_when_binary_absent(self, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_child_nice", 0)
|
||||
monkeypatch.setattr(settings, "download_child_ionice_class", 3)
|
||||
monkeypatch.setattr(runner.os, "environ", {})
|
||||
monkeypatch.setattr(runner.shutil, "which", lambda name: None)
|
||||
ran = []
|
||||
monkeypatch.setattr(runner.subprocess, "run", lambda cmd, **kw: ran.append(cmd))
|
||||
runner._deprioritize()
|
||||
assert ran == []
|
||||
@@ -14,6 +14,13 @@ def test_real_ytdlp_messages_map_to_categories():
|
||||
"ERROR: [youtube] abc: Sign in to confirm you're not a bot": "login_required",
|
||||
"ERROR: [youtube] abc: Join this channel to get access to members-only content": "login_required",
|
||||
"ERROR: [Reddit] 1vi1x56: Account authentication is required.": "login_required",
|
||||
# Closed-group / private-post auth walls: yt-dlp either says to sign in or suggests cookies.
|
||||
"ERROR: [facebook] 123: You must be logged in to see this post": "login_required",
|
||||
"ERROR: [facebook] 123: Login required. Use --cookies for the authentication.": "login_required",
|
||||
"ERROR: [instagram] abc: This content is only available for registered users": "login_required",
|
||||
"ERROR: [youtube] abc: Sign in to confirm... Use --cookies-from-browser or --cookies": "login_required",
|
||||
# A site yt-dlp has no extractor for at all — distinct from an auth wall, still non-retryable.
|
||||
"ERROR: Unsupported URL: https://example.com/whatever": "unsupported_host",
|
||||
"ERROR: [youtube] abc: The uploader has not made this video available in your country": "geo_blocked",
|
||||
"ERROR: [youtube] abc: Requested format is not available": "format_unavailable",
|
||||
"ERROR: [youtube] abc: Only images are available for download": "format_unavailable",
|
||||
|
||||
@@ -5,6 +5,8 @@ import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import gc as gcmod
|
||||
from app.downloads.gc import _sweep_orphan_files
|
||||
|
||||
|
||||
@@ -69,3 +71,38 @@ def test_sweep_removes_orphans_but_keeps_owned_system_and_fresh(tmp_path: Path):
|
||||
assert not (root / "OldChan").exists()
|
||||
assert sysfile.exists() # system tree untouched
|
||||
assert fresh.exists() # too new to sweep
|
||||
|
||||
|
||||
def test_sweep_throttles_after_each_batch(tmp_path: Path, monkeypatch):
|
||||
"""The whole-tree walk yields the IO channel after every `batch` files so it can't monopolize
|
||||
disk IO in the API process. batch=2 over 5 touched files → 2 pauses (at file 2 and 4)."""
|
||||
root = tmp_path
|
||||
d = root / "Chan"
|
||||
d.mkdir()
|
||||
for i in range(5):
|
||||
(d / f"orphan_{i}.dat").write_bytes(b"x")
|
||||
for f in d.iterdir():
|
||||
_aged(f, 5) # old enough that every file is "touched" by the walk
|
||||
|
||||
monkeypatch.setattr(settings, "download_gc_sweep_pause_ms", 10)
|
||||
monkeypatch.setattr(settings, "download_gc_sweep_batch", 2)
|
||||
pauses = []
|
||||
monkeypatch.setattr(gcmod.time, "sleep", lambda s: pauses.append(s))
|
||||
|
||||
_sweep_orphan_files(_StubDB([]), str(root))
|
||||
|
||||
assert pauses == [0.01, 0.01] # 5 files, batch 2 → yield at #2 and #4
|
||||
|
||||
|
||||
def test_sweep_no_throttle_when_pause_disabled(tmp_path: Path, monkeypatch):
|
||||
(tmp_path / "Chan").mkdir()
|
||||
f = tmp_path / "Chan" / "orphan.dat"
|
||||
f.write_bytes(b"x")
|
||||
_aged(f, 5)
|
||||
monkeypatch.setattr(settings, "download_gc_sweep_pause_ms", 0)
|
||||
pauses = []
|
||||
monkeypatch.setattr(gcmod.time, "sleep", lambda s: pauses.append(s))
|
||||
|
||||
_sweep_orphan_files(_StubDB([]), str(tmp_path))
|
||||
|
||||
assert pauses == []
|
||||
|
||||
@@ -4,7 +4,14 @@ token, rides in the file URL (a <video src> can't send a header), and expires.""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.downloads.links import _GRANT_TTL, check_grant, is_expired, make_grant, new_token
|
||||
from app.downloads.links import (
|
||||
_GRANT_TTL,
|
||||
check_grant,
|
||||
is_expired,
|
||||
make_grant,
|
||||
new_token,
|
||||
public_meta,
|
||||
)
|
||||
|
||||
|
||||
def test_new_token_is_unguessable_and_unique():
|
||||
@@ -48,6 +55,44 @@ class TestGrant:
|
||||
assert check_grant("tok", bad, 0) is False
|
||||
|
||||
|
||||
def _asset(**over):
|
||||
base = dict(
|
||||
title="Clip",
|
||||
uploader="Someone",
|
||||
uploader_url=None,
|
||||
duration_s=42,
|
||||
width=1280,
|
||||
height=720,
|
||||
vcodec="h264",
|
||||
acodec="aac",
|
||||
source_webpage_url=None,
|
||||
source_kind="url",
|
||||
)
|
||||
base.update(over)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
class TestPublicMetaAudioFlag:
|
||||
"""The watch page picks <audio> vs <video> from this flag; WebKit refuses an audio-only mp3 in a
|
||||
<video>, so an audio-only asset (codec present, no video stream) must report audio=True."""
|
||||
|
||||
_link = SimpleNamespace(allow_download=True)
|
||||
|
||||
def test_audio_only_asset_is_flagged_audio(self):
|
||||
asset = _asset(vcodec=None, acodec="mp3", width=None, height=None)
|
||||
meta = public_meta(self._link, None, asset, "/f.mp3")
|
||||
assert meta["audio"] is True
|
||||
|
||||
def test_a_video_asset_is_not_audio(self):
|
||||
meta = public_meta(self._link, None, _asset(), "/f.mp4")
|
||||
assert meta["audio"] is False
|
||||
|
||||
def test_a_silent_video_is_not_audio(self):
|
||||
# Video present but no audio track — still a video player, not the audio bar.
|
||||
meta = public_meta(self._link, None, _asset(acodec=None), "/f.mp4")
|
||||
assert meta["audio"] is False
|
||||
|
||||
|
||||
class TestIsExpired:
|
||||
def test_none_expiry_never_expires(self):
|
||||
assert is_expired(SimpleNamespace(expires_at=None)) is False
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""S7 — two-way (3-way) YouTube playlist merge.
|
||||
|
||||
The load-bearing property: an item added (or deleted) on EITHER side survives the sync — the old
|
||||
push was one-directional (local wins, deletes everything not local from YouTube), so YouTube-side
|
||||
adds were lost. Pure-merge classification runs with no DB; the apply path uses the `db` lane with a
|
||||
fake YouTube client so both the local rows and the simulated YouTube state can be asserted.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||
from app.sync import playlists as pl_mod
|
||||
from app.sync.playlists import (
|
||||
merge_plan,
|
||||
merge_playlist,
|
||||
parse_baseline,
|
||||
parse_unavailable,
|
||||
plan_merge,
|
||||
)
|
||||
from app.youtube.client import YouTubeError
|
||||
|
||||
# --- pure merge classification (no DB) --------------------------------------------------------
|
||||
|
||||
|
||||
class TestMergePlan:
|
||||
def test_no_baseline_unions_losslessly(self):
|
||||
# First-ever merge: can't tell an add from a delete, so keep everything, delete nothing.
|
||||
mp = merge_plan(None, local=["a", "b"], youtube=["b", "c"])
|
||||
assert mp.add_to_yt == ["a"] # local-only pushed out
|
||||
assert mp.add_to_local == ["c"] # youtube-only pulled in
|
||||
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
||||
assert mp.final == ["a", "b", "c"] # local order, youtube-only appended
|
||||
assert mp.baseline_known is False
|
||||
|
||||
def test_adds_on_both_sides_both_survive(self):
|
||||
# baseline b,c ; local added 'a' ; youtube added 'd' — nothing deleted.
|
||||
mp = merge_plan(["b", "c"], local=["a", "b", "c"], youtube=["b", "c", "d"])
|
||||
assert mp.add_to_yt == ["a"]
|
||||
assert mp.add_to_local == ["d"]
|
||||
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
||||
assert mp.final == ["a", "b", "c", "d"]
|
||||
|
||||
def test_genuine_local_delete_propagates_to_youtube(self):
|
||||
# baseline a,b,c ; local removed 'b' ; youtube unchanged -> remove b from youtube.
|
||||
mp = merge_plan(["a", "b", "c"], local=["a", "c"], youtube=["a", "b", "c"])
|
||||
assert mp.remove_from_yt == ["b"]
|
||||
assert mp.add_to_yt == [] and mp.add_to_local == []
|
||||
assert mp.final == ["a", "c"]
|
||||
|
||||
def test_genuine_youtube_delete_propagates_to_local(self):
|
||||
# baseline a,b,c ; youtube removed 'b' ; local unchanged -> remove b from local.
|
||||
mp = merge_plan(["a", "b", "c"], local=["a", "b", "c"], youtube=["a", "c"])
|
||||
assert mp.remove_from_local == ["b"]
|
||||
assert mp.final == ["a", "c"]
|
||||
|
||||
def test_delete_on_both_sides_is_gone_not_a_conflict(self):
|
||||
mp = merge_plan(["a", "b", "c"], local=["a", "c"], youtube=["a", "c"])
|
||||
assert mp.remove_from_yt == [] and mp.remove_from_local == []
|
||||
assert mp.final == ["a", "c"]
|
||||
|
||||
def test_a_local_delete_beats_a_stale_youtube_copy(self):
|
||||
# 'b' was synced, deleted locally, still present on youtube -> it must be removed there,
|
||||
# NOT resurrected into local as a youtube-add.
|
||||
mp = merge_plan(["a", "b"], local=["a"], youtube=["a", "b"])
|
||||
assert mp.add_to_local == []
|
||||
assert mp.remove_from_yt == ["b"]
|
||||
assert "b" not in mp.final
|
||||
|
||||
|
||||
# --- apply path (db lane + fake YouTube) ------------------------------------------------------
|
||||
|
||||
|
||||
class FakeYouTube:
|
||||
"""Enough of YouTubeClient for merge_playlist: an in-memory playlist of (item_id, video_id)
|
||||
and the write ops that mutate it. Every catalog video is pre-seeded so _ensure_videos never
|
||||
needs get_videos."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
title: str,
|
||||
items: list[str],
|
||||
unavailable: set[str] | None = None,
|
||||
fail_add: set[str] | None = None,
|
||||
):
|
||||
self.title = title
|
||||
self._seq = 0
|
||||
self.items = [(self._new_id(), v) for v in items]
|
||||
# Ids the Data API won't return (private/deleted) — get_videos omits them, so _ensure_videos
|
||||
# can't ingest them. They still live in the playlist (`items`) like a real un-ingestable video.
|
||||
self.unavailable = set(unavailable or ())
|
||||
# Ids whose add_playlist_item raises, to exercise the partial-failure baseline-hold path.
|
||||
self.fail_add = set(fail_add or ())
|
||||
|
||||
def _new_id(self) -> str:
|
||||
self._seq += 1
|
||||
return f"it{self._seq}"
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def create_playlist(self, title, description="", privacy="private"):
|
||||
self.title = title
|
||||
return "PLnew"
|
||||
|
||||
def get_playlist_snippet(self, pid):
|
||||
return {"title": self.title}
|
||||
|
||||
def iter_playlist_items_with_ids(self, pid):
|
||||
return [{"item_id": iid, "video_id": v} for iid, v in self.items]
|
||||
|
||||
def get_videos(self, ids):
|
||||
# Mirror the Data API: a minimal ingestable record for each id, EXCEPT the un-ingestable
|
||||
# ones (private/deleted), which are simply absent from the response.
|
||||
return [
|
||||
{
|
||||
"id": v,
|
||||
"snippet": {
|
||||
"channelId": "UCtest",
|
||||
"channelTitle": "Test",
|
||||
"publishedAt": "2020-01-01T00:00:00Z",
|
||||
},
|
||||
}
|
||||
for v in ids
|
||||
if v not in self.unavailable
|
||||
]
|
||||
|
||||
def update_playlist_title(self, pid, title):
|
||||
self.title = title
|
||||
|
||||
def add_playlist_item(self, pid, vid, position=None):
|
||||
if vid in self.fail_add:
|
||||
raise YouTubeError("simulated add failure")
|
||||
iid = self._new_id()
|
||||
self.items.append((iid, vid))
|
||||
return iid
|
||||
|
||||
def delete_playlist_item(self, item_id):
|
||||
self.items = [(iid, v) for iid, v in self.items if iid != item_id]
|
||||
|
||||
def move_playlist_item(self, item_id, pid, vid, position):
|
||||
row = next((r for r in self.items if r[0] == item_id), None)
|
||||
if row is not None:
|
||||
self.items.remove(row)
|
||||
self.items.insert(position, row)
|
||||
|
||||
@property
|
||||
def video_ids(self) -> list[str]:
|
||||
return [v for _, v in self.items]
|
||||
|
||||
|
||||
def _seed_videos(db, ids):
|
||||
db.add(Channel(id="UCtest", title="Test"))
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in ids:
|
||||
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _linked_playlist(db, user, *, local_ids, baseline):
|
||||
pl = Playlist(
|
||||
user_id=user.id,
|
||||
name="Mix",
|
||||
source="local",
|
||||
yt_playlist_id="PLyt",
|
||||
synced_video_ids=None if baseline is None else pl_mod._dump_ids(baseline),
|
||||
)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for pos, v in enumerate(local_ids):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos))
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
def _local_ids(db, pl):
|
||||
return pl_mod._local_video_ids(db, pl)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _fake_yt(monkeypatch):
|
||||
holder = {}
|
||||
|
||||
def install(fake):
|
||||
holder["fake"] = fake
|
||||
monkeypatch.setattr(pl_mod, "YouTubeClient", lambda db, user: fake)
|
||||
return fake
|
||||
|
||||
return install
|
||||
|
||||
|
||||
def test_adds_on_both_sides_survive_on_both_sides(db, make_user, _fake_yt):
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b", "c", "d"])
|
||||
# baseline b,c ; local added 'a' ; youtube added 'd'
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "b", "c"], baseline=["b", "c"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c", "d"]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["failures"] == []
|
||||
assert set(_local_ids(db, pl)) == {"a", "b", "c", "d"}, "youtube-add 'd' pulled into Siftlode"
|
||||
assert set(fake.video_ids) == {"a", "b", "c", "d"}, "local-add 'a' pushed to YouTube"
|
||||
assert r["inserted"] == 1 and r["pulled"] == 1
|
||||
# Baseline advanced to the merged set so a re-sync is a no-op.
|
||||
assert set(parse_baseline(pl)) == {"a", "b", "c", "d"}
|
||||
|
||||
|
||||
def test_a_genuine_delete_propagates_without_collateral_loss(db, make_user, _fake_yt):
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b", "c", "d"])
|
||||
# baseline a,b,c ; local removed 'b' ; youtube added 'd' (a real add, must NOT be collateral)
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "c"], baseline=["a", "b", "c"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "c", "d"]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["failures"] == []
|
||||
assert set(fake.video_ids) == {"a", "c", "d"}, "'b' removed on YouTube, 'd' kept"
|
||||
assert set(_local_ids(db, pl)) == {"a", "c", "d"}, "'d' pulled in, 'b' stays deleted"
|
||||
assert r["deleted"] == 1
|
||||
|
||||
|
||||
def test_first_merge_with_no_baseline_never_deletes(db, make_user, _fake_yt):
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b", "c"])
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=None)
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c"]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["deleted"] == 0 and r["removed_local"] == 0
|
||||
assert set(_local_ids(db, pl)) == {"a", "b", "c"}
|
||||
assert set(fake.video_ids) == {"a", "b", "c"}
|
||||
assert parse_baseline(pl) is not None, "a baseline is recorded after the first union"
|
||||
|
||||
|
||||
def test_create_path_appends_and_records_baseline(db, make_user, _fake_yt):
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b"])
|
||||
pl = Playlist(user_id=user.id, name="Fresh", source="local")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for pos, v in enumerate(["a", "b"]):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos))
|
||||
db.commit()
|
||||
_fake_yt(FakeYouTube(title="Fresh", items=[]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["created"] == 1 and r["inserted"] == 2
|
||||
assert pl.yt_playlist_id # created and linked
|
||||
assert parse_baseline(pl) == ["a", "b"]
|
||||
|
||||
|
||||
def test_reorder_reaches_the_merged_order_on_youtube(db, make_user, _fake_yt):
|
||||
# Same membership on both sides, but the user reordered locally to c,a,b. The apply-side
|
||||
# insertion-sort must drive YouTube to that exact order — asserted as an ordered list, not a
|
||||
# set, so an off-by-one or a dropped reorder pass can't pass unnoticed.
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b", "c"])
|
||||
pl = _linked_playlist(db, user, local_ids=["c", "a", "b"], baseline=["a", "b", "c"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "c"]))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["failures"] == []
|
||||
assert fake.video_ids == ["c", "a", "b"], "YouTube order matches the merged (local) order"
|
||||
assert r["reordered"] >= 1
|
||||
assert parse_baseline(pl) == ["c", "a", "b"]
|
||||
|
||||
|
||||
def test_partial_youtube_failure_holds_the_baseline_for_retry(db, make_user, _fake_yt):
|
||||
# A local-add whose YouTube insert fails must NOT advance the baseline — otherwise the retry
|
||||
# sync would read the un-pushed add as a genuine YouTube-side delete and silently drop it.
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b", "c"])
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "b", "c"], baseline=["b", "c"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["b", "c"], fail_add={"a"}))
|
||||
|
||||
r = merge_playlist(db, user, pl)
|
||||
|
||||
assert r["failures"] == ["a"], "the failed local-add is reported"
|
||||
assert "a" not in fake.video_ids, "the add never landed on YouTube"
|
||||
# Baseline is UNCHANGED so the next sync re-classifies 'a' as a local-add and re-pushes it,
|
||||
# rather than treating it as a YouTube delete and removing it locally.
|
||||
assert parse_baseline(pl) == ["b", "c"]
|
||||
assert "a" in set(_local_ids(db, pl)), "'a' stays local for the retry"
|
||||
|
||||
|
||||
def test_unavailable_youtube_video_converges(db, make_user, _fake_yt):
|
||||
# A YouTube-only video that can't be ingested (private/deleted) must not make the playlist
|
||||
# loop forever re-classifying it as a pending pull. It is recorded as un-ingestable and skipped
|
||||
# by the next merge; the playlist reports "up to date".
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b"]) # note: 'x' is NOT in the catalog and can't be ingested
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"}))
|
||||
|
||||
r1 = merge_playlist(db, user, pl)
|
||||
|
||||
assert r1["failures"] == []
|
||||
assert r1["pulled"] == 0, "the un-ingestable video did not become a local row"
|
||||
assert set(_local_ids(db, pl)) == {"a", "b"}
|
||||
assert parse_unavailable(pl) == {"x"}, "'x' recorded as un-ingestable"
|
||||
assert "x" in fake.video_ids, "'x' left untouched on YouTube"
|
||||
|
||||
# Second sync: the preview reports nothing pending (plan_merge filters 'x' via the stored
|
||||
# unavailable set). The sync itself re-attempts ingestion of 'x' (the heal path), but since it
|
||||
# still won't ingest, 'x' stays recorded as unavailable — no perpetual pending pull.
|
||||
plan = plan_merge(db, user, pl)
|
||||
assert plan["add_local"] == 0, "no perpetual pending pull — the playlist is up to date"
|
||||
r2 = merge_playlist(db, user, pl)
|
||||
assert r2["pulled"] == 0 and r2["failures"] == []
|
||||
assert parse_unavailable(pl) == {"x"}
|
||||
assert "x" in fake.video_ids
|
||||
|
||||
|
||||
def test_unavailable_video_that_becomes_available_is_pulled_in(db, make_user, _fake_yt):
|
||||
# A video marked un-ingestable (e.g. private) that later becomes available (public) must
|
||||
# re-enter the merge on a normal sync — not stay skipped forever.
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b"])
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"}))
|
||||
|
||||
merge_playlist(db, user, pl)
|
||||
assert parse_unavailable(pl) == {"x"}, "recorded un-ingestable on the first sync"
|
||||
|
||||
# 'x' becomes available; the next sync re-attempts ingestion and pulls it in.
|
||||
fake.unavailable.clear()
|
||||
r2 = merge_playlist(db, user, pl)
|
||||
|
||||
assert r2["failures"] == []
|
||||
assert "x" in set(_local_ids(db, pl)), "the now-available video is pulled into Siftlode"
|
||||
assert parse_unavailable(pl) == set(), "'x' is no longer marked un-ingestable"
|
||||
|
||||
|
||||
def test_unavailable_id_that_leaves_youtube_is_pruned(db, make_user, _fake_yt):
|
||||
# A recorded un-ingestable id that is later removed from the YouTube playlist must be pruned
|
||||
# from the persisted `unavailable` set (the `& yt_id_set` self-heal), not carried forever.
|
||||
user = make_user()
|
||||
_seed_videos(db, ["a", "b"])
|
||||
pl = _linked_playlist(db, user, local_ids=["a", "b"], baseline=["a", "b"])
|
||||
fake = _fake_yt(FakeYouTube(title="Mix", items=["a", "b", "x"], unavailable={"x"}))
|
||||
|
||||
merge_playlist(db, user, pl)
|
||||
assert parse_unavailable(pl) == {"x"}, "recorded un-ingestable on the first sync"
|
||||
|
||||
# 'x' is removed from the YouTube playlist entirely; the next sync must drop it.
|
||||
fake.items = [row for row in fake.items if row[1] != "x"]
|
||||
r2 = merge_playlist(db, user, pl)
|
||||
|
||||
assert r2["failures"] == []
|
||||
assert parse_unavailable(pl) == set(), "'x' pruned once it left YouTube"
|
||||
@@ -9,11 +9,12 @@ Two independent pieces:
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import PlexItem, PlexLibrary, PlexState, User
|
||||
from app.plex.watch_sync import _pull_apply
|
||||
from app.plex.watch_sync import _pull_apply, _reconcile_inprogress_movies
|
||||
from app.routes.plex.playback import (
|
||||
_PUSH_THROTTLE_S,
|
||||
_resume_push_superseded,
|
||||
@@ -33,6 +34,16 @@ def _seed_item_and_user(db):
|
||||
return item.id, user.id
|
||||
|
||||
|
||||
class _FakePlex:
|
||||
"""Minimal PlexClient stand-in for the movie-probe tests: metadata(rating_key) → a leaf dict."""
|
||||
|
||||
def __init__(self, by_rk: dict[str, dict]):
|
||||
self._by_rk = by_rk
|
||||
|
||||
def metadata(self, rating_key: str, markers: bool = False):
|
||||
return self._by_rk.get(str(rating_key))
|
||||
|
||||
|
||||
def test_pull_plex_wins_over_diverging_synced_local_resume(db):
|
||||
"""The core S2 change: a SYNCED local row (its state already reached Plex) that later diverges from
|
||||
Plex = a real external Plex-side change, so the incremental pull resolves to PLEX's position — Plex
|
||||
@@ -198,6 +209,130 @@ def test_pull_settles_flag_on_matching_dirty_row(db):
|
||||
assert st.synced_to_plex is True, "the dirty flag is settled so it isn't re-pushed endlessly"
|
||||
|
||||
|
||||
def test_movie_probe_flips_stale_in_progress_to_watched(db):
|
||||
"""S1 core: a local in-progress MOVIE that Plex has since finished (viewCount>0) and dropped from
|
||||
Continue-Watching flips to watched on the incremental pass — without waiting for the daily reconcile.
|
||||
This is the "movie stuck in progress after scrubbing the credits" bug."""
|
||||
item_id, user_id = _seed_item_and_user(db)
|
||||
now = datetime.now(timezone.utc)
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user_id,
|
||||
item_id=item_id,
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=now,
|
||||
synced_to_plex=True, # was created by a prior pull; the user then watched it on Plex
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
# Plex now reports the movie watched (viewCount>0, no offset) and it is NOT on the deck.
|
||||
plex = _FakePlex({"m1": {"viewCount": 1, "lastViewedAt": int(now.timestamp())}})
|
||||
out = _reconcile_inprogress_movies(
|
||||
db, plex, SimpleNamespace(user_id=user_id), on_deck_ids=set()
|
||||
)
|
||||
db.flush()
|
||||
|
||||
assert out == {"watched": 1, "in_progress": 0, "probed": 1}
|
||||
st = (
|
||||
db.execute(
|
||||
select(PlexState).where(PlexState.user_id == user_id, PlexState.item_id == item_id)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
assert st.status == "watched"
|
||||
assert st.position_seconds == 0
|
||||
|
||||
|
||||
def test_movie_probe_skips_ondeck_and_episodes(db):
|
||||
"""The probe must NOT touch (a) a movie Plex still lists on the deck — it's legitimately in
|
||||
progress — nor (b) an in-progress EPISODE (episodes ride onDeck/history + rollup). Only stale
|
||||
in-progress movies are probed, and metadata() is called for those alone."""
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
on_deck_movie = PlexItem(rating_key="mDeck", library_id=lib.id, kind="movie", title="On deck")
|
||||
episode = PlexItem(rating_key="e1", library_id=lib.id, kind="episode", title="An episode")
|
||||
db.add_all([on_deck_movie, episode])
|
||||
user = User(email="probe2@example.com")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
now = datetime.now(timezone.utc)
|
||||
for iid in (on_deck_movie.id, episode.id):
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=iid,
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=now,
|
||||
synced_to_plex=True,
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
# Plex would report both watched — but neither should be probed at all, so the fake returning
|
||||
# "watched" must have NO effect.
|
||||
plex = _FakePlex(
|
||||
{
|
||||
"mDeck": {"viewCount": 1, "lastViewedAt": int(now.timestamp())},
|
||||
"e1": {"viewCount": 1, "lastViewedAt": int(now.timestamp())},
|
||||
}
|
||||
)
|
||||
out = _reconcile_inprogress_movies(
|
||||
db, plex, SimpleNamespace(user_id=user.id), on_deck_ids={on_deck_movie.id}
|
||||
)
|
||||
db.flush()
|
||||
|
||||
assert out == {"watched": 0, "in_progress": 0, "probed": 0}, "neither item is probed"
|
||||
for iid in (on_deck_movie.id, episode.id):
|
||||
st = (
|
||||
db.execute(
|
||||
select(PlexState).where(PlexState.user_id == user.id, PlexState.item_id == iid)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
assert st.status == "new" and st.position_seconds == 300, "left untouched"
|
||||
|
||||
|
||||
def test_movie_probe_leaves_row_when_plex_has_no_signal(db):
|
||||
"""If Plex reports no watch signal (viewCount 0, no offset) for a probed movie, the incremental pass
|
||||
must NOT clear the local in-progress row — un-watch is the daily reconcile's job, and a Siftlode-only
|
||||
in-progress state must survive."""
|
||||
item_id, user_id = _seed_item_and_user(db)
|
||||
now = datetime.now(timezone.utc)
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user_id,
|
||||
item_id=item_id,
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=now,
|
||||
synced_to_plex=True,
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
plex = _FakePlex({"m1": {"viewCount": 0}})
|
||||
out = _reconcile_inprogress_movies(
|
||||
db, plex, SimpleNamespace(user_id=user_id), on_deck_ids=set()
|
||||
)
|
||||
db.flush()
|
||||
|
||||
assert out == {"watched": 0, "in_progress": 0, "probed": 1}
|
||||
st = (
|
||||
db.execute(
|
||||
select(PlexState).where(PlexState.user_id == user_id, PlexState.item_id == item_id)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
assert st.status == "new" and st.position_seconds == 300, "in-progress row preserved"
|
||||
|
||||
|
||||
def test_should_push_resume_scopes_by_user_and_item():
|
||||
"""The throttle window is keyed by the FULL (user, item) tuple: pushing for one key must not
|
||||
suppress a distinct user or a distinct item within the same window."""
|
||||
|
||||
@@ -34,6 +34,7 @@ import type { FeedView } from "../lib/feedView";
|
||||
import { channelYouTubeUrl, formatViews } from "../lib/format";
|
||||
import { ChannelTagsControl } from "./ChannelTags";
|
||||
import { useChannelTagActions } from "../lib/useChannelTagActions";
|
||||
import { Button } from "./ui/Button";
|
||||
|
||||
// A dedicated channel page: an "About"-style header (banner, avatar, stats, subscribe) over the
|
||||
// channel's videos (the catalog filtered to this channel). For an un-subscribed channel it
|
||||
@@ -441,21 +442,18 @@ export default function ChannelPage({
|
||||
it) so a large channel's paced tail reads as visible progress rather than a stuck
|
||||
spinner. */}
|
||||
{ch?.subscribed && !ch.backfill_done && !me.is_demo && (
|
||||
<button
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => !ch.deep_requested && deepBackfill.mutate()}
|
||||
disabled={ch.deep_requested || deepBackfill.isPending}
|
||||
loading={ch.deep_requested}
|
||||
icon={<History className="w-4 h-4" />}
|
||||
title={
|
||||
ch.deep_requested
|
||||
? t("channel.fullHistoryLoadingHint")
|
||||
: t("channel.fullHistory")
|
||||
}
|
||||
className="inline-flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm border border-border text-muted hover:text-accent hover:border-accent disabled:opacity-60 disabled:hover:text-muted disabled:hover:border-border transition"
|
||||
>
|
||||
{ch.deep_requested ? (
|
||||
<Loader2 className="w-4 h-4 animate-spin" />
|
||||
) : (
|
||||
<History className="w-4 h-4" />
|
||||
)}
|
||||
<span className="hidden sm:inline whitespace-nowrap">
|
||||
{!ch.deep_requested
|
||||
? t("channel.fullHistory")
|
||||
@@ -466,7 +464,7 @@ export default function ChannelPage({
|
||||
})
|
||||
: t("channel.fullHistoryLoading")}
|
||||
</span>
|
||||
</button>
|
||||
</Button>
|
||||
)}
|
||||
{!me.is_demo && (
|
||||
<button
|
||||
@@ -504,24 +502,24 @@ export default function ChannelPage({
|
||||
{!me.is_demo &&
|
||||
!ch?.blocked &&
|
||||
(ch?.subscribed ? (
|
||||
<button
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={onUnsubscribe}
|
||||
disabled={unsubscribe.isPending}
|
||||
title={t("channel.unsubscribe")}
|
||||
className="inline-flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm border border-border text-muted hover:text-fg disabled:opacity-50"
|
||||
icon={<Check className="w-4 h-4" />}
|
||||
>
|
||||
<Check className="w-4 h-4" />
|
||||
{t("channel.subscribedState")}
|
||||
</button>
|
||||
</Button>
|
||||
) : (
|
||||
<button
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => subscribe.mutate()}
|
||||
disabled={subscribe.isPending}
|
||||
className="inline-flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm btn-primary disabled:opacity-50"
|
||||
icon={<Plus className="w-4 h-4" />}
|
||||
>
|
||||
<Plus className="w-4 h-4" />
|
||||
{t("channel.subscribe")}
|
||||
</button>
|
||||
</Button>
|
||||
))}
|
||||
{/* Tagging is a subscription-scoped action (attach_tag → _user_subscription 404s
|
||||
otherwise), and personal tags are for your subscriptions — so only offer it on a
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createContext, useCallback, useContext, useState, type ReactNode } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import Modal from "./Modal";
|
||||
import { Button } from "./ui/Button";
|
||||
|
||||
// App-styled, promise-based replacement for window.confirm. Wrap the tree in
|
||||
// <ConfirmProvider> once, then anywhere: const confirm = useConfirm();
|
||||
@@ -100,26 +101,25 @@ export function ConfirmProvider({ children }: { children: ReactNode }) {
|
||||
<div
|
||||
className={multi ? "flex flex-col gap-2 mt-5" : "flex justify-end gap-2 mt-5 flex-wrap"}
|
||||
>
|
||||
<button
|
||||
<Button
|
||||
variant="outline"
|
||||
autoFocus={anyDanger}
|
||||
onClick={() => close(null)}
|
||||
className={`px-4 py-2 rounded-lg text-sm border border-border text-muted hover:text-fg hover:bg-surface transition ${
|
||||
multi ? "w-full order-last" : ""
|
||||
}`}
|
||||
fullWidth={multi}
|
||||
className={multi ? "order-last" : ""}
|
||||
>
|
||||
{state.opts.cancelLabel ?? t("common.cancel")}
|
||||
</button>
|
||||
</Button>
|
||||
{choices.map((c, i) => (
|
||||
<button
|
||||
<Button
|
||||
key={c.id}
|
||||
variant={c.danger ? "danger" : "primary"}
|
||||
autoFocus={!anyDanger && i === choices.length - 1}
|
||||
onClick={() => close(c.id)}
|
||||
className={`px-4 py-2 rounded-lg text-sm font-semibold transition ${
|
||||
multi ? "w-full" : ""
|
||||
} ${c.danger ? "btn-danger" : "btn-primary"}`}
|
||||
fullWidth={multi}
|
||||
>
|
||||
{c.label || t("common.confirm")}
|
||||
</button>
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
</Modal>
|
||||
|
||||
@@ -31,6 +31,7 @@ import { useMe } from "../lib/useMe";
|
||||
import { api, type DownloadJob } from "../lib/api";
|
||||
import { invalidateDownloads } from "../lib/downloads";
|
||||
import { inputCls } from "./ui/form";
|
||||
import { Button } from "./ui/Button";
|
||||
import { DownloadRow, IconBtn, UsageBar } from "./downloadRow";
|
||||
import { MetaEditModal, ErrorModal } from "./downloadModals";
|
||||
import { AdminSystem } from "./downloadAdmin";
|
||||
@@ -51,7 +52,13 @@ export default function DownloadCenter() {
|
||||
const [editJob, setEditJob] = useState<DownloadJob | null>(null);
|
||||
const [errorJob, setErrorJob] = useState<DownloadJob | null>(null);
|
||||
|
||||
const jobsQ = useLiveQuery(qk.downloads(), api.downloads, { intervalMs: 2000 });
|
||||
// Poll fast (2s) while work is actually in flight; back off to 15s once the queue is settled
|
||||
// (only done/error/paused rows left, which don't advance on their own) so an idle Download Center
|
||||
// isn't hitting the API — and adding to the shared box's load — every 2s for nothing.
|
||||
const jobsQ = useLiveQuery(qk.downloads(), api.downloads, {
|
||||
intervalMs: (data) =>
|
||||
data?.some((j) => j.status === "queued" || j.status === "running") ? 2000 : 15000,
|
||||
});
|
||||
const sharedQ = useQuery({
|
||||
queryKey: qk.downloadsShared(),
|
||||
queryFn: api.downloadsShared,
|
||||
@@ -163,20 +170,24 @@ export default function DownloadCenter() {
|
||||
placeholder={t("downloads.page.addUrl")}
|
||||
className={inputCls}
|
||||
/>
|
||||
<button
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
iconOnly
|
||||
icon={<ClipboardPaste className="w-4 h-4" />}
|
||||
label={t("downloads.page.paste")}
|
||||
onClick={() => pasteFromClipboard(true)}
|
||||
title={t("downloads.page.paste")}
|
||||
aria-label={t("downloads.page.paste")}
|
||||
className="inline-flex items-center justify-center px-2.5 py-1.5 rounded-lg text-muted hover:text-fg hover:bg-surface transition shrink-0"
|
||||
>
|
||||
<ClipboardPaste className="w-4 h-4" />
|
||||
</button>
|
||||
<button
|
||||
className="shrink-0"
|
||||
/>
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
icon={<Plus className="w-4 h-4" />}
|
||||
onClick={() => addUrl.trim() && setAddSource(addUrl.trim())}
|
||||
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-sm font-medium btn-primary transition shrink-0"
|
||||
className="shrink-0"
|
||||
>
|
||||
<Plus className="w-4 h-4" /> {t("downloads.page.add")}
|
||||
</button>
|
||||
{t("downloads.page.add")}
|
||||
</Button>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
{queue.map((job) => (
|
||||
|
||||
@@ -3,6 +3,7 @@ import { qk } from "../lib/queryKeys";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import Modal from "./Modal";
|
||||
import { Button } from "./ui/Button";
|
||||
import { api } from "../lib/api";
|
||||
import { invalidateDownloads } from "../lib/downloads";
|
||||
|
||||
@@ -89,19 +90,12 @@ export default function DownloadDialog({
|
||||
/>
|
||||
|
||||
<div className="flex justify-end gap-2">
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="px-3 py-1.5 rounded-lg text-sm text-muted hover:text-fg hover:bg-surface transition"
|
||||
>
|
||||
<Button variant="ghost" size="sm" onClick={onClose}>
|
||||
{t("downloads.actions.cancel")}
|
||||
</button>
|
||||
<button
|
||||
onClick={submit}
|
||||
disabled={busy || !chosen}
|
||||
className="px-3 py-1.5 rounded-lg text-sm font-medium btn-primary disabled:opacity-50 transition"
|
||||
>
|
||||
</Button>
|
||||
<Button variant="primary" size="sm" onClick={submit} disabled={busy || !chosen}>
|
||||
{t("downloads.dialog.submit")}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{manage && (
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useSyncExternalStore } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { dismissError, getError, subscribeError } from "../lib/errorDialog";
|
||||
import Modal from "./Modal";
|
||||
import { Button } from "./ui/Button";
|
||||
|
||||
// Renders the current app error (if any) as a modal the user must acknowledge.
|
||||
export default function ErrorDialog() {
|
||||
@@ -12,12 +13,7 @@ export default function ErrorDialog() {
|
||||
<Modal title={err.title} onClose={dismissError} maxWidth="max-w-md">
|
||||
<p className="text-sm leading-relaxed text-fg/90">{err.message}</p>
|
||||
<div className="flex justify-end mt-5">
|
||||
<button
|
||||
onClick={dismissError}
|
||||
className="px-4 py-2 rounded-lg btn-primary text-sm font-medium transition"
|
||||
>
|
||||
{t("errors.ok")}
|
||||
</button>
|
||||
<Button onClick={dismissError}>{t("errors.ok")}</Button>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
import { CSS } from "@dnd-kit/utilities";
|
||||
import { coercePlexFilters, type FeedFilters, type PlexFilters, type SavedView } from "../lib/api";
|
||||
import Modal from "./Modal";
|
||||
import { Button } from "./ui/Button";
|
||||
import { useSavedViews, asFilters } from "./SavedViewsProvider";
|
||||
import { useActiveFilters } from "../lib/useActiveFilters";
|
||||
import { buildPlexActiveFilters } from "./usePlexActiveFilters";
|
||||
@@ -202,14 +203,16 @@ export default function PersonalViewsModal() {
|
||||
placeholder={t("views.namePlaceholder")}
|
||||
className="min-w-0 flex-1 rounded-lg border border-border bg-surface px-2 py-1.5 text-sm focus:border-accent focus:outline-hidden focus:ring-2 focus:ring-accent/30"
|
||||
/>
|
||||
<button
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
iconOnly
|
||||
icon={<BookmarkPlus className="h-4 w-4" />}
|
||||
label={t("views.save")}
|
||||
onClick={onCreate}
|
||||
disabled={!createName.trim()}
|
||||
title={t("views.save")}
|
||||
className="inline-flex shrink-0 items-center gap-1 rounded-lg btn-primary px-2.5 py-1.5 text-sm font-semibold disabled:opacity-40"
|
||||
>
|
||||
<BookmarkPlus className="h-4 w-4" />
|
||||
</button>
|
||||
className="shrink-0"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -272,13 +275,14 @@ export default function PersonalViewsModal() {
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-auto flex flex-wrap items-center gap-2 border-t border-border/60 pt-3">
|
||||
<button
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
icon={<Play className="h-3.5 w-3.5" />}
|
||||
onClick={() => apply(selected)}
|
||||
className="inline-flex items-center gap-1.5 rounded-lg btn-primary px-3 py-1.5 text-sm font-semibold"
|
||||
>
|
||||
<Play className="h-3.5 w-3.5" />
|
||||
{t("views.apply")}
|
||||
</button>
|
||||
</Button>
|
||||
<button
|
||||
onClick={async () => {
|
||||
const saved = await saveCurrent(selected.id);
|
||||
|
||||
@@ -362,7 +362,15 @@ export default function Playlists() {
|
||||
setPushing(true);
|
||||
try {
|
||||
const plan = await api.playlistPushPlan(selectedId);
|
||||
if (plan.action === "update" && !plan.to_insert && !plan.to_delete && !plan.to_reorder) {
|
||||
const nothingToDo =
|
||||
plan.action === "update" &&
|
||||
!plan.add_yt &&
|
||||
!plan.add_local &&
|
||||
!plan.remove_yt &&
|
||||
!plan.remove_local &&
|
||||
!plan.to_reorder &&
|
||||
!plan.rename;
|
||||
if (nothingToDo) {
|
||||
notify({ level: "info", message: t("playlists.pushUpToDate", { name: plName(detail) }) });
|
||||
return;
|
||||
}
|
||||
@@ -376,21 +384,37 @@ export default function Playlists() {
|
||||
});
|
||||
return;
|
||||
}
|
||||
let message =
|
||||
plan.action === "create"
|
||||
? t("playlists.pushPlanCreate", {
|
||||
count: plan.to_insert,
|
||||
units: plan.units_estimate,
|
||||
left: plan.remaining_today,
|
||||
})
|
||||
: t("playlists.pushPlanUpdate", {
|
||||
insert: plan.to_insert,
|
||||
del: plan.to_delete,
|
||||
reorder: plan.to_reorder,
|
||||
units: plan.units_estimate,
|
||||
left: plan.remaining_today,
|
||||
});
|
||||
if (plan.yt_extra > 0) message += " " + t("playlists.pushDiverged", { count: plan.yt_extra });
|
||||
let message: string;
|
||||
if (plan.action === "create") {
|
||||
message = t("playlists.pushPlanCreate", {
|
||||
count: plan.add_yt,
|
||||
units: plan.units_estimate,
|
||||
left: plan.remaining_today,
|
||||
});
|
||||
} else {
|
||||
// One sentence per side that actually changes; the untouched side gets a short "stays
|
||||
// unchanged" reassurance instead of a "+0 / −0" the reader has to decode.
|
||||
const here: string[] = [];
|
||||
if (plan.add_local) here.push(t("playlists.mergeAddLocal", { count: plan.add_local }));
|
||||
if (plan.remove_local)
|
||||
here.push(t("playlists.mergeRemoveLocal", { count: plan.remove_local }));
|
||||
const yt: string[] = [];
|
||||
if (plan.add_yt) yt.push(t("playlists.mergeAddYt", { count: plan.add_yt }));
|
||||
if (plan.remove_yt) yt.push(t("playlists.mergeRemoveYt", { count: plan.remove_yt }));
|
||||
if (plan.to_reorder) yt.push(t("playlists.mergeReorder", { count: plan.to_reorder }));
|
||||
if (plan.rename) yt.push(t("playlists.mergeRename"));
|
||||
const sentences = [...here, ...yt];
|
||||
if (here.length && !yt.length) sentences.push(t("playlists.mergeYtUnchanged"));
|
||||
if (yt.length && !here.length) sentences.push(t("playlists.mergeLocalUnchanged"));
|
||||
const quota =
|
||||
plan.units_estimate > 0
|
||||
? t("playlists.mergeQuota", {
|
||||
units: plan.units_estimate,
|
||||
left: plan.remaining_today,
|
||||
})
|
||||
: t("playlists.mergeQuotaFree");
|
||||
message = sentences.join(" ") + quota;
|
||||
}
|
||||
const ok = await confirm({
|
||||
title: t("playlists.pushTitle"),
|
||||
message,
|
||||
|
||||
@@ -7,6 +7,7 @@ import Modal from "./Modal";
|
||||
import { useConfirm } from "./ConfirmProvider";
|
||||
import { api, type DownloadProfile, type DownloadSpec } from "../lib/api";
|
||||
import { inputCls } from "./ui/form";
|
||||
import { Button } from "./ui/Button";
|
||||
|
||||
const HEIGHTS = [null, 2160, 1440, 1080, 720, 480, 360];
|
||||
|
||||
@@ -145,12 +146,9 @@ export default function ProfileEditor({ onClose }: { onClose: () => void }) {
|
||||
</div>
|
||||
|
||||
{!formOpen ? (
|
||||
<button
|
||||
onClick={startNew}
|
||||
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-sm font-medium btn-primary transition"
|
||||
>
|
||||
<Plus className="w-4 h-4" /> {t("downloads.profiles.new")}
|
||||
</button>
|
||||
<Button variant="primary" size="sm" icon={<Plus className="w-4 h-4" />} onClick={startNew}>
|
||||
{t("downloads.profiles.new")}
|
||||
</Button>
|
||||
) : (
|
||||
<div className="rounded-xl border border-border p-4 space-y-3">
|
||||
<div>
|
||||
@@ -263,19 +261,17 @@ export default function ProfileEditor({ onClose }: { onClose: () => void }) {
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-end gap-2 pt-1">
|
||||
<button
|
||||
onClick={() => setFormOpen(false)}
|
||||
className="px-3 py-1.5 rounded-lg text-sm text-muted hover:text-fg hover:bg-surface transition"
|
||||
>
|
||||
<Button variant="ghost" size="sm" onClick={() => setFormOpen(false)}>
|
||||
{t("downloads.actions.cancel")}
|
||||
</button>
|
||||
<button
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
onClick={() => save.mutate()}
|
||||
disabled={!name.trim() || save.isPending}
|
||||
className="px-3 py-1.5 rounded-lg text-sm font-medium btn-primary disabled:opacity-50 transition"
|
||||
>
|
||||
{editing ? t("downloads.profiles.save") : t("downloads.profiles.create")}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// S6 — the per-user download-cookies section is WRITE-ONLY: the stored blob is never fetched or
|
||||
// shown back, so opening "Set"/"Update" always reveals an EMPTY field, and saving posts exactly the
|
||||
// pasted value to the matching host. This drives the real DownloadAuth against a mocked api so we can
|
||||
// assert the field never carries a prior value and the right host is written.
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { render, screen, cleanup, fireEvent, waitFor } from "@testing-library/react";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
|
||||
const h = vi.hoisted(() => ({
|
||||
downloadCredentials: vi.fn(),
|
||||
setDownloadCredential: vi.fn(),
|
||||
deleteDownloadCredential: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("react-i18next", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("react-i18next")>();
|
||||
return {
|
||||
...actual,
|
||||
useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }),
|
||||
};
|
||||
});
|
||||
vi.mock("../lib/api", () => ({ api: h }));
|
||||
// Partial mock: SettingsPanel's module graph (PrefsProvider) also pulls getNotifSettings from here.
|
||||
vi.mock("../lib/notifications", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../lib/notifications")>();
|
||||
return { ...actual, notify: vi.fn() };
|
||||
});
|
||||
vi.mock("./ConfirmProvider", () => ({ useConfirm: () => vi.fn(async () => true) }));
|
||||
|
||||
import { DownloadAuth } from "./SettingsPanel";
|
||||
|
||||
const HOSTS = [
|
||||
{ host_key: "facebook", label: "Facebook", configured: false, updated_at: null },
|
||||
{
|
||||
host_key: "instagram",
|
||||
label: "Instagram",
|
||||
configured: true,
|
||||
updated_at: "2026-08-01T00:00:00Z",
|
||||
},
|
||||
];
|
||||
|
||||
function renderPanel() {
|
||||
const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } });
|
||||
return render(
|
||||
<QueryClientProvider client={qc}>
|
||||
<DownloadAuth />
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
describe("DownloadAuth (write-only cookies)", () => {
|
||||
beforeEach(() => {
|
||||
h.downloadCredentials.mockResolvedValue({ hosts: HOSTS });
|
||||
h.setDownloadCredential.mockResolvedValue({ ok: true, hosts: HOSTS });
|
||||
h.deleteDownloadCredential.mockResolvedValue({ ok: true, hosts: HOSTS });
|
||||
});
|
||||
afterEach(cleanup);
|
||||
|
||||
it("lists every supported host with its configured state", async () => {
|
||||
renderPanel();
|
||||
await screen.findByTestId("cookie-host-facebook");
|
||||
expect(screen.getByTestId("cookie-status-facebook").textContent).toContain(
|
||||
"settings.downloadAuth.notSet",
|
||||
);
|
||||
expect(screen.getByTestId("cookie-status-instagram").textContent).toContain(
|
||||
"settings.downloadAuth.configured",
|
||||
);
|
||||
});
|
||||
|
||||
it("opens an EMPTY field on Set and posts the pasted value to that host", async () => {
|
||||
renderPanel();
|
||||
await screen.findByTestId("cookie-host-facebook");
|
||||
// The Set button is the outline action on the facebook row.
|
||||
const setBtn = screen.getByText("settings.downloadAuth.set");
|
||||
fireEvent.click(setBtn);
|
||||
const field = (await screen.findByTestId("cookie-input-facebook")) as HTMLTextAreaElement;
|
||||
expect(field.value).toBe(""); // write-only: never prefilled
|
||||
|
||||
fireEvent.change(field, {
|
||||
target: { value: "# Netscape\n.facebook.com\tTRUE\t/\tT\t0\tc\t1\n" },
|
||||
});
|
||||
fireEvent.click(screen.getByText("settings.downloadAuth.save"));
|
||||
await waitFor(() =>
|
||||
expect(h.setDownloadCredential).toHaveBeenCalledWith(
|
||||
"facebook",
|
||||
"# Netscape\n.facebook.com\tTRUE\t/\tT\t0\tc\t1\n",
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it("re-opening Update on a configured host still shows an empty field", async () => {
|
||||
renderPanel();
|
||||
await screen.findByTestId("cookie-host-instagram");
|
||||
fireEvent.click(screen.getByText("settings.downloadAuth.update"));
|
||||
const field = (await screen.findByTestId("cookie-input-instagram")) as HTMLTextAreaElement;
|
||||
expect(field.value).toBe(""); // the stored blob is never fetched into the field
|
||||
});
|
||||
|
||||
it("renders an honest error state with retry when the fetch fails, not an empty section", async () => {
|
||||
// Honest-states: a failed load must NOT render as a bare empty section (no host rows).
|
||||
h.downloadCredentials.mockRejectedValueOnce(new Error("boom"));
|
||||
h.downloadCredentials.mockResolvedValue({ hosts: HOSTS });
|
||||
renderPanel();
|
||||
// The error message shows and no host rows render.
|
||||
await screen.findByText("settings.downloadAuth.loadError");
|
||||
expect(screen.queryByTestId("cookie-host-facebook")).toBeNull();
|
||||
// Retry refetches → the list appears.
|
||||
fireEvent.click(screen.getByText("common.retry"));
|
||||
await screen.findByTestId("cookie-host-facebook");
|
||||
});
|
||||
});
|
||||
@@ -11,8 +11,10 @@ import Avatar from "./Avatar";
|
||||
import { notify, type NotifSettings } from "../lib/notifications";
|
||||
import Tooltip from "./Tooltip";
|
||||
import { inputCls, Section, SettingRow, Switch } from "./ui/form";
|
||||
import { Button } from "./ui/Button";
|
||||
import { type SaveState } from "./ui/DraftSaveBar";
|
||||
import { useConfirm } from "./ConfirmProvider";
|
||||
import { StateMessage } from "./QueryState";
|
||||
import { useWizard } from "./WizardProvider";
|
||||
import { usePrefs } from "./PrefsProvider";
|
||||
import Tabs from "./Tabs";
|
||||
@@ -472,13 +474,9 @@ function SignInMethods({ me }: { me: Me }) {
|
||||
className={inputCls}
|
||||
/>
|
||||
{err && <p className="text-xs text-danger">{err}</p>}
|
||||
<button
|
||||
type="submit"
|
||||
disabled={busy || !next}
|
||||
className="px-3 py-1.5 rounded-lg text-sm btn-primary font-medium disabled:opacity-40 transition"
|
||||
>
|
||||
<Button type="submit" variant="primary" size="sm" disabled={busy || !next}>
|
||||
{busy ? t("settings.account.password.saving") : t("settings.account.password.save")}
|
||||
</button>
|
||||
</Button>
|
||||
</form>
|
||||
)}
|
||||
</div>
|
||||
@@ -586,6 +584,161 @@ function PlexWatchSync() {
|
||||
);
|
||||
}
|
||||
|
||||
// Per-user download authentication (S6): store your own login cookies for an auth-walled host
|
||||
// (e.g. a private Facebook group you belong to) so downloads from it use YOUR login. Write-only —
|
||||
// the stored blob is never shown back; each row shows only the host + whether it's set + when.
|
||||
// A cookie-authed download is kept private to your account (never cache-shared with another user).
|
||||
// Exported for a focused DOM test (write-only behaviour); rendered inside Account below.
|
||||
export function DownloadAuth() {
|
||||
const { t, i18n } = useTranslation();
|
||||
const qc = useQueryClient();
|
||||
const confirm = useConfirm();
|
||||
const q = useQuery({
|
||||
queryKey: qk.downloadCredentials(),
|
||||
queryFn: () => api.downloadCredentials(),
|
||||
});
|
||||
const [openHost, setOpenHost] = useState<string | null>(null);
|
||||
const [draft, setDraft] = useState("");
|
||||
|
||||
const save = useMutation({
|
||||
mutationFn: ({ host, cookies }: { host: string; cookies: string }) =>
|
||||
api.setDownloadCredential(host, cookies),
|
||||
onSuccess: (res) => {
|
||||
qc.setQueryData(qk.downloadCredentials(), res);
|
||||
setOpenHost(null);
|
||||
setDraft("");
|
||||
notify({ level: "success", message: t("settings.downloadAuth.saved") });
|
||||
},
|
||||
// Validation errors (empty / not-a-cookies.txt) surface via the global error modal, which
|
||||
// localizes the structured { code: "cookies" } detail (see lib/api core localizeDetail).
|
||||
});
|
||||
const remove = useMutation({
|
||||
mutationFn: (host: string) => api.deleteDownloadCredential(host),
|
||||
onSuccess: (res) => {
|
||||
qc.setQueryData(qk.downloadCredentials(), res);
|
||||
notify({ level: "success", message: t("settings.downloadAuth.removed") });
|
||||
},
|
||||
});
|
||||
|
||||
const hosts = q.data?.hosts ?? [];
|
||||
const onRemove = async (host: string) => {
|
||||
const ok = await confirm({
|
||||
title: t("settings.downloadAuth.removeConfirmTitle"),
|
||||
message: t("settings.downloadAuth.removeConfirmBody"),
|
||||
confirmLabel: t("settings.downloadAuth.removeConfirmAction"),
|
||||
danger: true,
|
||||
});
|
||||
if (ok) remove.mutate(host);
|
||||
};
|
||||
|
||||
return (
|
||||
<Section title={t("settings.downloadAuth.title")}>
|
||||
<p className="text-xs text-muted leading-relaxed mb-2">{t("settings.downloadAuth.intro")}</p>
|
||||
{/* The host list is server-provided (always non-empty on success), so an empty `hosts` means
|
||||
the fetch failed or is still loading — render an honest state, never a bare empty section
|
||||
that reads as "nothing to configure here". */}
|
||||
{q.isError && hosts.length === 0 ? (
|
||||
<StateMessage tone="error" onRetry={() => q.refetch()}>
|
||||
{t("settings.downloadAuth.loadError")}
|
||||
</StateMessage>
|
||||
) : q.isLoading ? (
|
||||
<StateMessage>{t("common.loading")}</StateMessage>
|
||||
) : (
|
||||
<div className="divide-y divide-border">
|
||||
{hosts.map((h) => {
|
||||
const when = h.updated_at
|
||||
? new Date(h.updated_at).toLocaleDateString(i18n.language)
|
||||
: "";
|
||||
const open = openHost === h.host_key;
|
||||
return (
|
||||
<div key={h.host_key} className="py-2" data-testid={`cookie-host-${h.host_key}`}>
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<div className="text-sm font-medium">{h.label}</div>
|
||||
<p
|
||||
className="text-xs text-muted mt-0.5"
|
||||
data-testid={`cookie-status-${h.host_key}`}
|
||||
>
|
||||
{h.configured
|
||||
? t("settings.downloadAuth.configured", { when })
|
||||
: t("settings.downloadAuth.notSet")}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 shrink-0">
|
||||
{h.configured && (
|
||||
<Button
|
||||
variant="danger-outline"
|
||||
size="sm"
|
||||
onClick={() => onRemove(h.host_key)}
|
||||
disabled={remove.isPending}
|
||||
>
|
||||
{t("settings.downloadAuth.remove")}
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
setOpenHost(open ? null : h.host_key);
|
||||
setDraft("");
|
||||
}}
|
||||
>
|
||||
{h.configured
|
||||
? t("settings.downloadAuth.update")
|
||||
: t("settings.downloadAuth.set")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
{open && (
|
||||
<div className="mt-2 space-y-2">
|
||||
{/* Write-only: the field is ALWAYS empty on open — the stored blob is never
|
||||
fetched or prefilled. */}
|
||||
<textarea
|
||||
value={draft}
|
||||
onChange={(e) => setDraft(e.target.value)}
|
||||
rows={5}
|
||||
placeholder={t("settings.downloadAuth.placeholder")}
|
||||
className={`${inputCls} font-mono text-xs resize-y`}
|
||||
data-testid={`cookie-input-${h.host_key}`}
|
||||
spellCheck={false}
|
||||
autoComplete="off"
|
||||
/>
|
||||
<p className="text-2xs text-muted leading-relaxed">
|
||||
{t("settings.downloadAuth.help")}
|
||||
</p>
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
disabled={!draft.trim() || save.isPending}
|
||||
onClick={() => save.mutate({ host: h.host_key, cookies: draft })}
|
||||
>
|
||||
{save.isPending
|
||||
? t("settings.downloadAuth.saving")
|
||||
: t("settings.downloadAuth.save")}
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
setOpenHost(null);
|
||||
setDraft("");
|
||||
}}
|
||||
>
|
||||
{t("settings.downloadAuth.cancel")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</Section>
|
||||
);
|
||||
}
|
||||
|
||||
function Account({ me, onOpenWizard }: { me: Me; onOpenWizard: () => void }) {
|
||||
const { t } = useTranslation();
|
||||
const confirm = useConfirm();
|
||||
@@ -628,6 +781,8 @@ function Account({ me, onOpenWizard }: { me: Me; onOpenWizard: () => void }) {
|
||||
|
||||
{me.role === "admin" && me.plex_enabled && <PlexWatchSync />}
|
||||
|
||||
{!me.is_demo && <DownloadAuth />}
|
||||
|
||||
{me.is_demo ? (
|
||||
<Section title={t("settings.account.youtubeAccess")}>
|
||||
<p className="text-xs text-muted leading-relaxed">{t("settings.account.demoNotice")}</p>
|
||||
|
||||
@@ -9,7 +9,8 @@ import { useConfirm } from "./ConfirmProvider";
|
||||
import { notify } from "../lib/notifications";
|
||||
import { copyText } from "../lib/clipboard";
|
||||
import { api, type DownloadJob, type ShareLink } from "../lib/api";
|
||||
import { inputCls, btnCls } from "./ui/form";
|
||||
import { inputCls } from "./ui/form";
|
||||
import { Button } from "./ui/Button";
|
||||
const EXPIRY_DAYS = [null, 1, 7, 30] as const;
|
||||
|
||||
// --- share with a registered user (ACL) -----------------------------------------------------
|
||||
@@ -343,31 +344,29 @@ function LinkShare({ job }: { job: DownloadJob }) {
|
||||
autoComplete="new-password"
|
||||
/>
|
||||
<div className="flex justify-end gap-2">
|
||||
<button
|
||||
onClick={() => setCreating(false)}
|
||||
className={clsx(btnCls, "text-muted hover:text-fg hover:bg-surface")}
|
||||
>
|
||||
<Button variant="ghost" size="sm" onClick={() => setCreating(false)}>
|
||||
{t("common.cancel")}
|
||||
</button>
|
||||
<button
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
onClick={() => create.mutate()}
|
||||
disabled={create.isPending}
|
||||
className={clsx(btnCls, "btn-primary")}
|
||||
>
|
||||
{t("downloads.share.createLink")}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<button
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => setCreating(true)}
|
||||
className={clsx(
|
||||
btnCls,
|
||||
"mt-2 border border-border text-muted hover:text-fg hover:border-muted inline-flex items-center gap-1.5",
|
||||
)}
|
||||
icon={<Link2 className="w-4 h-4" />}
|
||||
className="mt-2"
|
||||
>
|
||||
<Link2 className="w-4 h-4" /> {t("downloads.share.newLink")}
|
||||
</button>
|
||||
{t("downloads.share.newLink")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -55,7 +55,7 @@ export default function SortControl({
|
||||
const summary = activeOpt?.label ?? defaultLabel ?? options[0]?.label ?? "";
|
||||
|
||||
const segCls = (on: boolean) =>
|
||||
`flex flex-1 items-center justify-center gap-1 rounded-md border px-2 py-1 text-2xs transition ${
|
||||
`flex flex-1 items-center justify-center gap-1 whitespace-nowrap rounded-md border px-2 py-1 text-2xs transition ${
|
||||
on
|
||||
? "border-accent bg-accent/15 text-accent"
|
||||
: "border-border text-muted hover:border-accent/50 hover:text-fg"
|
||||
@@ -92,7 +92,7 @@ export default function SortControl({
|
||||
<PopoverSurface
|
||||
popover={pop}
|
||||
ariaLabel={label}
|
||||
className="glass-menu min-w-52 rounded-xl p-1.5 shadow-lg animate-[popIn_0.16s_ease]"
|
||||
className="glass-menu min-w-56 rounded-xl p-1.5 shadow-lg animate-[popIn_0.16s_ease]"
|
||||
>
|
||||
{/* One exclusive choice of sort key (radios), scoped by a group so AT announces which is
|
||||
checked — the surface is a menu (see the mode above), never a listbox with orphaned options. */}
|
||||
|
||||
@@ -2,22 +2,13 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { qk } from "../lib/queryKeys";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import {
|
||||
Crop,
|
||||
Eye,
|
||||
EyeOff,
|
||||
Pause,
|
||||
Play,
|
||||
Scissors,
|
||||
SplitSquareHorizontal,
|
||||
Loader2,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { Crop, Eye, EyeOff, Pause, Play, Scissors, SplitSquareHorizontal, X } from "lucide-react";
|
||||
import clsx from "clsx";
|
||||
import Modal from "./Modal";
|
||||
import { notify } from "../lib/notifications";
|
||||
import { api, type DownloadJob, type EditSpec } from "../lib/api";
|
||||
import { inputCls, btnCls } from "./ui/form";
|
||||
import { Button } from "./ui/Button";
|
||||
|
||||
// mm:ss.d timecode <-> seconds.
|
||||
function tc(s: number): string {
|
||||
@@ -621,28 +612,23 @@ export default function VideoEditor({ job, onClose }: { job: DownloadJob; onClos
|
||||
{reencode ? t("editor.willReencode") : t("editor.willCopy")}
|
||||
</span>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={onClose}
|
||||
className={clsx(btnCls, "text-muted hover:text-fg hover:bg-surface")}
|
||||
>
|
||||
<Button variant="ghost" size="sm" onClick={onClose}>
|
||||
{t("common.cancel")}
|
||||
</button>
|
||||
<button
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
onClick={() => create.mutate()}
|
||||
disabled={!valid || create.isPending}
|
||||
className={clsx(btnCls, "btn-primary inline-flex items-center gap-1.5")}
|
||||
disabled={!valid}
|
||||
loading={create.isPending}
|
||||
icon={<Scissors className="w-4 h-4" />}
|
||||
>
|
||||
{create.isPending ? (
|
||||
<Loader2 className="w-4 h-4 animate-spin" />
|
||||
) : (
|
||||
<Scissors className="w-4 h-4" />
|
||||
)}
|
||||
{willJoin
|
||||
? t("editor.createJoin")
|
||||
: kept.length > 1
|
||||
? t("editor.createN", { count: kept.length })
|
||||
: t("editor.create")}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// S3 item 5a: the public watch page must render a native <audio> element for an audio-only asset
|
||||
// (mp3) and a <video> for everything else. WebKit (all iOS browsers) refuses to play an audio-only
|
||||
// mp3 inside a <video>, which is exactly why the branch exists — this locks it so a refactor can't
|
||||
// silently drop the audio player and reintroduce the iPad-broken behavior.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { render, waitFor, cleanup } from "@testing-library/react";
|
||||
import WatchPage from "./WatchPage";
|
||||
|
||||
function mockMeta(meta: Record<string, unknown>) {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => ({ ok: true, json: async () => meta })),
|
||||
);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("WatchPage media element (S3 · 5a)", () => {
|
||||
it("renders a native <audio> for an audio-only asset", async () => {
|
||||
window.history.replaceState({}, "", "/watch/tok-audio");
|
||||
mockMeta({ needs_password: false, audio: true, file_url: "/f.mp3", title: "A song" });
|
||||
const { container } = render(<WatchPage />);
|
||||
await waitFor(() => expect(container.querySelector("audio")).not.toBeNull());
|
||||
expect(container.querySelector("video")).toBeNull();
|
||||
});
|
||||
|
||||
it("renders a <video> for a normal video asset", async () => {
|
||||
window.history.replaceState({}, "", "/watch/tok-video");
|
||||
mockMeta({ needs_password: false, audio: false, file_url: "/f.mp4", title: "A clip" });
|
||||
const { container } = render(<WatchPage />);
|
||||
await waitFor(() => expect(container.querySelector("video")).not.toBeNull());
|
||||
expect(container.querySelector("audio")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Download, Link2, Lock, PlayCircle } from "lucide-react";
|
||||
import { Button } from "./ui/Button";
|
||||
|
||||
// Standalone, login-free video page for a shared /watch/<token> link. Rendered OUTSIDE the app
|
||||
// tree (no auth, no providers) — like the /privacy and /terms pages. Uses plain fetch and a tiny
|
||||
@@ -11,10 +12,11 @@ import { Download, Link2, Lock, PlayCircle } from "lucide-react";
|
||||
// playing in the background on Brave iOS. We deliberately do NOT route the audio through the Web
|
||||
// Audio API: `createMediaElementSource` pulls the element out of the normal media pipeline that the
|
||||
// background feature (and iOS) hooks into, and iOS then suspends the AudioContext on background —
|
||||
// which STOPPED playback instead of keeping it alive (measured on iPad Brave). A plain <video> +
|
||||
// MediaSession leaves the browser's native background handling in charge.
|
||||
// which STOPPED playback instead of keeping it alive (measured on iPad Brave). A plain <video>/<audio>
|
||||
// + MediaSession leaves the browser's native background handling in charge. Typed on HTMLMediaElement
|
||||
// so it drives either the video or the audio-only player.
|
||||
function useMediaSession(
|
||||
videoRef: React.RefObject<HTMLVideoElement | null>,
|
||||
videoRef: React.RefObject<HTMLMediaElement | null>,
|
||||
meta: Meta | null,
|
||||
ready: boolean,
|
||||
) {
|
||||
@@ -45,6 +47,7 @@ type Meta = {
|
||||
source_url?: string | null;
|
||||
extra_links?: string[];
|
||||
duration_s?: number | null;
|
||||
audio?: boolean;
|
||||
allow_download?: boolean;
|
||||
file_url?: string;
|
||||
poster_url?: string | null;
|
||||
@@ -112,8 +115,13 @@ export default function WatchPage() {
|
||||
const [password, setPassword] = useState("");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [unlocking, setUnlocking] = useState(false);
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
useMediaSession(videoRef, meta, status === "ready");
|
||||
// One ref for whichever element renders (video or audio-only) — a callback ref so it can bind to
|
||||
// both without a per-element typed ref.
|
||||
const mediaRef = useRef<HTMLMediaElement | null>(null);
|
||||
const setMedia = (el: HTMLMediaElement | null) => {
|
||||
mediaRef.current = el;
|
||||
};
|
||||
useMediaSession(mediaRef, meta, status === "ready");
|
||||
|
||||
useEffect(() => {
|
||||
if (!token) {
|
||||
@@ -193,31 +201,43 @@ export default function WatchPage() {
|
||||
className="w-full rounded-lg bg-surface border border-border px-3 py-2 text-sm outline-hidden focus:ring-2 focus:ring-accent/40"
|
||||
/>
|
||||
{error && <div className="text-sm text-danger mt-2">{error}</div>}
|
||||
<button
|
||||
<Button
|
||||
variant="primary"
|
||||
size="md"
|
||||
fullWidth
|
||||
onClick={unlock}
|
||||
disabled={!password || unlocking}
|
||||
className="mt-4 w-full rounded-lg btn-primary disabled:opacity-50 py-2 text-sm font-medium transition"
|
||||
className="mt-4"
|
||||
>
|
||||
{T.watch}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status === "ready" && meta && (
|
||||
<div>
|
||||
{/* Shrink-wrap the player to the video so a vertical/short clip renders as a narrow
|
||||
player instead of a wide box with black bars; a landscape clip fills the width. */}
|
||||
<div className="rounded-xl overflow-hidden bg-black border border-border w-fit max-w-full mx-auto">
|
||||
<video
|
||||
ref={videoRef}
|
||||
controls
|
||||
playsInline
|
||||
src={meta.file_url}
|
||||
poster={meta.poster_url || undefined}
|
||||
className="max-h-[80vh] max-w-full mx-auto block"
|
||||
/>
|
||||
</div>
|
||||
{meta.audio ? (
|
||||
/* Audio-only asset (mp3): WebKit won't play it in a <video>, so render a native
|
||||
<audio>. No poster exists for these, so present it as a deliberate padded bar on
|
||||
the surface (not the black video box) so it looks intentional without artwork. */
|
||||
<div className="rounded-xl border border-border bg-surface p-4">
|
||||
<audio ref={setMedia} controls src={meta.file_url} className="w-full block" />
|
||||
</div>
|
||||
) : (
|
||||
/* Shrink-wrap the player to the video so a vertical/short clip renders as a narrow
|
||||
player instead of a wide box with black bars; a landscape clip fills the width. */
|
||||
<div className="rounded-xl overflow-hidden bg-black border border-border w-fit max-w-full mx-auto">
|
||||
<video
|
||||
ref={setMedia}
|
||||
controls
|
||||
playsInline
|
||||
src={meta.file_url}
|
||||
poster={meta.poster_url || undefined}
|
||||
className="max-h-[80vh] max-w-full mx-auto block"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
<div className="mt-3 flex items-start gap-3">
|
||||
<div className="min-w-0 flex-1">
|
||||
<h1 className="text-lg font-semibold leading-snug">{meta.title || "Video"}</h1>
|
||||
|
||||
@@ -8,6 +8,7 @@ import { useLiveQuery } from "../lib/useLiveQuery";
|
||||
import { api } from "../lib/api";
|
||||
import { formatBytes } from "../lib/format";
|
||||
import { inputCls } from "./ui/form";
|
||||
import { Button } from "./ui/Button";
|
||||
import { StatusPill, jobTitle } from "./downloadRow";
|
||||
|
||||
// The admin "System" tab — storage overview, per-user footprint, quota editor, and the global job
|
||||
@@ -110,19 +111,17 @@ function QuotaModal({
|
||||
{t("downloads.admin.unlimited")}
|
||||
</label>
|
||||
<div className="flex justify-between pt-1">
|
||||
<button
|
||||
onClick={() => reset.mutate()}
|
||||
className="px-3 py-1.5 rounded-lg text-sm text-muted hover:text-fg hover:bg-surface transition"
|
||||
>
|
||||
<Button variant="ghost" size="sm" onClick={() => reset.mutate()}>
|
||||
{t("downloads.admin.reset")}
|
||||
</button>
|
||||
<button
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
onClick={() => save.mutate()}
|
||||
disabled={save.isPending}
|
||||
className="px-3 py-1.5 rounded-lg text-sm font-medium btn-primary disabled:opacity-50 transition"
|
||||
>
|
||||
{t("downloads.admin.save")}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
|
||||
@@ -6,6 +6,7 @@ import { Plus, Trash2 } from "lucide-react";
|
||||
import Modal from "./Modal";
|
||||
import { api, type DownloadJob } from "../lib/api";
|
||||
import { inputCls } from "./ui/form";
|
||||
import { Button } from "./ui/Button";
|
||||
import { IconBtn } from "./downloadRow";
|
||||
|
||||
// Per-item user modals split out of DownloadCenter (self-contained, prop-driven): rename/metadata
|
||||
@@ -101,22 +102,26 @@ export function MetaEditModal({ job, onClose }: { job: DownloadJob; onClose: ()
|
||||
</div>
|
||||
<p className="text-xs text-muted">{t("downloads.edit.hint")}</p>
|
||||
<div className="flex justify-end">
|
||||
<button
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
onClick={() => save.mutate()}
|
||||
disabled={!name.trim() || save.isPending}
|
||||
className="px-3 py-1.5 rounded-lg text-sm font-medium btn-primary disabled:opacity-50 transition"
|
||||
>
|
||||
{t("downloads.edit.save")}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
// Transient failures worth an in-modal Retry button; the rest are permanent (private/geo/DRM/…),
|
||||
// where a retry can't help — the queue row still keeps its own retry control regardless.
|
||||
const RETRYABLE_ERRORS = ["postprocess", "network", "unknown"];
|
||||
// Transient failures worth an in-modal Retry button; the rest are permanent (private/geo/DRM/
|
||||
// sign-in/unsupported-host/…), where a retry can't help — the queue row still keeps its own retry
|
||||
// control regardless. `unknown` is deliberately NOT here: an unclassified failure is most often an
|
||||
// auth wall we couldn't parse (a login-walled Facebook post reads as a bare "no format"), so a
|
||||
// prominent Retry would mislead; force-retrying from the row stays available for genuine transients.
|
||||
const RETRYABLE_ERRORS = ["postprocess", "network"];
|
||||
|
||||
// A readable "why did this fail" window for a failed job: a category headline + plain explanation,
|
||||
// the raw yt-dlp/ffmpeg text behind a disclosure, and a Retry when the failure may be transient.
|
||||
@@ -155,22 +160,20 @@ export function ErrorModal({
|
||||
)}
|
||||
<div className="flex justify-end gap-2 pt-1">
|
||||
{retryable && (
|
||||
<button
|
||||
<Button
|
||||
variant="primary"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
onRetry();
|
||||
onClose();
|
||||
}}
|
||||
className="px-3 py-1.5 rounded-lg text-sm font-medium btn-primary transition"
|
||||
>
|
||||
{t("downloads.actions.retry")}
|
||||
</button>
|
||||
</Button>
|
||||
)}
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="px-3 py-1.5 rounded-lg text-sm text-muted hover:text-fg hover:bg-surface transition"
|
||||
>
|
||||
<Button variant="ghost" size="sm" onClick={onClose}>
|
||||
{t("common.close")}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
|
||||
@@ -278,8 +278,12 @@ export function UsageBar() {
|
||||
const { t } = useTranslation();
|
||||
// Poll live (like the library list) so the footprint updates the moment the worker finishes an
|
||||
// edit/download — enqueue only creates a queued job (0 bytes), the size lands asynchronously, so
|
||||
// a one-shot fetch at enqueue time would show a stale 0 B until a manual reload.
|
||||
const usage = useLiveQuery(qk.downloadUsage(), api.downloadUsage, { intervalMs: 2000 });
|
||||
// a one-shot fetch at enqueue time would show a stale 0 B until a manual reload. The footprint
|
||||
// only moves while a job is in flight, so poll fast (2s) then and idle-back-off to 15s otherwise
|
||||
// (running_jobs counts queued+running — it flips back to 2s on the first poll after a new enqueue).
|
||||
const usage = useLiveQuery(qk.downloadUsage(), api.downloadUsage, {
|
||||
intervalMs: (u) => (u?.running_jobs ? 2000 : 15000),
|
||||
});
|
||||
const u = usage.data;
|
||||
if (!u) return null;
|
||||
const pct =
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// S5: the shared <Button> primitive. These lock the load-bearing contract the 120+ migrated call
|
||||
// sites rely on: variant/size paint, icon-only accessibility (name must survive when the text is
|
||||
// dropped), the loading state (disabled + spinner + aria-busy), and the type="button" default that
|
||||
// stops an action button from submitting a surrounding form.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { render, screen, cleanup, fireEvent } from "@testing-library/react";
|
||||
import { Button } from "./Button";
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
describe("Button primitive (S5)", () => {
|
||||
it("defaults to type=button so it never submits a form", () => {
|
||||
render(<Button>Go</Button>);
|
||||
expect(screen.getByRole("button", { name: "Go" }).getAttribute("type")).toBe("button");
|
||||
});
|
||||
|
||||
it("applies the variant paint recipe", () => {
|
||||
render(
|
||||
<Button variant="danger" size="sm">
|
||||
Delete
|
||||
</Button>,
|
||||
);
|
||||
const btn = screen.getByRole("button", { name: "Delete" });
|
||||
expect(btn.className).toContain("btn-danger");
|
||||
expect(btn.className).toContain("py-1.5");
|
||||
});
|
||||
|
||||
it("exposes an accessible name for an icon-only button while dropping the text", () => {
|
||||
render(
|
||||
<Button iconOnly icon={<svg data-testid="ic" />} label="Block channel">
|
||||
this text is dropped
|
||||
</Button>,
|
||||
);
|
||||
const btn = screen.getByRole("button", { name: "Block channel" });
|
||||
expect(btn.getAttribute("title")).toBe("Block channel");
|
||||
expect(btn.textContent).not.toContain("this text is dropped");
|
||||
expect(btn.querySelector("[data-testid=ic]")).not.toBeNull();
|
||||
});
|
||||
|
||||
it("loading disables the button, marks aria-busy, and blocks clicks", () => {
|
||||
const onClick = vi.fn();
|
||||
render(
|
||||
<Button loading icon={<svg data-testid="ic" />} onClick={onClick}>
|
||||
Save
|
||||
</Button>,
|
||||
);
|
||||
const btn = screen.getByRole("button", { name: "Save" }) as HTMLButtonElement;
|
||||
expect(btn.disabled).toBe(true);
|
||||
expect(btn.getAttribute("aria-busy")).toBe("true");
|
||||
// the leading icon is swapped for a spinner
|
||||
expect(btn.querySelector("[data-testid=ic]")).toBeNull();
|
||||
fireEvent.click(btn);
|
||||
expect(onClick).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("loading disables even when an explicit disabled={false} is passed", () => {
|
||||
// Regression: `disabled ?? loading` let an explicit boolean disabled shadow loading, so a
|
||||
// call-site with `disabled={!valid} loading={pending}` (valid → disabled=false) stayed
|
||||
// clickable mid-mutation and double-submitted. The contract is: loading always disables.
|
||||
const onClick = vi.fn();
|
||||
render(
|
||||
<Button disabled={false} loading onClick={onClick}>
|
||||
Create
|
||||
</Button>,
|
||||
);
|
||||
const btn = screen.getByRole("button", { name: "Create" }) as HTMLButtonElement;
|
||||
expect(btn.disabled).toBe(true);
|
||||
fireEvent.click(btn);
|
||||
expect(onClick).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fullWidth adds the w-full utility", () => {
|
||||
render(<Button fullWidth>Wide</Button>);
|
||||
expect(screen.getByRole("button", { name: "Wide" }).className).toContain("w-full");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
import { forwardRef } from "react";
|
||||
import type { ButtonHTMLAttributes, ReactNode } from "react";
|
||||
import clsx from "clsx";
|
||||
import { Loader2 } from "lucide-react";
|
||||
|
||||
// The ONE action-button primitive. Before this there were 385 raw <button>s styling themselves with
|
||||
// ad-hoc utility strings over the CSS paint-recipes (.btn-primary/.btn-danger, index.css) + the btnCls
|
||||
// constant. This centralises the geometry+paint so an "action button" (submit/cancel/subscribe/delete…)
|
||||
// is one component with named variants. It deliberately does NOT cover the app's OTHER button families,
|
||||
// which keep their own primitives: aria-pressed segmented pills (SegmentedToggle/SortControl), the
|
||||
// Switch toggle, .chip removers, dropdown/popover trigger pills, and slider/overlay/card-click buttons.
|
||||
//
|
||||
// No focus-visible here on purpose: @layer base in index.css already rings every <button> for keyboard
|
||||
// users. Adding one would double the ring.
|
||||
|
||||
type ButtonVariant =
|
||||
| "primary" // solid accent — the main affirmative action
|
||||
| "danger" // solid danger — destructive confirm
|
||||
| "outline" // bordered, quiet — secondary action (Full history, Subscribe, Cancel-with-border)
|
||||
| "ghost" // borderless, hover-fill — tertiary/dismiss (dialog Cancel, close)
|
||||
| "glass" // glass card — the Plex/overlay action buttons
|
||||
| "danger-outline"; // bordered danger — a destructive action that shouldn't shout
|
||||
|
||||
type ButtonSize =
|
||||
| "xs" // dense inline action
|
||||
| "sm" // the btnCls geometry (px-3 py-1.5)
|
||||
| "md" // default action button (px-3 py-2)
|
||||
| "lg" // prominent CTA
|
||||
| "icon"; // square icon-only (w-9 h-9)
|
||||
|
||||
const VARIANTS: Record<ButtonVariant, string> = {
|
||||
// .btn-primary / .btn-danger own the fill + one hover-dim (index.css @layer components).
|
||||
primary: "btn-primary",
|
||||
danger: "btn-danger",
|
||||
outline: "border border-border text-muted hover:text-fg hover:border-fg/40",
|
||||
ghost: "text-muted hover:text-fg hover:bg-surface",
|
||||
glass: "glass-card glass-hover hover:text-accent",
|
||||
"danger-outline": "border border-danger/60 text-danger hover:bg-danger/10 hover:border-danger",
|
||||
};
|
||||
|
||||
const SIZES: Record<ButtonSize, string> = {
|
||||
xs: "px-2 py-1 text-xs rounded-md gap-1",
|
||||
sm: "px-3 py-1.5 text-sm rounded-lg gap-1.5",
|
||||
md: "px-3 py-2 text-sm rounded-lg gap-1.5",
|
||||
lg: "px-4 py-2.5 text-base rounded-lg gap-2",
|
||||
icon: "w-9 h-9 rounded-lg justify-center",
|
||||
};
|
||||
|
||||
const BASE =
|
||||
"inline-flex items-center justify-center font-medium transition disabled:opacity-50 disabled:cursor-not-allowed";
|
||||
|
||||
export interface ButtonProps extends ButtonHTMLAttributes<HTMLButtonElement> {
|
||||
variant?: ButtonVariant;
|
||||
size?: ButtonSize;
|
||||
/** Leading icon (an lucide element). Replaced by a spinner while `loading`. */
|
||||
icon?: ReactNode;
|
||||
/** Render only the icon: children are dropped and `label` supplies the accessible name. */
|
||||
iconOnly?: boolean;
|
||||
/** Accessible name + tooltip — required for `iconOnly`, optional otherwise. */
|
||||
label?: string;
|
||||
/** Disable + swap the icon for a spinner + mark aria-busy. */
|
||||
loading?: boolean;
|
||||
fullWidth?: boolean;
|
||||
}
|
||||
|
||||
/** The shared action button. `type` defaults to "button" so it never accidentally submits a form. */
|
||||
export const Button = forwardRef<HTMLButtonElement, ButtonProps>(function Button(
|
||||
{
|
||||
variant = "primary",
|
||||
size = "md",
|
||||
icon,
|
||||
iconOnly = false,
|
||||
label,
|
||||
loading = false,
|
||||
fullWidth = false,
|
||||
className,
|
||||
children,
|
||||
disabled,
|
||||
type,
|
||||
...rest
|
||||
},
|
||||
ref,
|
||||
) {
|
||||
const leading = loading ? <Loader2 className="w-4 h-4 animate-spin" /> : icon;
|
||||
return (
|
||||
<button
|
||||
ref={ref}
|
||||
type={type ?? "button"}
|
||||
// `loading` must always disable (see the prop's contract above), even when a call-site also
|
||||
// passes an explicit boolean `disabled` — so `||`, not `??`. With `??`, `disabled={false}`
|
||||
// would shadow `loading` and leave the button clickable mid-mutation (double-submit).
|
||||
disabled={disabled || loading}
|
||||
aria-busy={loading || undefined}
|
||||
aria-label={label}
|
||||
title={label}
|
||||
className={clsx(BASE, VARIANTS[variant], SIZES[size], fullWidth && "w-full", className)}
|
||||
{...rest}
|
||||
>
|
||||
{leading}
|
||||
{!iconOnly && children}
|
||||
</button>
|
||||
);
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { createPortal } from "react-dom";
|
||||
import { Check, RotateCcw, Save } from "lucide-react";
|
||||
import { Button } from "./Button";
|
||||
|
||||
// The Save / Discard bar shown while an edited draft has unsaved changes (Settings prefs,
|
||||
// admin Configuration). Two visual variants share one state machine + markup:
|
||||
@@ -58,24 +59,27 @@ export function DraftSaveBar({
|
||||
)}
|
||||
</span>
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
<Button
|
||||
data-testid="draft-discard"
|
||||
variant="glass"
|
||||
size="sm"
|
||||
icon={<RotateCcw className="w-4 h-4" />}
|
||||
onClick={onDiscard}
|
||||
disabled={saving || !dirty}
|
||||
className="glass-card glass-hover flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-sm disabled:opacity-40 transition"
|
||||
>
|
||||
<RotateCcw className="w-4 h-4" />
|
||||
{labels.discard}
|
||||
</button>
|
||||
<button
|
||||
</Button>
|
||||
<Button
|
||||
data-testid="draft-save"
|
||||
variant="primary"
|
||||
size="sm"
|
||||
icon={<Save className="w-4 h-4" />}
|
||||
onClick={onSave}
|
||||
disabled={saving || !dirty}
|
||||
className="flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-sm btn-primary font-medium shadow-md shadow-accent/20 disabled:opacity-40 transition"
|
||||
className="shadow-md shadow-accent/20"
|
||||
>
|
||||
<Save className="w-4 h-4" />
|
||||
{saving ? labels.saving : labels.save}
|
||||
</button>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -17,5 +17,7 @@
|
||||
"sync_pause": "Sync paused",
|
||||
"sync_resume": "Sync resumed",
|
||||
"job_run": "Scheduler job run",
|
||||
"audit_clear": "Audit log cleared"
|
||||
"audit_clear": "Audit log cleared",
|
||||
"download_credential_set": "Download cookies set",
|
||||
"download_credential_delete": "Download cookies removed"
|
||||
}
|
||||
|
||||
@@ -218,7 +218,11 @@
|
||||
},
|
||||
"login_required": {
|
||||
"short": "Sign-in required",
|
||||
"body": "This video needs a signed-in, age-verified, or members-only account to view, so it can't be downloaded here."
|
||||
"body": "This video needs a signed-in account (age-restricted, members-only, or a private group) to view. Add your cookies for this site under Settings → Account → Download authentication to download it."
|
||||
},
|
||||
"unsupported_host": {
|
||||
"short": "Site not supported",
|
||||
"body": "This site isn't supported for downloading, so nothing could be fetched from this link."
|
||||
},
|
||||
"geo_blocked": {
|
||||
"short": "Region-blocked",
|
||||
@@ -249,6 +253,11 @@
|
||||
"body": "Something went wrong while downloading this item. The technical details below may help."
|
||||
}
|
||||
}
|
||||
},
|
||||
"cookies": {
|
||||
"empty": "Paste your cookies first.",
|
||||
"format": "That doesn't look like a cookies.txt file (Netscape format).",
|
||||
"generic": "Those cookies couldn't be saved."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,8 +35,21 @@
|
||||
"pushTitle": "Sync to YouTube",
|
||||
"pushConfirm": "Sync",
|
||||
"pushPlanCreate": "Create a new YouTube playlist with {{count}} videos? (~{{units}} quota units; {{left}} left today.)",
|
||||
"pushPlanUpdate": "Update on YouTube — add {{insert}}, remove {{del}}, reorder {{reorder}}? (~{{units}} quota units; {{left}} left today.)",
|
||||
"pushDiverged": "{{count}} video(s) currently on YouTube will be removed.",
|
||||
"mergeAddLocal_one": "{{count}} video added to Siftlode from YouTube.",
|
||||
"mergeAddLocal_other": "{{count}} videos added to Siftlode from YouTube.",
|
||||
"mergeRemoveLocal_one": "{{count}} video removed from Siftlode.",
|
||||
"mergeRemoveLocal_other": "{{count}} videos removed from Siftlode.",
|
||||
"mergeAddYt_one": "{{count}} video added to YouTube.",
|
||||
"mergeAddYt_other": "{{count}} videos added to YouTube.",
|
||||
"mergeRemoveYt_one": "{{count}} video removed from YouTube.",
|
||||
"mergeRemoveYt_other": "{{count}} videos removed from YouTube.",
|
||||
"mergeReorder_one": "{{count}} video reordered on YouTube.",
|
||||
"mergeReorder_other": "{{count}} videos reordered on YouTube.",
|
||||
"mergeRename": "The playlist will be renamed on YouTube.",
|
||||
"mergeYtUnchanged": "Your YouTube playlist stays unchanged.",
|
||||
"mergeLocalUnchanged": "Your Siftlode playlist stays unchanged.",
|
||||
"mergeQuota": " (~{{units}} quota units; {{left}} left today.)",
|
||||
"mergeQuotaFree": " (~0 quota units.)",
|
||||
"pushNoQuota": "Not enough YouTube quota left today ({{left}}) for this sync (~{{units}} needed). Try again tomorrow.",
|
||||
"pushUpToDate": "“{{name}}” is already in sync with YouTube.",
|
||||
"pushDone": "“{{name}}” synced to YouTube ✓",
|
||||
|
||||
@@ -104,6 +104,27 @@
|
||||
"failed": "Couldn't sign out other sessions. Please try again."
|
||||
}
|
||||
},
|
||||
"downloadAuth": {
|
||||
"title": "Download authentication",
|
||||
"intro": "Some sites (e.g. a private Facebook group you belong to) only let logged-in members view a video. Paste your browser cookies for a site here and your downloads from it will use your login. Cookies are encrypted, kept private to your account, and only ever sent to the matching site.",
|
||||
"configured": "Set · updated {{when}}",
|
||||
"notSet": "Not set",
|
||||
"set": "Set cookies",
|
||||
"update": "Update",
|
||||
"remove": "Remove",
|
||||
"save": "Save",
|
||||
"cancel": "Cancel",
|
||||
"saving": "Saving…",
|
||||
"placeholder": "Paste the contents of your cookies.txt (Netscape format) here",
|
||||
"help": "Export a cookies.txt for this site with a browser extension, then paste it here. It's write-only — we never show it back.",
|
||||
"saved": "Cookies saved.",
|
||||
"removed": "Cookies removed.",
|
||||
"failed": "Couldn't save those cookies.",
|
||||
"removeConfirmTitle": "Remove cookies?",
|
||||
"removeConfirmBody": "Downloads from this site will stop using your login.",
|
||||
"removeConfirmAction": "Remove",
|
||||
"loadError": "Couldn't load your download authentication settings."
|
||||
},
|
||||
"demo": {
|
||||
"title": "Demo access",
|
||||
"intro": "Emails here can enter the shared demo account straight from the login page — no Google sign-in. They just type the address into the field and are let in after a moment.",
|
||||
|
||||
@@ -17,5 +17,7 @@
|
||||
"sync_pause": "Szinkron szüneteltetve",
|
||||
"sync_resume": "Szinkron folytatva",
|
||||
"job_run": "Ütemező job lefutott",
|
||||
"audit_clear": "Napló ürítve"
|
||||
"audit_clear": "Napló ürítve",
|
||||
"download_credential_set": "Letöltési cookie-k beállítva",
|
||||
"download_credential_delete": "Letöltési cookie-k eltávolítva"
|
||||
}
|
||||
|
||||
@@ -218,7 +218,11 @@
|
||||
},
|
||||
"login_required": {
|
||||
"short": "Bejelentkezés szükséges",
|
||||
"body": "Ehhez a videóhoz bejelentkezett, kor-ellenőrzött vagy tagsági fiók kell, ezért itt nem tölthető le."
|
||||
"body": "Ehhez a videóhoz bejelentkezett fiók kell (korhatáros, csak tagoknak szóló vagy zárt csoport). Töltsd fel a cookie-kat ehhez az oldalhoz a Beállítások → Fiók → Letöltés-hitelesítés alatt a letöltéshez."
|
||||
},
|
||||
"unsupported_host": {
|
||||
"short": "Nem támogatott oldal",
|
||||
"body": "Ez az oldal nem támogatott a letöltéshez, ezért erről a linkről semmit sem sikerült lekérni."
|
||||
},
|
||||
"geo_blocked": {
|
||||
"short": "Régió szerint tiltva",
|
||||
@@ -249,6 +253,11 @@
|
||||
"body": "Valami hiba történt az elem letöltése közben. Az alábbi technikai részletek segíthetnek."
|
||||
}
|
||||
}
|
||||
},
|
||||
"cookies": {
|
||||
"empty": "Előbb illeszd be a cookie-kat.",
|
||||
"format": "Ez nem tűnik cookies.txt fájlnak (Netscape formátum).",
|
||||
"generic": "A cookie-kat nem sikerült menteni."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,8 +35,21 @@
|
||||
"pushTitle": "Szinkron YouTube-ra",
|
||||
"pushConfirm": "Szinkron",
|
||||
"pushPlanCreate": "Létrehozol egy új YouTube-listát {{count}} videóval? (~{{units}} kvótaegység; ma még {{left}} maradt.)",
|
||||
"pushPlanUpdate": "Frissítés a YouTube-on — {{insert}} hozzáadása, {{del}} eltávolítása, {{reorder}} átrendezése? (~{{units}} kvótaegység; ma még {{left}} maradt.)",
|
||||
"pushDiverged": "{{count}} videó, amely jelenleg a YouTube-on van, el lesz távolítva.",
|
||||
"mergeAddLocal_one": "{{count}} videó bekerül a Siftlode-listádba a YouTube-ról.",
|
||||
"mergeAddLocal_other": "{{count}} videó bekerül a Siftlode-listádba a YouTube-ról.",
|
||||
"mergeRemoveLocal_one": "{{count}} videó törlődik a Siftlode-listádból.",
|
||||
"mergeRemoveLocal_other": "{{count}} videó törlődik a Siftlode-listádból.",
|
||||
"mergeAddYt_one": "{{count}} videó felkerül a YouTube-ra.",
|
||||
"mergeAddYt_other": "{{count}} videó felkerül a YouTube-ra.",
|
||||
"mergeRemoveYt_one": "{{count}} videó törlődik a YouTube-ról.",
|
||||
"mergeRemoveYt_other": "{{count}} videó törlődik a YouTube-ról.",
|
||||
"mergeReorder_one": "{{count}} videó átrendeződik a YouTube-on.",
|
||||
"mergeReorder_other": "{{count}} videó átrendeződik a YouTube-on.",
|
||||
"mergeRename": "A lista átnevezésre kerül a YouTube-on.",
|
||||
"mergeYtUnchanged": "A YouTube-listád nem változik.",
|
||||
"mergeLocalUnchanged": "A Siftlode-listád nem változik.",
|
||||
"mergeQuota": " (~{{units}} kvótaegység; ma még {{left}} maradt.)",
|
||||
"mergeQuotaFree": " (~0 kvótaegység.)",
|
||||
"pushNoQuota": "Nincs elég YouTube-kvóta mára ({{left}}) ehhez a szinkronhoz (~{{units}} kell). Próbáld újra holnap.",
|
||||
"pushUpToDate": "A(z) „{{name}}” már szinkronban van a YouTube-bal.",
|
||||
"pushDone": "„{{name}}” szinkronizálva a YouTube-ra ✓",
|
||||
|
||||
@@ -104,6 +104,27 @@
|
||||
"failed": "Nem sikerült kijelentkeztetni a többi munkamenetet. Próbáld újra."
|
||||
}
|
||||
},
|
||||
"downloadAuth": {
|
||||
"title": "Letöltés-hitelesítés",
|
||||
"intro": "Egyes oldalak (pl. egy zárt Facebook-csoport, aminek tagja vagy) csak bejelentkezett tagoknak mutatják a videót. Illeszd be ide a böngésződ cookie-jait egy oldalhoz, és az onnan induló letöltések a te bejelentkezésedet használják. A cookie-k titkosítva, csak a te fiókodhoz kötve tárolódnak, és kizárólag az adott oldalnak küldjük el őket.",
|
||||
"configured": "Beállítva · frissítve: {{when}}",
|
||||
"notSet": "Nincs beállítva",
|
||||
"set": "Cookie-k beállítása",
|
||||
"update": "Frissítés",
|
||||
"remove": "Eltávolítás",
|
||||
"save": "Mentés",
|
||||
"cancel": "Mégse",
|
||||
"saving": "Mentés…",
|
||||
"placeholder": "Illeszd be ide a cookies.txt tartalmát (Netscape formátum)",
|
||||
"help": "Exportálj cookies.txt-t ehhez az oldalhoz egy böngészőbővítménnyel, majd illeszd be ide. Csak írható — sosem mutatjuk vissza.",
|
||||
"saved": "Cookie-k mentve.",
|
||||
"removed": "Cookie-k eltávolítva.",
|
||||
"failed": "A cookie-kat nem sikerült menteni.",
|
||||
"removeConfirmTitle": "Cookie-k eltávolítása?",
|
||||
"removeConfirmBody": "Az erről az oldalról induló letöltések nem használják tovább a bejelentkezésedet.",
|
||||
"removeConfirmAction": "Eltávolítás",
|
||||
"loadError": "Nem sikerült betölteni a letöltési hitelesítési beállításokat."
|
||||
},
|
||||
"demo": {
|
||||
"title": "Demo hozzáférés",
|
||||
"intro": "Az itt szereplő e-mailek a bejelentkező oldalról közvetlenül beléphetnek a közös demo fiókba — Google-bejelentkezés nélkül. Csak beírják a címet a mezőbe, és pár pillanat múlva bekerülnek.",
|
||||
|
||||
@@ -43,6 +43,8 @@ function localizeDetail(d: unknown): string {
|
||||
}
|
||||
if (o?.code === "edit")
|
||||
return t(`downloads.errors.edit.${o.reason}`, t("downloads.errors.edit.generic"));
|
||||
if (o?.code === "cookies")
|
||||
return t(`downloads.errors.cookies.${o.reason}`, t("downloads.errors.cookies.generic"));
|
||||
return t("errors.generic");
|
||||
}
|
||||
|
||||
|
||||
@@ -137,7 +137,31 @@ export interface AdminDownloadQuota {
|
||||
unlimited: boolean;
|
||||
}
|
||||
|
||||
// Per-user download authentication (cookies). Write-only: the API never returns the stored blob,
|
||||
// only which supported hosts are configured + when. See backend downloads.credentials.
|
||||
export interface DownloadCredentialHost {
|
||||
host_key: string;
|
||||
label: string;
|
||||
configured: boolean;
|
||||
updated_at: string | null;
|
||||
}
|
||||
|
||||
export const downloadsApi = {
|
||||
downloadCredentials: (): Promise<{ hosts: DownloadCredentialHost[] }> =>
|
||||
req("/api/downloads/credentials"),
|
||||
setDownloadCredential: (
|
||||
hostKey: string,
|
||||
cookies: string,
|
||||
): Promise<{ ok: true; hosts: DownloadCredentialHost[] }> =>
|
||||
req(`/api/downloads/credentials/${encodeURIComponent(hostKey)}`, {
|
||||
method: "PUT",
|
||||
body: JSON.stringify({ cookies }),
|
||||
}),
|
||||
deleteDownloadCredential: (
|
||||
hostKey: string,
|
||||
): Promise<{ ok: true; hosts: DownloadCredentialHost[] }> =>
|
||||
req(`/api/downloads/credentials/${encodeURIComponent(hostKey)}`, { method: "DELETE" }),
|
||||
|
||||
downloadProfiles: (): Promise<DownloadProfile[]> => req("/api/downloads/profiles"),
|
||||
createDownloadProfile: (p: { name: string; spec: DownloadSpec }): Promise<DownloadProfile> =>
|
||||
req("/api/downloads/profiles", { method: "POST", body: JSON.stringify(p) }),
|
||||
|
||||
@@ -31,12 +31,16 @@ export interface PlaylistDetail {
|
||||
items: Video[];
|
||||
}
|
||||
|
||||
// Two-way sync plan: what each side gains/loses under the 3-way merge (neither side wins
|
||||
// wholesale). Local edits are free; only the YouTube writes cost quota.
|
||||
export interface PushPlan {
|
||||
action: "create" | "update";
|
||||
to_insert: number;
|
||||
to_delete: number;
|
||||
add_yt: number; // local-adds to push out to YouTube
|
||||
add_local: number; // YouTube-adds to pull into Siftlode
|
||||
remove_yt: number; // genuine local deletes propagated to YouTube
|
||||
remove_local: number; // genuine YouTube deletes propagated to Siftlode
|
||||
to_reorder: number;
|
||||
yt_extra: number; // items on YouTube that the push would remove (divergence)
|
||||
rename: boolean;
|
||||
units_estimate: number;
|
||||
remaining_today: number;
|
||||
affordable: boolean;
|
||||
@@ -44,9 +48,11 @@ export interface PushPlan {
|
||||
|
||||
export interface PushResult {
|
||||
created: number;
|
||||
inserted: number;
|
||||
deleted: number;
|
||||
inserted: number; // added on YouTube
|
||||
deleted: number; // removed on YouTube
|
||||
reordered: number;
|
||||
pulled: number; // added into Siftlode from YouTube
|
||||
removed_local: number; // removed from Siftlode (deleted on YouTube)
|
||||
failures: string[];
|
||||
yt_playlist_id: string | null;
|
||||
}
|
||||
|
||||
@@ -78,6 +78,7 @@ export const qk = {
|
||||
downloadIndex: () => ["download-index"] as const,
|
||||
downloadUsage: () => ["download-usage"] as const,
|
||||
downloadProfiles: () => ["download-profiles"] as const,
|
||||
downloadCredentials: () => ["download-credentials"] as const,
|
||||
downloadLinks: (jobId: number) => ["download-links", jobId] as const,
|
||||
dlQuota: (userId: number) => ["dl-quota", userId] as const,
|
||||
storyboard: (jobId: number) => ["storyboard", jobId] as const,
|
||||
|
||||
@@ -14,6 +14,26 @@ export interface ReleaseEntry {
|
||||
}
|
||||
|
||||
export const RELEASE_NOTES: ReleaseEntry[] = [
|
||||
{
|
||||
version: "0.75.0",
|
||||
date: "2026-08-29",
|
||||
summary:
|
||||
"A grab-bag batch across Plex, downloads and playlists: your watch state stays truer to Plex, shared audio plays on iPad, downloads from a site you're logged into work with your own cookies, and syncing a playlist to YouTube is now a true two-way merge with a preview.",
|
||||
features: [
|
||||
"Syncing a playlist to YouTube is now a true two-way merge: a video you added on YouTube and one you added in Siftlode both survive, genuine removals carry across, and you always see a preview of exactly what will change on each side before anything is written.",
|
||||
"Download from a site that only shows a video to logged-in members (like a private Facebook group you belong to): paste your own browser cookies for that site under Settings → Account, and downloads from it use your login. Cookies are encrypted, kept private to your account, and only ever sent to the matching site.",
|
||||
"Shared audio files (mp3) now play on iPhone and iPad, in a proper audio player instead of a broken video box.",
|
||||
],
|
||||
fixes: [
|
||||
'A movie you finish on Plex no longer lingers as "in progress" in Siftlode until the nightly reconcile — scrubbing through the end credits now marks it watched within the next sync.',
|
||||
"A download that fails because the site needs a login you haven't set up now says so clearly and doesn't offer a pointless Retry; a genuinely temporary failure still offers one.",
|
||||
"The Plex sort control's direction toggle no longer wraps onto two lines on the narrower popover, so it matches the feed's.",
|
||||
"Adding a download no longer makes the app feel sluggish while it's converted in the background — the conversion is de-prioritised so the rest of the app and your share links stay responsive.",
|
||||
],
|
||||
chores: [
|
||||
"Introduced one shared action-button primitive across the app (the channel Subscribe button now matches Full history), and put the whole batch through several multi-agent deep-review rounds — findings fixed and re-reviewed each round until it came back clean, including data-integrity fixes to the new playlist merge.",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: "0.74.0",
|
||||
date: "2026-08-28",
|
||||
|
||||
Reference in New Issue
Block a user