diff --git a/.env.example b/.env.example index fd17c2d5..376f9315 100644 --- a/.env.example +++ b/.env.example @@ -73,13 +73,13 @@ SMTP_FROM= # of them and false on the rest, to avoid double quota use and write races. SCHEDULER_ENABLED=true -# --- Download center --- -# The Download Center adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small +# --- Files library & downloads --- +# Downloading adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small # `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose. -# Downloaded media defaults to a Docker-managed named volume. Set DOWNLOAD_HOST_PATH to a host +# The Files library defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host # directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there. # The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 . -# DOWNLOAD_HOST_PATH=/mnt/media/youtube +# FILES_HOST_PATH=/mnt/media/youtube # --- Plex integration (optional) --- # The optional Plex module browses/plays your Plex library, playing the LOCAL physical file @@ -89,6 +89,6 @@ SCHEDULER_ENABLED=true # to /plex-media inside the container (the local side of the admin "path map", whose default is # /data=/plex-media). Leave unset if you don't use Plex. # PLEX_MEDIA_HOST_PATH=/mnt/media -# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to DOWNLOAD_ROOT; point it at +# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to FILES_ROOT; point it at # fast local storage if your download volume is slow. # PLEX_HLS_DIR=/var/tmp/plex-hls diff --git a/.gitignore b/.gitignore index 04718ff8..af6667dc 100644 --- a/.gitignore +++ b/.gitignore @@ -29,9 +29,9 @@ frontend/e2e/.auth/ *.sql.gz backups/ -# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount at the repo root). +# Download center: locally downloaded media (the FILES_ROOT bind mount at the repo root). # Anchored with a leading slash so it does NOT also ignore the backend/app/downloads package. -/downloads/ +/files/ # OS / editor .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a431d4b..03a2cda4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.106.0 — 2026-10-13 + +### New + +- Files: one library for everything you keep — downloads, clips and your own uploads of any file — in a folder per site plus folders of your own, each keeping its files as long as you choose. Documents, pictures and text open in a viewer. +- Downloads: paste a link anywhere on a page to download it, and the tray shows what is running and what is done. The Downloads page is gone: your files are in Files, and your download settings are under Settings › Downloads. + +### Fixed + +- Self-hosting: the library is now mounted at /files (FILES_HOST_PATH). Old settings keep working; if you used the default volume, copy it over once — see the upgrade notes. + ## 0.105.0 — 2026-10-10 ### New diff --git a/CLAUDE.md b/CLAUDE.md index ae8fa045..347e1f3a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ be developed that way. This file is the rules of the road: read it before you to loaded into every agent session, so keep it **short and true**. A rule here that has gone stale is worse than no rule — if you change how the code works, change this file in the same commit. -Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a download center). It is a +Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a files library with downloads). It is a **modular monolith**: one Postgres, one FastAPI process (uvicorn + APScheduler), one worker process (yt-dlp/ffmpeg), one PO-token sidecar. React 19 (with the React Compiler) + react-query frontend, with a router-free navigation provider. diff --git a/Dockerfile b/Dockerfile index 2a31b361..b04e23be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,13 +39,13 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app # Create the app user BEFORE the copies so files land already-owned via COPY --chown (a trailing -# `chown -R /app` would rewrite every file into a duplicate layer). /downloads must be appuser-owned +# `chown -R /app` would rewrite every file into a duplicate layer). /files must be appuser-owned # so a fresh named volume mounted there inherits that ownership (Docker copies the image dir's # ownership into a new empty volume) — the worker/API can then write without a manual chown. A bind # mount to a host path instead needs that path writable by this uid (see docs/self-hosting.md). RUN adduser --disabled-password --gecos "" appuser \ - && mkdir -p /downloads \ - && chown appuser /app /downloads + && mkdir -p /files \ + && chown appuser /app /files # ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode. # intel-media-va-driver (iHD): the VA driver ffmpeg's h264_vaapi needs — libva2 ships in the base but diff --git a/README.md b/README.md index 80ab33b7..b5424876 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,11 @@ blocker and SponsorBlock keep working. tags to slice the feed by. - **Playlists with two-way YouTube sync** — build them locally, keep them in sync in both directions. - **In-app player** with resume, plus keyboard/scroll controls. -- **Download Center** — save videos to the server with yt-dlp in a Plex-friendly layout (format - presets, per-user storage quota), trim / crop / split & join them in a built-in editor, play them +- **Files** — one library for everything you keep: paste a link anywhere in the app to download it + with yt-dlp (format presets, per-user storage quota, the running downloads in a tray), upload your + own files, sort them into folders, trim / crop / split & join videos in a built-in editor, play them where you left off, then save to your device, share with another user, or hand out a public watch - link. + link that plays on any phone. - **Plex library** *(optional)* — point Siftlode at your own Plex server to browse and play its movies and shows alongside your feed, with watch state kept in sync. - **Multi-user** with per-user private state, a shared catalog, and a fair daily API-quota guard. diff --git a/VERSION b/VERSION index f1b9cc4c..75602ab1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.105.0 +0.106.0 diff --git a/backend/.basedpyright/baseline.json b/backend/.basedpyright/baseline.json index cc5f74f1..824be795 100644 --- a/backend/.basedpyright/baseline.json +++ b/backend/.basedpyright/baseline.json @@ -727,30 +727,6 @@ } ], "./app/routes/downloads.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 29, - "endColumn": 33, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 29, - "endColumn": 33, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 40, - "endColumn": 50, - "lineCount": 1 - } - }, { "code": "reportOptionalMemberAccess", "range": { @@ -790,22 +766,6 @@ "endColumn": 54, "lineCount": 1 } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 29, - "endColumn": 33, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 40, - "endColumn": 50, - "lineCount": 1 - } } ], "./app/routes/playlists.py": [ @@ -1196,16 +1156,6 @@ } } ], - "./app/routes/public.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 57, - "endColumn": 71, - "lineCount": 1 - } - } - ], "./app/routes/scheduler.py": [ { "code": "reportArgumentType", diff --git a/backend/alembic/versions/0050_asset_poster.py b/backend/alembic/versions/0050_asset_poster.py index 7c210b7b..3059ea2b 100644 --- a/backend/alembic/versions/0050_asset_poster.py +++ b/backend/alembic/versions/0050_asset_poster.py @@ -6,7 +6,7 @@ Create Date: 2026-07-07 Some sources (a direct media URL like a reddit v.redd.it HLS manifest) carry no thumbnail, so the download shows a blank image box. The worker now extracts a representative frame with ffmpeg and -saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card / +saves it as a poster sidecar; `poster_path` (relative to files_root) records it so the card / watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail. """ diff --git a/backend/alembic/versions/0102_file_folders.py b/backend/alembic/versions/0102_file_folders.py new file mode 100644 index 00000000..30b965a6 --- /dev/null +++ b/backend/alembic/versions/0102_file_folders.py @@ -0,0 +1,193 @@ +"""The Files library: folders, and a file's retention on the user's job + +Revision ID: 0102_file_folders +Revises: 0101_plex_text_plain +Create Date: 2026-10-11 + +Every download, clip and upload of a user now sits in a folder of their Files library +(`app.downloads.folders`): a source folder made per origin (YouTube, Plex, uploads, one per other +site) or one of their own. How long a file stays moves from the shared asset to each user's job +(`download_jobs.expires_at`, stamped from the folder's `retention_days`), so one user keeping a file +no longer depends on another's copy; the asset's own `expires_at` / `gc_notified` go. + +This places every existing job in its source folder and carries each done job's expiry over from +the asset it holds (an upload keeps none). The source-key rule is copied here, frozen, from +`folders.source_key_for` as it is today — a migration must not follow the code's later changes. +""" + +from urllib.parse import urlsplit + +import sqlalchemy as sa +from alembic import op + +revision: str = "0102_file_folders" +down_revision: str | None = "0101_plex_text_plain" +branch_labels: str | None = None +depends_on: str | None = None + +_FRONT_DOORS = ("www.", "m.", "mobile.", "web.") +_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"} +_DEFAULT_RETENTION_DAYS = 30 # config.download_retention_days on the day this was written + + +def _site_key(url: str) -> str: + host = (urlsplit(url).hostname or "").lower().rstrip(".") + for door in _FRONT_DOORS: + if host.startswith(door) and host.count(".") > 1: + host = host[len(door) :] + break + if host in _YOUTUBE_HOSTS: + return "youtube" + return f"site:{host or 'unknown'}" + + +def _asset_key(kind: str | None, ref: str | None) -> str: + if kind == "youtube": + return "youtube" + if kind == "url" and ref: + return _site_key(ref) + return "upload" + + +def _key(job, jobs_by_id, assets) -> str: + kind = job.source_kind + if kind == "youtube": + return "youtube" + if kind == "upload": + return "upload" + if kind == "clip": + spec = job.edit_spec or {} + return "plex" if spec.get("source") == "plex" else "youtube" + if kind == "url": + return _site_key(job.source_ref) + if kind == "edit": + parent = jobs_by_id.get(job.source_job_id) + if parent is not None and parent.id != job.id: + return _key(parent, jobs_by_id, assets) + asset = assets.get(job.source_asset_id) + if asset is not None: + return _asset_key(asset.source_kind, asset.source_ref) + return "upload" + + +def upgrade() -> None: + op.create_table( + "file_folders", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("kind", sa.String(16), nullable=False), + sa.Column("source_key", sa.String(260)), # "site:" + a host (<= 253) + sa.Column("name", sa.String(255)), + sa.Column("retention_days", sa.Integer()), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"), + sa.CheckConstraint( + "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)" + " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)", + name="ck_file_folder_kind", + ), + ) + op.create_index("ix_file_folders_user_id", "file_folders", ["user_id"]) + op.create_index( + "uq_file_folder_own_name", + "file_folders", + ["user_id", sa.text("lower(name)")], + unique=True, + postgresql_where=sa.text("name IS NOT NULL"), + ) + + op.add_column( + "download_jobs", + sa.Column( + "folder_id", + sa.Integer(), + sa.ForeignKey("file_folders.id", ondelete="RESTRICT"), + ), + ) + op.add_column( + "download_jobs", + sa.Column("kept", sa.Boolean(), server_default="false", nullable=False), + ) + op.add_column("download_jobs", sa.Column("expires_at", sa.DateTime(timezone=True))) + op.add_column( + "download_jobs", + sa.Column("expiry_warned", sa.Boolean(), server_default="false", nullable=False), + ) + op.create_index("ix_download_jobs_folder_id", "download_jobs", ["folder_id"]) + op.create_index("ix_download_jobs_expires_at", "download_jobs", ["expires_at"]) + + conn = op.get_bind() + raw = conn.execute( + sa.text("SELECT value FROM system_config WHERE key = 'download_retention_days'") + ).scalar() + retention = int(raw) if raw not in (None, "") else _DEFAULT_RETENTION_DAYS + + jobs = conn.execute( + sa.text( + "SELECT j.id, j.user_id, j.source_kind, j.source_ref, j.edit_spec, j.source_job_id," + " j.source_asset_id, j.status, j.asset_id, a.expires_at AS asset_expires_at" + " FROM download_jobs j LEFT JOIN media_assets a ON a.id = j.asset_id" + ) + ).fetchall() + jobs_by_id = {j.id: j for j in jobs} + assets = { + a.id: a + for a in conn.execute(sa.text("SELECT id, source_kind, source_ref FROM media_assets")) + } + + folder_ids: dict[tuple[int, str], int] = {} + for job in jobs: + key = _key(job, jobs_by_id, assets) + slot = (job.user_id, key) + if slot not in folder_ids: + folder_ids[slot] = conn.execute( + sa.text( + "INSERT INTO file_folders (user_id, kind, source_key, retention_days)" + " VALUES (:u, 'source', :k, :r) RETURNING id" + ), + {"u": job.user_id, "k": key, "r": None if key == "upload" else retention}, + ).scalar_one() + # A done file keeps the expiry it had (its asset's); an upload had none and keeps none. + expires = job.asset_expires_at if job.status == "done" and key != "upload" else None + conn.execute( + sa.text("UPDATE download_jobs SET folder_id = :f, expires_at = :e WHERE id = :id"), + {"f": folder_ids[slot], "e": expires, "id": job.id}, + ) + + op.drop_column("media_assets", "expires_at") + op.drop_column("media_assets", "gc_notified") + + +def downgrade() -> None: + op.add_column("media_assets", sa.Column("expires_at", sa.DateTime(timezone=True))) + op.add_column( + "media_assets", + sa.Column("gc_notified", sa.Boolean(), server_default="false", nullable=False), + ) + # The asset goes when its last holder would: the latest of its holders' expiries (a holder that + # never expires keeps the asset unexpired). + op.execute( + "UPDATE media_assets a SET expires_at = sub.e FROM (" + " SELECT asset_id, CASE WHEN bool_or(expires_at IS NULL) THEN NULL ELSE max(expires_at) END" + " AS e FROM download_jobs WHERE asset_id IS NOT NULL AND status = 'done' GROUP BY asset_id" + ") sub WHERE a.id = sub.asset_id AND a.source_kind <> 'upload'" + ) + op.drop_index("ix_download_jobs_expires_at", table_name="download_jobs") + op.drop_index("ix_download_jobs_folder_id", table_name="download_jobs") + op.drop_column("download_jobs", "expiry_warned") + op.drop_column("download_jobs", "expires_at") + op.drop_column("download_jobs", "kept") + op.drop_column("download_jobs", "folder_id") + op.drop_index("uq_file_folder_own_name", table_name="file_folders") + op.drop_index("ix_file_folders_user_id", table_name="file_folders") + op.drop_table("file_folders") diff --git a/backend/alembic/versions/0103_files_sha256.py b/backend/alembic/versions/0103_files_sha256.py new file mode 100644 index 00000000..5e3e8629 --- /dev/null +++ b/backend/alembic/versions/0103_files_sha256.py @@ -0,0 +1,32 @@ +"""The Files library: a file's SHA-256, and the expired rows retired + +Revision ID: 0103_files_sha256 +Revises: 0102_file_folders +Create Date: 2026-10-11 + +`media_assets.sha256` holds the file's hash once it is known (an upload is hashed while it streams +in, any other file on the first ask), so a machine fetching a file can check what it got. + +An expired file now leaves its user's library (`downloads.gc`). Before, an expired download stayed +as a fileless "expired · download again" row (a done job with no asset). Those rows hold nothing on +disk and the library no longer shows them, so they are deleted here rather than left behind +unseen; their shares and public links go with them (ON DELETE CASCADE), and those already served +nothing. The downgrade cannot bring them back. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0103_files_sha256" +down_revision: str | None = "0102_file_folders" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column("media_assets", sa.Column("sha256", sa.String(64))) + op.execute("DELETE FROM download_jobs WHERE status = 'done' AND asset_id IS NULL") + + +def downgrade() -> None: + op.drop_column("media_assets", "sha256") diff --git a/backend/alembic/versions/0104_job_ready_at.py b/backend/alembic/versions/0104_job_ready_at.py new file mode 100644 index 00000000..3be74ad3 --- /dev/null +++ b/backend/alembic/versions/0104_job_ready_at.py @@ -0,0 +1,38 @@ +"""A file's "Added" is when it became done + +Revision ID: 0104_job_ready_at +Revises: 0103_files_sha256 +Create Date: 2026-10-12 + +`download_jobs.ready_at` is when a job became done, and the Files library sorts and dates by it: a +download that failed for days and then went through used to sit days down the list, under its +request date. Every path to done stamps it, held by `ck_download_jobs_ready_at`. + +A done job from before has no record of that moment (its "ready to watch" notice never named the +job, so it cannot be matched back), so it takes its request date, which is where it already sat. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0104_job_ready_at" +down_revision: str | None = "0103_files_sha256" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column("download_jobs", sa.Column("ready_at", sa.DateTime(timezone=True))) + op.execute( + """ + UPDATE download_jobs SET ready_at = created_at WHERE status = 'done' + """ + ) + op.create_check_constraint( + "ck_download_jobs_ready_at", "download_jobs", "status <> 'done' OR ready_at IS NOT NULL" + ) + + +def downgrade() -> None: + op.drop_constraint("ck_download_jobs_ready_at", "download_jobs", type_="check") + op.drop_column("download_jobs", "ready_at") diff --git a/backend/alembic/versions/0105_job_source_key.py b/backend/alembic/versions/0105_job_source_key.py new file mode 100644 index 00000000..6f9cc019 --- /dev/null +++ b/backend/alembic/versions/0105_job_source_key.py @@ -0,0 +1,37 @@ +"""A file remembers where it came from + +Revision ID: 0105_job_source_key +Revises: 0104_job_ready_at +Create Date: 2026-10-12 + +`download_jobs.source_key` is the origin a file came from — the source folder it may go back to — +recorded when it is placed. An edit used to work its origin out through its parent each time, and +once the parent was deleted or expired it fell back to "upload": moved into an own folder, it could +no longer go back to its folder, and deleting that folder sent it to Uploads to stay forever. + +A file in a source folder takes that folder's key. One already in an own folder is left NULL and +keeps being worked out (`folders.source_key_for`), as before. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0105_job_source_key" +down_revision: str | None = "0104_job_ready_at" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column("download_jobs", sa.Column("source_key", sa.String(260))) + op.execute( + """ + UPDATE download_jobs j SET source_key = f.source_key + FROM file_folders f + WHERE f.id = j.folder_id AND f.kind = 'source' + """ + ) + + +def downgrade() -> None: + op.drop_column("download_jobs", "source_key") diff --git a/backend/app/audit.py b/backend/app/audit.py index 14087279..c7180113 100644 --- a/backend/app/audit.py +++ b/backend/app/audit.py @@ -58,6 +58,17 @@ class AuditAction: # A user sent a debug session's issue to the tracker — the one act that takes their log # off this instance, so it is recorded (target = the session, summary = the issue's URL). ISSUE_CREATE = "issue_create" + # The Files library (`routes.files`): what a user put in, took out, kept or reorganised. Ids + # only — a file's or folder's name is the user's own text, not the admin's to read here. + FILE_UPLOAD = "file_upload" + FILE_DELETE = "file_delete" + FILE_KEEP = "file_keep" + FILE_UNKEEP = "file_unkeep" + FILE_MOVE = "file_move" + FILE_FOLDER_CREATE = "file_folder_create" + FILE_FOLDER_RENAME = "file_folder_rename" + FILE_FOLDER_RETENTION = "file_folder_retention" + FILE_FOLDER_DELETE = "file_folder_delete" def record( diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py index befa1d11..c0ecab4c 100644 --- a/backend/app/bodylimit.py +++ b/backend/app/bodylimit.py @@ -92,8 +92,8 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { # So this 413 is a refusal a real user meets on a default install, not a theoretical edge, and # it has to be as readable as the route's own: the client localizes `too_large` (`errors.tooLarge`) # exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before - # sending it at all, off the `upload_max_bytes` that `GET /api/downloads/usage` reports. - ("POST", "/api/downloads/upload"): settings.download_upload_max_bytes, + # sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports. + ("POST", "/api/files/upload"): settings.download_upload_max_bytes, ("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES, } diff --git a/backend/app/config.py b/backend/app/config.py index 8edf93aa..fd477645 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Literal, get_args, get_origin -from pydantic import Field, ValidationInfo, field_validator, model_validator +from pydantic import AliasChoices, Field, ValidationInfo, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict # Shipped placeholder; production must override it. Used by the startup guard below. @@ -171,9 +171,13 @@ class Settings(BaseSettings): feed_default_hidden_live: str = "live,upcoming" # --- Download center (yt-dlp) --- - # Root directory (inside the container) for downloaded media — a Docker volume/bind mount. + # Root directory (inside the container) for the files library — a Docker volume/bind mount. # Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it. - download_root: str = "/downloads" + # DOWNLOAD_ROOT is its old name: an install still on the old compose file (mounted at + # /downloads, DOWNLOAD_ROOT=/downloads) keeps finding its files after it pulls a new image. + files_root: str = Field( + default="/files", validation_alias=AliasChoices("FILES_ROOT", "DOWNLOAD_ROOT", "files_root") + ) # Whether THIS process runs the download worker loop. On for the dedicated worker # container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled. worker_enabled: bool = False @@ -345,8 +349,8 @@ class Settings(BaseSettings): plex_transcode_crf: int = 20 plex_transcode_max_height: int = 1080 # Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls` - # dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point - # this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast + # dir under files_root. On localdev the files_root is a slow Windows bind-mount, so point + # this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's files_root is fast # local disk, so the default is fine there. plex_hls_dir: str = "" diff --git a/backend/app/downloads/backfill.py b/backend/app/downloads/backfill.py index b793a770..13d035ab 100644 --- a/backend/app/downloads/backfill.py +++ b/backend/app/downloads/backfill.py @@ -89,7 +89,7 @@ def run() -> dict: if a.poster_path is None: # Record a self-hosted poster for every download (the reliable og:image source): # ensure_poster returns the existing .jpg thumbnail sidecar, or cuts a frame. - rel = editmod.ensure_poster(a, settings.download_root) + rel = editmod.ensure_poster(a, settings.files_root) if rel: a.poster_path = rel filled_poster += 1 diff --git a/backend/app/downloads/edit.py b/backend/app/downloads/edit.py index 21b82983..2ce485b1 100644 --- a/backend/app/downloads/edit.py +++ b/backend/app/downloads/edit.py @@ -732,7 +732,7 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path return dest if dest.exists() else None -def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None: +def ensure_poster(asset, files_root: str, timeout_s: int = 30) -> str | None: """Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's `.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned @@ -740,7 +740,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None: short clip) dodges black intro frames.""" if not asset.rel_path: return None - root = Path(download_root) + root = Path(files_root) src = root / asset.rel_path if not src.exists(): return None @@ -756,7 +756,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None: return rel if dest.exists() else None -def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None: +def ensure_storyboard(asset, files_root: str, timeout_s: int = 90) -> dict | None: """Return the filmstrip meta for a ready source asset, generating the sprite on first use. Best-effort + time-bounded: for a very long/high-res source the one-pass decode can be slow, @@ -768,7 +768,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | if not asset.rel_path: return None geom = storyboard_geometry(asset.duration_s) - root = Path(download_root) + root = Path(files_root) rel = f".storyboards/asset-{asset.id}.jpg" dest = root / rel diff --git a/backend/app/downloads/errors.py b/backend/app/downloads/errors.py index 0f50f62d..846dcb03 100644 --- a/backend/app/downloads/errors.py +++ b/backend/app/downloads/errors.py @@ -144,6 +144,13 @@ _RULES: tuple[tuple[str, str], ...] = ( ("postprocessing", "postprocess"), ("produced no output file", "postprocess"), ("no output file", "postprocess"), + # yt-dlp's own verdict that its extractor did not understand the site's answer: it appends this + # request for a bug report to an UNEXPECTED extraction failure, never to an expected one (a private, + # removed or geo-blocked video says so in words matched above). Measured 2026-10-12: a TikTok link + # failed with it on every attempt under yt-dlp 2026.07.04 and downloads under 2026.08.19 — the + # site changed and a newer yt-dlp is the cure. After the media-format and post-processing rules, + # whose texts can carry the same request; before the network ones, which are about the transport. + ("please report this issue", "site_changed"), # Network / transport. ("unable to download", "network"), ("http error", "network"), diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py new file mode 100644 index 00000000..6908f06c --- /dev/null +++ b/backend/app/downloads/folders.py @@ -0,0 +1,286 @@ +"""The folders of the Files library — where each of a user's files sits. + +A folder is a row, never a directory: on disk a file stays where the download layout put it (one +shared asset can be held by several users), so a rename or a move is a column update. Two kinds: + +- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips, + uploads, one per other site). Its `source_key` names the origin; the client renders its name + from the key, in the reader's language. +- an OWN folder is the user's, with the name they gave it. + +Every file of a user is in exactly one folder. The folder a new file lands in is decided here +(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file +moves only into one of the user's own folders or back to its own source folder (`move`): a YouTube +file in the TikTok folder would be a lie the folder's name tells. + +Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the +folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A +done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user +kept it — and its `ready_at`, the moment it finished (`stamp_done`). +""" + +from datetime import datetime, timedelta, timezone +from enum import StrEnum +from urllib.parse import urlsplit + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app import sysconfig +from app.downloads.status import JobStatus +from app.models import DownloadJob, FileFolder, MediaAsset + + +class FolderKind(StrEnum): + SOURCE = "source" + OWN = "own" + + +class SourceKey(StrEnum): + """The fixed origins. Any other site is `site:` (see `site_key`).""" + + YOUTUBE = "youtube" + PLEX = "plex" + UPLOAD = "upload" + + +SITE_PREFIX = "site:" + +# A host's leading label that only says "the same site, another front door". +_FRONT_DOORS = ("www.", "m.", "mobile.", "web.") +_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"} + + +def site_key(url: str) -> str: + """The source key of a page on some site: `site:` without its front door (www., m.), + or the YouTube key for a YouTube address that arrived as a plain URL.""" + host = (urlsplit(url).hostname or "").lower().rstrip(".") + for door in _FRONT_DOORS: + if host.startswith(door) and host.count(".") > 1: + host = host[len(door) :] + break + if host in _YOUTUBE_HOSTS: + return SourceKey.YOUTUBE + return f"{SITE_PREFIX}{host or 'unknown'}" + + +def source_key_for(db: Session, job: DownloadJob) -> str: + """The origin a job's file came from — the source folder it belongs in. + + The one recorded when it was placed, else: youtube → YouTube; a clip → the clip's source + (YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file + came from (its parent job, or, once that is deleted, the parent's asset).""" + if job.source_key: + return job.source_key + kind = job.source_kind + if kind == "youtube": + return SourceKey.YOUTUBE + if kind == "upload": + return SourceKey.UPLOAD + if kind == "clip": + return ( + SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE + ) + if kind == "url": + return site_key(job.source_ref) + if kind == "edit": + parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None + if parent is not None and parent.id != job.id: + return source_key_for(db, parent) + asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None + if asset is not None: + if asset.source_kind == "youtube": + return SourceKey.YOUTUBE + if asset.source_kind == "upload": + return SourceKey.UPLOAD + if asset.source_kind == "url": + return site_key(asset.source_ref) + return SourceKey.UPLOAD + + +def source_folder(db: Session, user_id: int, key: str) -> FileFolder: + """The user's source folder for `key`, made on first use. A race on the unique (user, key) is + settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive.""" + stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key) + folder = db.execute(stmt).scalar_one_or_none() + if folder is not None: + return folder + retention = ( + None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days") + ) + folder = FileFolder( + user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention + ) + try: + with db.begin_nested(): + db.add(folder) + db.flush() + # silent-ok: the expected outcome of the race the savepoint exists for — another request made + # the same folder first; re-selecting it is the whole handling. + except IntegrityError: + folder = db.execute(stmt).scalar_one() + return folder + + +def place(db: Session, job: DownloadJob) -> None: + """Put a new job's file in its source folder, and record that origin on it. Called wherever a + job is created; the job must carry its `user_id`, `source_kind` and source refs already.""" + key = source_key_for(db, job) + job.source_key = key + job.folder_id = source_folder(db, job.user_id, key).id + + +def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None: + """When a file that finishes now goes: its folder's retention from now, or never (kept, or a + folder that keeps files).""" + if kept or folder_id is None: + return None + # Read under a share lock, held to the caller's commit: a retention change (`set_retention` locks + # the folder) waits for this file and restamps it too, or this read waits for the change and + # takes the new value — never the old value stamped on a file the change did not see. + days = db.execute( + select(FileFolder.retention_days) + .where(FileFolder.id == folder_id) + .with_for_update(read=True) + ).scalar() + return None if days is None else datetime.now(timezone.utc) + timedelta(days=days) + + +def stamp_done(db: Session, job: DownloadJob) -> None: + """Stamp a job that just became done (in this session): when it did, and its expiry.""" + job.ready_at = datetime.now(timezone.utc) + job.expires_at = done_expiry(db, job.folder_id, job.kept) + job.expiry_warned = False + + +class FolderError(Exception): + """A folder change the rules refuse; `reason` is the refusal the route answers with.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder: + """The user's folder `folder_id` that a file is about to be put in — held (FOR SHARE) until the + caller's transaction ends, so a delete of the folder waits for the file and then takes it home + with the rest (`delete_own` locks it FOR UPDATE), never failing on its foreign key. `hold=False` + only looks (a check ahead of a long wait, which must not hold a lock through it). Another user's + folder answers like a missing one.""" + stmt = select(FileFolder).where(FileFolder.id == folder_id) + if hold: + stmt = stmt.with_for_update(read=True) + folder = db.execute(stmt.execution_options(populate_existing=True)).scalar_one_or_none() + if folder is None or folder.user_id != user_id: + raise FolderError("file_folder_gone") + return folder + + +def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder: + """The folder an upload asked to land in, if it may: one of the user's own folders, or their + Uploads folder. A file put in a site's folder would claim an origin it does not have.""" + folder = own_folder(db, user_id, folder_id, hold=hold) + if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD: + raise FolderError("file_folder_not_its_source") + return folder + + +def restamp(db: Session, job: DownloadJob) -> None: + """Start a file's clock again from now: it moved to another folder, or its user stopped keeping + it. A file not done yet gets its expiry when it finishes (`stamp_done`), so it has none now.""" + done = job.status == JobStatus.DONE + job.expires_at = done_expiry(db, job.folder_id, job.kept) if done else None + job.expiry_warned = False + + +def set_kept(db: Session, job: DownloadJob, kept: bool) -> None: + """Keep a file until its user deletes it, or hand it back to its folder's retention.""" + job.kept = kept + restamp(db, job) + + +def move(db: Session, job: DownloadJob, folder: FileFolder) -> None: + """Put a file in `folder` — one of its user's own folders, or the source folder it came from. + The caller has checked that `folder` is the job's user's.""" + if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job): + raise FolderError("file_folder_not_its_source") + if folder.id == job.folder_id: + return + job.folder_id = folder.id + restamp(db, job) + + +def _hold_for_change(db: Session, folder: FileFolder) -> None: + """Lock `folder` FOR UPDATE and read it afresh into the same object: a change waits out a file + being put in it or stamped from it right now (`own_folder`, `done_expiry`), and works from what + another change that landed first left, not from what the caller read before the wait. The row is + consumed on purpose: an ORM result refreshes the object only as its rows are fetched.""" + fresh = db.execute( + select(FileFolder) + .where(FileFolder.id == folder.id) + .with_for_update() + .execution_options(populate_existing=True) + ).scalar_one_or_none() + if fresh is None: + raise FolderError("file_folder_gone") + + +def set_retention(db: Session, folder: FileFolder, days: int | None) -> int | None: + """Change how long `folder` keeps a file (None = until deleted). The done files already in it + move by the difference — each keeps the age it has — but none ends up closer than the GC's + warning window, so a shortened retention still warns before it takes anything. Returns the + retention it replaced (equal to `days` when nothing changed).""" + _hold_for_change(db, folder) # so the file arriving now is among the ones restamped below + old = folder.retention_days + if old == days: + return old + folder.retention_days = days + now = datetime.now(timezone.utc) + floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days")) + # Locked as they are read: one moved out or deleted meanwhile is no longer this folder's to time. + jobs = db.execute( + select(DownloadJob) + .where( + DownloadJob.folder_id == folder.id, + DownloadJob.status == JobStatus.DONE, + DownloadJob.kept.is_(False), + ) + .with_for_update() + .execution_options(populate_existing=True) + ).scalars() + for job in jobs: + if days is None: + job.expires_at = None + elif old is None or job.expires_at is None: + job.expires_at = max(now + timedelta(days=days), floor) + else: + job.expires_at = max(job.expires_at + timedelta(days=days - old), floor) + job.expiry_warned = False + return old + + +def delete_own(db: Session, folder: FileFolder) -> int: + """Delete one of a user's own folders. Its files are not deleted: each goes back to the source + folder it came from, its clock starting there now. Returns how many moved.""" + if folder.kind != FolderKind.OWN: + raise FolderError("file_folder_not_own") + _hold_for_change(db, folder) # so the file arriving now is taken home too + # Locked as they are read: a file moved out meanwhile stays where it went, and one deleted + # meanwhile is not written back (its gone row would fail the delete). + jobs = ( + db.execute( + select(DownloadJob) + .where(DownloadJob.folder_id == folder.id) + .with_for_update() + .execution_options(populate_existing=True) + ) + .scalars() + .all() + ) + for job in jobs: + job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id + restamp(db, job) + db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete + db.delete(folder) + return len(jobs) diff --git a/backend/app/downloads/gc.py b/backend/app/downloads/gc.py index 7a9c2afd..32d9a83a 100644 --- a/backend/app/downloads/gc.py +++ b/backend/app/downloads/gc.py @@ -1,12 +1,13 @@ """Download retention garbage collector (a scheduler job). Five passes each run: - 1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder - once (gc_notified flag) so a download isn't silently deleted. - 2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the - holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI. - 3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed - ready assets (never an upload) until back under it. + 1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls + within the grace window, and that its user did not keep, warns its user once (`expiry_warned`). + 2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold + released, so the shared file goes with its last holder. + 3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed ready + assets (never an upload, never one a user kept) until back under it; each holder's file leaves + their library, with a notice. 4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so passes 1-3 skip them; benign row-count hygiene). 5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a @@ -14,19 +15,20 @@ Five passes each run: It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour, so an in-flight download is never touched. -Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota. +Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota. """ import time from datetime import datetime, timedelta, timezone from pathlib import Path -from sqlalchemy import func, or_, select +from sqlalchemy import func, or_, select, update from sqlalchemy.orm import Session from app import logs, sysconfig from app.config import settings -from app.downloads import quota, storage +from app.downloads import quota, service, storage +from app.downloads.status import JobStatus from app.models import DownloadJob, MediaAsset from app.notifications import create_notification @@ -37,42 +39,118 @@ _UPLOAD_TREE = ".uploads" _ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit -def _holders(db: Session, asset_id: int) -> list[int]: - return list( - db.execute( - select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct() - ).scalars() +def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) -> None: + title = job.display_name or (asset.title if asset else None) or "download" + create_notification( + db, + user_id=job.user_id, + type=ntype, + title=title, # English fallback; the client renders a translated message from data + data={ + "kind": ntype, + "title": title, + "video_id": asset.source_ref if asset else None, + "job_id": job.id, + }, + commit=False, ) -def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None: - title = asset.title or "download" - for uid in _holders(db, asset.id): - create_notification( - db, - user_id=uid, - type=ntype, - title=title, # English fallback; the client renders a translated message from data - data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref}, - commit=False, +def _relock_expired( + db: Session, job_id: int, asset_id: int | None, now: datetime +) -> DownloadJob | None: + """Re-read an expired job under the lock order the user routes take (`own_job_for_update`: the + asset, then the job) and say whether it is still to go. Between the pass's SELECT and here a + user may have deleted or re-downloaded it (dropping it again would release its asset a second + time, freeing a file another job still holds) or kept or moved it (it is no longer expired).""" + if asset_id is not None: + db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update()) + job = db.execute( + select(DownloadJob) + .where(DownloadJob.id == job_id) + .with_for_update() + .execution_options(populate_existing=True) + ).scalar_one_or_none() + still = ( + job is not None + and job.asset_id == asset_id + and job.status == JobStatus.DONE + and not job.kept + and job.expires_at is not None + and job.expires_at <= now + ) + return job if still else None + + +def _mark_warned(db: Session, job_id: int, expires_at: datetime) -> DownloadJob | None: + """Mark one file warned if it still goes when the pass saw it go — not deleted, kept, moved or + re-timed since (each of those sets a fresh expiry, and its own warning to come) — and return it. + The UPDATE's row lock holds it so until the commit.""" + hit = db.execute( + update(DownloadJob) + .where( + DownloadJob.id == job_id, + DownloadJob.status == JobStatus.DONE, + DownloadJob.kept.is_(False), + DownloadJob.expiry_warned.is_(False), + DownloadJob.expires_at == expires_at, ) + .values(expiry_warned=True) + .returning(DownloadJob.id) + .execution_options(synchronize_session=False) + ).scalar_one_or_none() + return db.get(DownloadJob, job_id) if hit is not None else None -def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None: - if asset.rel_path: - storage.delete_asset_files(settings.download_root, asset.rel_path) - _notify_holders(db, asset, ntype) - db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL +def _on_disk(db: Session) -> int: + """The bytes the cache holds now (every asset whose file is on disk).""" + return ( + db.execute( + select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK) + ).scalar() + or 0 + ) -def _sweep_orphan_files(db: Session, download_root: str) -> int: - """Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset +def _relock_evictable(db: Session, asset_id: int) -> tuple[MediaAsset | None, list[DownloadJob]]: + """Re-read an eviction candidate and its holders under their locks (the asset, then its jobs — + the order the user routes take): between the pass's SELECT and here a holder may have kept it, + or every holder deleted it.""" + asset = db.execute( + select(MediaAsset) + .where(MediaAsset.id == asset_id) + .with_for_update() + .execution_options(populate_existing=True) + ).scalar_one_or_none() + holders = ( + db.execute( + select(DownloadJob) + .where(DownloadJob.asset_id == asset_id) + .with_for_update() + .execution_options(populate_existing=True) + ) + .scalars() + .all() + ) + return asset, list(holders) + + +def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None: + """Take a file out of its user's library: tell them, release the job's hold on the asset (the + file goes with the last holder), delete the row (its shares and links cascade).""" + _notify(db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, ntype) + service.release_asset(db, job) + db.delete(job) + + +def _sweep_orphan_files(db: Session, files_root: str) -> int: + """Delete media/sidecar files (and now-empty dirs) under FILES_ROOT that no `ready` asset owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload row names at all. Skips the other dot-prefixed system trees and only removes files older than `_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files NOT owned by a surviving asset.""" - root = Path(download_root) + root = Path(files_root) if not root.is_dir(): return 0 keep: set[Path] = set() @@ -151,74 +229,95 @@ def run_download_gc(db: Session) -> dict: grace_days = sysconfig.get_int(db, "download_gc_grace_days") warned = expired = evicted = reaped = 0 - # 1. Pre-expiry warnings. + # 1. Pre-expiry warnings — one per file, the first run inside the grace window. Each mark is + # conditional on the expiry the SELECT saw and committed on its own: a file deleted since is + # passed over (a write to its gone row would fail the whole run), and one kept, moved or re-timed + # since keeps its fresh expiry unwarned. warn_before = now + timedelta(days=grace_days) - soon = ( - db.execute( - select(MediaAsset).where( - MediaAsset.status == "ready", - MediaAsset.gc_notified.is_(False), - MediaAsset.expires_at.is_not(None), - MediaAsset.expires_at > now, - MediaAsset.expires_at <= warn_before, - ) + soon = db.execute( + select(DownloadJob.id, DownloadJob.expires_at).where( + DownloadJob.status == JobStatus.DONE, + DownloadJob.kept.is_(False), + DownloadJob.expiry_warned.is_(False), + DownloadJob.expires_at.is_not(None), + DownloadJob.expires_at > now, + DownloadJob.expires_at <= warn_before, ) - .scalars() - .all() - ) - for asset in soon: - _notify_holders(db, asset, "download_expiring") - asset.gc_notified = True - warned += 1 - if soon: + ).all() + db.rollback() # the pass-1 snapshot holds no locks; each mark takes its own below + for job_id, expires_at in soon: + job = _mark_warned(db, job_id, expires_at) + if job is not None: + asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None + _notify(db, job, asset, "download_expiring") + warned += 1 db.commit() - # 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL). - dead = ( - db.execute( - select(MediaAsset).where( - MediaAsset.status.in_(("ready", "error")), - MediaAsset.expires_at.is_not(None), - MediaAsset.expires_at <= now, - ) + # 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1). + # Each file is re-checked under its locks and committed on its own: the release unlinks the + # file at once, so it must never act on what the SELECT saw a while ago. + dead = db.execute( + select(DownloadJob.id, DownloadJob.asset_id).where( + DownloadJob.status == JobStatus.DONE, + DownloadJob.kept.is_(False), + DownloadJob.expires_at.is_not(None), + DownloadJob.expires_at <= now, ) - .scalars() - .all() - ) - for asset in dead: - _delete_asset(db, asset, "download_expired") - expired += 1 - if dead: + ).all() + db.rollback() # the pass-2 snapshot holds no locks; each file takes its own below + for job_id, asset_id in dead: + job = _relock_expired(db, job_id, asset_id, now) + if job is not None: + _drop_job(db, job, "download_expired") + expired += 1 db.commit() - # 3. LRU eviction over the total cache cap. + # 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to + # fetch again) and never a file anyone kept; an evicted file leaves every holder's library. cap = sysconfig.get_int(db, "download_total_max_bytes") if cap > 0: - total = ( - db.execute( - select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK) - ).scalar() - or 0 - ) - if total > cap: - # An upload counts toward the total (it is disk like any other) but is never evicted: - # it has no source to fetch again, and it lives until its owner deletes it. - candidates = ( + if _on_disk(db) > cap: + kept_assets = select(DownloadJob.asset_id).where( + DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None) + ) + candidate_ids = ( db.execute( - select(MediaAsset) - .where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload") + select(MediaAsset.id) + .where( + MediaAsset.status == "ready", + MediaAsset.source_kind != "upload", + MediaAsset.id.not_in(kept_assets), + ) .order_by(MediaAsset.last_access_at.asc().nulls_first()) ) .scalars() .all() ) - for asset in candidates: - if total <= cap: + db.rollback() + for asset_id in candidate_ids: + # Counted afresh before each: a file deleted since (this one, an upload, a candidate + # further down) has freed its space, and no other user's file goes for that space. + if _on_disk(db) <= cap: break - total -= asset.size_bytes or 0 - _delete_asset(db, asset, "download_evicted") + # Locked and re-read like an expiry: since the SELECT a holder may have kept it, + # every holder deleted it, or someone asked for it again (a fresh cache hit). + asset, holders = _relock_evictable(db, asset_id) + if ( + asset is None + or asset.status != "ready" + or any(j.kept for j in holders) + or (asset.last_access_at is not None and asset.last_access_at > now) + ): + db.commit() + continue + for job in holders: + _notify(db, job, asset, "download_evicted") + db.delete(job) + if asset.rel_path: + storage.delete_asset_files(settings.files_root, asset.rel_path) + db.delete(asset) evicted += 1 - db.commit() + db.commit() # 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still- # referenced errored row to pending on re-enqueue, so these only accumulate once every @@ -257,7 +356,7 @@ def run_download_gc(db: Session) -> dict: ) for asset in unheld: if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row - storage.delete_asset_files(settings.download_root, asset.rel_path) + storage.delete_asset_files(settings.files_root, asset.rel_path) db.delete(asset) reaped += 1 if unheld: @@ -265,7 +364,7 @@ def run_download_gc(db: Session) -> dict: # 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale # posters). Runs last so the keep-set reflects the assets that survived passes 2-4. - swept = _sweep_orphan_files(db, settings.download_root) + swept = _sweep_orphan_files(db, settings.files_root) result = { "warned": warned, diff --git a/backend/app/downloads/jobdb.py b/backend/app/downloads/jobdb.py index a0935749..a8e95b29 100644 --- a/backend/app/downloads/jobdb.py +++ b/backend/app/downloads/jobdb.py @@ -68,8 +68,8 @@ def _reset_asset_pending(asset_id: int) -> None: def _staging_dir(asset_id: int) -> Path: - return Path(settings.download_root) / ".staging" / f"asset-{asset_id}" + return Path(settings.files_root) / ".staging" / f"asset-{asset_id}" def _edit_staging_dir(asset_id: int) -> Path: - return Path(settings.download_root) / ".staging" / f"edit-{asset_id}" + return Path(settings.files_root) / ".staging" / f"edit-{asset_id}" diff --git a/backend/app/downloads/og.py b/backend/app/downloads/og.py index 2bc7bf47..99cd2154 100644 --- a/backend/app/downloads/og.py +++ b/backend/app/downloads/og.py @@ -135,7 +135,7 @@ def _watch_tags(token: str, db: Session) -> str | None: if image_dims is None and image.endswith("poster.jpg") and asset.poster_path: # safe_abs_path returns None if the poster file is missing (gc'd) or escapes the root — # guard it, else _jpeg_size(None) would raise an uncaught AttributeError on this public page. - poster_abs = storage.safe_abs_path(settings.download_root, asset.poster_path) + poster_abs = storage.safe_abs_path(settings.files_root, asset.poster_path) image_dims = _jpeg_size(poster_abs) if poster_abs else None if image_dims: tags.append(_tag("og:image:width", str(image_dims[0]))) diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index 2b937438..307d3886 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -3,7 +3,7 @@ import re import shutil import time -from datetime import date, datetime, timedelta, timezone +from datetime import date, datetime, timezone from pathlib import Path import yt_dlp @@ -95,16 +95,24 @@ def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, notification failure must never mask the completed job, so it is isolated + rolled back (mirrors scheduler._notify_done). `kind` ("download"/"edit"/"upload") lets the inbox pick accurate copy; the English title here is only a fallback for an unknown type. `user_id`/`title` are captured by the - caller BEFORE its commit, so reading them here needs no post-commit refresh of an expired row.""" + caller BEFORE its commit; only the folder is read here, once, for the notice's "Open".""" if user_id is None: return try: + # The notice's "Open" lands on the file in its folder: both ride along (a folder the file + # has left since is the page's to notice — it then looks among every file). + job = db.get(DownloadJob, job_id) create_notification( db, user_id=user_id, type="download_done", title=f"Ready to watch: {title}" if title else "Ready to watch", - data={"title": title, "kind": kind}, + data={ + "title": title, + "kind": kind, + "job_id": job_id, + "folder_id": job.folder_id if job else None, + }, ) except Exception: db.rollback() @@ -434,14 +442,12 @@ def _extract_with_fallback( raise last_exc # unreachable (loop either returns or raises) -def _download_settings() -> tuple[str, int]: - """Layout + retention days from the DB config (admin-overridable on the Configuration page), - falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect.""" +def _download_layout() -> str: + """The on-disk layout from the DB config (admin-overridable on the Configuration page), falling + back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long + a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)""" with SessionLocal() as db: - return ( - sysconfig.get_str(db, "download_layout"), - sysconfig.get_int(db, "download_retention_days"), - ) + return sysconfig.get_str(db, "download_layout") def _ensure_phone_safe( @@ -588,19 +594,18 @@ def _download( thumbnail_url=info.get("thumbnail"), ) - layout, retention_days = _download_settings() + layout = _download_layout() rel = storage.rel_path(meta, ext, layout, asset_id=asset_id) thumb = _find_thumbnail(staging, produced) # Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the # poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure # this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead. poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg") - storage.place_file(produced, settings.download_root, rel) - nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg) + storage.place_file(produced, settings.files_root, rel) + nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg) - final = storage.abs_path(settings.download_root, rel) + final = storage.abs_path(settings.files_root, rel) size = final.stat().st_size if final.exists() else None - expires = datetime.now(timezone.utc) + timedelta(days=retention_days) with SessionLocal() as db: asset = db.get(MediaAsset, asset_id) @@ -626,12 +631,11 @@ def _download( # (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires # and can't be relied on cross-origin. write_sidecars already wrote .jpg from # the thumbnail; ensure_poster returns that, or cuts a frame if there was none. - asset.poster_path = editmod.ensure_poster(asset, settings.download_root) + asset.poster_path = editmod.ensure_poster(asset, settings.files_root) asset.source_webpage_url = info.get("webpage_url") asset.nfo_written = nfo_ok asset.error = None asset.last_access_at = datetime.now(timezone.utc) - asset.expires_at = expires job = db.get(DownloadJob, job_id) # Capture the owner before commit expires the row (avoids a refresh SELECT just to notify). owner_id = job.user_id if job else None @@ -840,7 +844,7 @@ def _process_edit(job_id: int, asset_id: int) -> None: if not src_ready or not src_rel: _fail_edit(job_id, asset_id, "The source download is no longer available.") return - src_path = storage.abs_path(settings.download_root, src_rel) + src_path = storage.abs_path(settings.files_root, src_rel) if not src_path.exists(): _fail_edit(job_id, asset_id, "The source file is missing.") return @@ -887,13 +891,11 @@ def _process_edit(job_id: int, asset_id: int) -> None: raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE) rel = storage.edit_rel_path(user_id, asset_id, out_ext) - storage.place_file(dest_staging, settings.download_root, rel) - final = storage.abs_path(settings.download_root, rel) + storage.place_file(dest_staging, settings.files_root, rel) + final = storage.abs_path(settings.files_root, rel) size = final.stat().st_size if final.exists() else None # The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`). _, _, out_w, out_h, _ = probe.probe_media(final) - _, retention_days = _download_settings() - expires = datetime.now(timezone.utc) + timedelta(days=retention_days) with SessionLocal() as db: asset = db.get(MediaAsset, asset_id) @@ -912,7 +914,6 @@ def _process_edit(job_id: int, asset_id: int) -> None: asset.width, asset.height = src_w, src_h asset.error = None asset.last_access_at = datetime.now(timezone.utc) - asset.expires_at = expires job = db.get(DownloadJob, job_id) # Capture owner + title before commit expires the rows (avoids a refresh SELECT). owner_id = job.user_id if job else None @@ -1086,10 +1087,9 @@ def _process_clip(job_id: int, asset_id: int) -> None: raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE) ext = out.suffix.lstrip(".").lower() or "mp4" rel = storage.edit_rel_path(user_id, asset_id, ext) - storage.place_file(out, settings.download_root, rel) - final = storage.abs_path(settings.download_root, rel) + storage.place_file(out, settings.files_root, rel) + final = storage.abs_path(settings.files_root, rel) vcodec, acodec, width, height, _ = probe.probe_media(final) - _, retention_days = _download_settings() with SessionLocal() as db: asset = db.get(MediaAsset, asset_id) @@ -1108,10 +1108,9 @@ def _process_clip(job_id: int, asset_id: int) -> None: asset.media_kind = mediakind.derive(vcodec, acodec) # The clip's own frame as its poster: the link preview shows the moment, and a Plex # item has no public thumbnail at all. - asset.poster_path = editmod.ensure_poster(asset, settings.download_root) + asset.poster_path = editmod.ensure_poster(asset, settings.files_root) asset.error = None asset.last_access_at = datetime.now(timezone.utc) - asset.expires_at = datetime.now(timezone.utc) + timedelta(days=retention_days) fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None} # A video outside the catalog was queued as "Clip (clip)": the fetch has its title now. placeholder = service.clip_title(None, None) @@ -1230,7 +1229,7 @@ def _free_unheld_upload(asset_id: int) -> bool: if asset is None or asset.ref_count != 0: return False if asset.rel_path: - storage.delete_asset_files(settings.download_root, asset.rel_path) + storage.delete_asset_files(settings.files_root, asset.rel_path) db.delete(asset) db.commit() return True @@ -1253,7 +1252,7 @@ def _convert_upload(job_id: int, asset_id: int) -> None: duration = asset.duration_s if asset else None job = db.get(DownloadJob, job_id) owner_id = job.user_id if job else None - src = storage.safe_abs_path(settings.download_root, old_rel) if old_rel else None + src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None if old_rel is None or src is None or not src.exists(): _fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload") return @@ -1267,10 +1266,10 @@ def _convert_upload(job_id: int, asset_id: int) -> None: new_rel = storage.with_ext(old_rel, old_ext, ext) if safe != src: _check_converted_fits(owner_id, old_size, safe.stat().st_size) - storage.place_file(safe, settings.download_root, new_rel) + storage.place_file(safe, settings.files_root, new_rel) if new_rel != old_rel: - placed = storage.abs_path(settings.download_root, new_rel) - final = storage.abs_path(settings.download_root, new_rel) + placed = storage.abs_path(settings.files_root, new_rel) + final = storage.abs_path(settings.files_root, new_rel) vcodec, acodec, width, height, dur = probe.probe_media(final) with SessionLocal() as db: asset = db.get(MediaAsset, asset_id) diff --git a/backend/app/downloads/preview.py b/backend/app/downloads/preview.py new file mode 100644 index 00000000..9a8cb593 --- /dev/null +++ b/backend/app/downloads/preview.py @@ -0,0 +1,107 @@ +"""What a library file previews as — the single home of the preview allow-list. + +A generic file (`media_kind == "file"`; video and audio have their players) previews by its stored +`container`, the upload's cleaned extension, and ONLY through this table: an image is shown as an +image, a PDF in the browser's own viewer, and a text format as plain text. Anything else has no +preview. The served Content-Type comes from here too, never from a guess on the name, and HTML / SVG +sit in the TEXT list on purpose: rendered on this origin, an uploaded page would run with the +reader's session. The route adds `CSP: sandbox` on top, so even a mislabelled file (a "png" that is +really a page) gets no script and no origin. + +The serializer hands the kind to the client (`asset.preview`), so the library and the server can +never disagree on which files open a preview. +""" + +import codecs +from typing import Literal + +from app.downloads import mediakind + +PreviewKind = Literal["image", "pdf", "text"] + +IMAGE_TYPES: dict[str, str] = { + "png": "image/png", + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "gif": "image/gif", + "webp": "image/webp", + "avif": "image/avif", + "bmp": "image/bmp", +} + +PDF_TYPE = "application/pdf" + +# Read as text: notes, data, subtitles, config and source. HTML and SVG belong here (see above). +TEXT_EXTS: frozenset[str] = frozenset( + { + "txt", "text", "md", "markdown", "log", "csv", "tsv", "json", "jsonl", "ndjson", "xml", + "yaml", "yml", "toml", "ini", "conf", "cfg", "properties", "env", "srt", "vtt", "ass", "ssa", + "sub", "nfo", "cue", "m3u", "m3u8", "html", "htm", "xhtml", "svg", "css", "scss", "less", + "js", "mjs", "cjs", "ts", "tsx", "jsx", "py", "rb", "php", "pl", "lua", "go", "rs", "java", + "kt", "kts", "swift", "c", "h", "cc", "cpp", "hpp", "cs", "sh", "bash", "zsh", "fish", "ps1", + "bat", "cmd", "sql", "r", "dart", "diff", "patch", "tex", "rst", "adoc", "gradle", "dockerfile", + } +) # fmt: skip + +TEXT_TYPE = "text/plain; charset=utf-8" + +# A text preview is the file's head: enough to read, never a whole log loaded into a tab. +TEXT_PREVIEW_BYTES = 1 << 20 +# A NUL byte in the first this-many bytes means the "text" is not text. +BINARY_SNIFF_BYTES = 8192 + + +def kind(media_kind: str | None, container: str | None) -> PreviewKind | None: + """The preview a library file gets, or None (a media file plays instead; others download).""" + if media_kind != mediakind.FILE: + return None + ext = (container or "").lower() + if ext in IMAGE_TYPES: + return "image" + if ext == "pdf": + return "pdf" + if ext in TEXT_EXTS: + return "text" + return None + + +def media_type(preview: PreviewKind, container: str | None) -> str: + """The Content-Type a preview is served with: from this table, never guessed from the name.""" + if preview == "image": + return IMAGE_TYPES[(container or "").lower()] + if preview == "pdf": + return PDF_TYPE + return TEXT_TYPE + + +# A text that is not UTF-8 is read as Central European Windows: what an old Hungarian subtitle or +# note is written in (its ő and ű are not in Latin-1). +FALLBACK_ENCODING = "cp1250" +_BOMS = ( + (codecs.BOM_UTF8, "utf-8-sig"), + (codecs.BOM_UTF16_LE, "utf-16"), + (codecs.BOM_UTF16_BE, "utf-16"), +) + + +def looks_binary(head: bytes) -> bool: + """Whether the start of a "text" file is binary after all (a NUL in it).""" + return b"\x00" in head[:BINARY_SNIFF_BYTES] + + +def text_utf8(head: bytes) -> bytes | None: + """A text file's head as UTF-8, the one charset the preview is served in; None when it is not text. + A byte-order mark names its encoding (UTF-16 is full of NULs, so it is read before the binary + check); else UTF-8 is kept as it is — a character cut at the size cap included, the reader shows + that one as a replacement mark — and anything else is read as `FALLBACK_ENCODING`.""" + for bom, encoding in _BOMS: + if head.startswith(bom): + return head.decode(encoding, errors="replace").encode() + if looks_binary(head): + return None + try: + head.decode("utf-8") + except UnicodeDecodeError as e: # silent-ok: not UTF-8 is an answer — read it as the fallback + if not (e.reason == "unexpected end of data" and e.end == len(head)): + return head.decode(FALLBACK_ENCODING, errors="replace").encode() + return head diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index 96770af0..bbe5d82c 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -19,7 +19,7 @@ from app.downloads import adapters from app.downloads import clip as clipmod from app.downloads import credentials from app.downloads import edit as editmod -from app.downloads import formats, mediakind, quota, storage +from app.downloads import folders, formats, mediakind, quota, storage from app.downloads.status import HOLDING, JobKind, JobStatus from app.models import ( DISPLAY_TEXT_MAX, @@ -49,7 +49,7 @@ def source_url(source_kind: str, source_ref: str) -> str: def reference_url(job, asset) -> str | None: - """The clean "downloaded from" URL for a job — shared by the private Downloads page and the public + """The clean "downloaded from" URL for a job — shared by the Files library and the public watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's own earlier download, not a web page) and for a Plex clip (the library is not a public page); a YouTube clip links its video at the clip's start; else the canonical page URL yt-dlp recorded; else @@ -65,6 +65,23 @@ def reference_url(job, asset) -> str | None: return None +def library_left_behind(db: Session, files_root: str) -> bool: + """True when the library names files and none of the newest of them is under `files_root` — a + fresh, empty volume mounted in the library's place (a compose file changed under an install, its + old volume left behind). The mount check cannot see that: the new volume is a real mount.""" + rels = ( + db.execute( + select(MediaAsset.rel_path) + .where(MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)) + .order_by(MediaAsset.id.desc()) + .limit(20) + ) + .scalars() + .all() + ) + return bool(rels) and not any(storage.abs_path(files_root, r).exists() for r in rels if r) + + def get_or_create_asset( db: Session, source_kind: str, source_ref: str, format_sig: str ) -> MediaAsset: @@ -77,7 +94,12 @@ def get_or_create_asset( MediaAsset.source_ref == source_ref, MediaAsset.format_sig == format_sig, ) - asset = db.execute(stmt).scalar_one_or_none() + # Locked to the caller's commit: a release or the GC freeing this asset meanwhile (they lock it + # too) either waits for the new hold, or has deleted the row by the time this read returns — and + # the caller then gets a fresh one, never a job pointing at a row that is gone. + asset = db.execute( + stmt.with_for_update().execution_options(populate_existing=True) + ).scalar_one_or_none() if asset is not None: # A previously-FAILED shared asset must be re-attempted for the new job about to hold it — # reset it to pending (+clear the error) so the worker actually re-downloads instead of @@ -198,12 +220,14 @@ def enqueue( queue_pos=next_queue_pos(db), ) db.add(job) + folders.place(db, job) incr_ref_count(db, asset.id) # Cache hit: the file already exists — no worker round-trip needed. if asset.status == "ready": job.status = JobStatus.DONE job.progress = 100 + folders.stamp_done(db, job) asset.last_access_at = datetime.now(timezone.utc) db.commit() @@ -277,11 +301,13 @@ def enqueue_edit( queue_pos=next_queue_pos(db), ) db.add(job) + folders.place(db, job) incr_ref_count(db, asset.id) if asset.status == "ready": # identical clip already produced for this user → instant job.status = JobStatus.DONE job.progress = 100 + folders.stamp_done(db, job) asset.last_access_at = datetime.now(timezone.utc) db.commit() @@ -364,11 +390,13 @@ def enqueue_clip( queue_pos=next_queue_pos(db), ) db.add(job) + folders.place(db, job) incr_ref_count(db, asset.id) if asset.status == "ready": # the same range was already cut for this user → instant job.status = JobStatus.DONE job.progress = 100 + folders.stamp_done(db, job) asset.last_access_at = datetime.now(timezone.utc) db.commit() @@ -392,6 +420,8 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool: if not fields: return False db.flush() # asset-then-job lock order: push pending asset writes before we lock the job row + if fields.get("status") == JobStatus.DONE: + return _finalize_done(db, job_id, {**fields, "ready_at": datetime.now(timezone.utc)}) # A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the # rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this # is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound @@ -405,6 +435,41 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool: return result.rowcount > 0 +def _finalize_done(db: Session, job_id: int, fields: dict) -> bool: + """`finalize_if_running` for a job that finished: done is when a file's retention starts, its + folder's unless the user kept it. The expiry is written only while the job is still where it was + read from — a move, a keep, or its folder's delete (which sends its files home) landing while the + folder's retention was read sends the read round again, never stamping a folder's retention on a + file now in another one.""" + while True: + held = db.execute( + select(DownloadJob.folder_id, DownloadJob.kept, DownloadJob.status).where( + DownloadJob.id == job_id + ) + ).one_or_none() + if held is None or held.status != JobStatus.RUNNING: + return False + stamped = { + **fields, + "expires_at": folders.done_expiry(db, held.folder_id, held.kept), + "expiry_warned": False, + } + hit = db.execute( + update(DownloadJob) + .where( + DownloadJob.id == job_id, + DownloadJob.status == JobStatus.RUNNING.value, + DownloadJob.folder_id.is_not_distinct_from(held.folder_id), + DownloadJob.kept == held.kept, + ) + .values(**stamped) + .returning(DownloadJob.id) + .execution_options(synchronize_session=False) + ).scalar_one_or_none() + if hit is not None: + return True + + def release_asset(db: Session, job: DownloadJob) -> None: """Drop this job's hold on its asset when it leaves a holding state. Once NO job holds the asset anymore, delete the file + row immediately — a deleted download should free its disk, @@ -436,7 +501,7 @@ def release_asset(db: Session, job: DownloadJob) -> None: ) if new_count == 0 and freeable: if asset.rel_path: - storage.delete_asset_files(settings.download_root, asset.rel_path) + storage.delete_asset_files(settings.files_root, asset.rel_path) db.delete(asset) @@ -565,4 +630,6 @@ def redownload(db: Session, job: DownloadJob) -> None: # name would survive the rebuild). job.display_name = None job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot + job.expires_at = None # stamped again when the rebuild is done + job.expiry_warned = False db.commit() diff --git a/backend/app/downloads/status.py b/backend/app/downloads/status.py index 7849930e..f8d6da0d 100644 --- a/backend/app/downloads/status.py +++ b/backend/app/downloads/status.py @@ -53,3 +53,7 @@ NOT_REFETCHABLE = frozenset({JobKind.EDIT, JobKind.UPLOAD, JobKind.CLIP}) # States a route sets to STOP a running job. The worker's supervisor must step in (kill the child, free # the asset, leave the route's status intact) when it observes one of these on a job it is running. STOPPED_BY_ROUTE = frozenset({JobStatus.PAUSED, JobStatus.CANCELED}) + + +# A job that will not move again on its own: the queue lists it only briefly after it got here. +FINISHED = frozenset({JobStatus.DONE, JobStatus.CANCELED}) diff --git a/backend/app/downloads/storage.py b/backend/app/downloads/storage.py index 4a3ea17e..c3de9bf8 100644 --- a/backend/app/downloads/storage.py +++ b/backend/app/downloads/storage.py @@ -11,6 +11,7 @@ a Plex library using a metadata agent picks up rich info. Channel = "show", vide date-based — the community-standard mapping for YouTube-in-Plex. """ +import os import re import shutil import unicodedata @@ -67,6 +68,57 @@ _FOLD_MAP = str.maketrans( ) +def _mount_points(mountinfo: str) -> list[str]: + """The mount points a /proc//mountinfo table lists (its fifth field, octal escapes undone).""" + points = [] + for line in mountinfo.splitlines(): + fields = line.split(" ") + if len(fields) > 4: + points.append(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), fields[4])) + return points + + +def _in_container() -> bool: + return Path("/.dockerenv").exists() or Path("/run/.containerenv").exists() + + +def _read_mountinfo() -> str | None: + try: + return Path("/proc/self/mountinfo").read_text() + # silent-ok: no mount table to read (not Linux) — the caller then cannot tell, and refuses nothing. + except OSError: + return None + + +def files_root_problem( + root: str, mountinfo: str | None = None, in_container: bool | None = None +) -> str | None: + """Why `root` cannot be the files library's volume, or None when it can. In a container, a root + that is no mount of its own (nor inside one) is the image's empty directory: the library would + answer 404 for every file, and what a download writes would go when the container does — the + state an install is in when its compose file still mounts the library somewhere else. + + Asked of the kernel's mount table, not by comparing devices: a storage driver that keeps the + container's root on the host's own disk (vfs, the fallback under Docker-in-LXC on older ZFS) gives + a mounted volume the same device as `/`. Outside a container a plain directory is a fine root.""" + path = Path(root) + if not path.is_dir(): + return f"{root} does not exist" + if in_container if in_container is not None else _in_container(): + table = mountinfo if mountinfo is not None else _read_mountinfo() + if table is not None: + real = os.path.realpath(root) + mounted = any( + m != "/" and (real == m or real.startswith(m.rstrip("/") + "/")) + for m in _mount_points(table) + ) + if not mounted: + return f"{root} is not a mounted volume (it is the container's own empty directory)" + if not os.access(path, os.W_OK): + return f"{root} is not writable" + return None + + def _ascii_fold(text: str) -> str: """Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have @@ -122,7 +174,7 @@ def display_filename(name: str) -> str: def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | None = None) -> str: - """Path of the media file relative to DOWNLOAD_ROOT (forward slashes). + """Path of the media file relative to FILES_ROOT (forward slashes). `asset_id` disambiguates the file. A MediaAsset is unique on (source_kind, source_ref, format_sig), so the SAME video downloaded in two formats is two distinct assets — but without @@ -144,7 +196,7 @@ def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | No def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str: - """On-disk path (relative to DOWNLOAD_ROOT) for a phase-2 editor derivative. + """On-disk path (relative to FILES_ROOT) for a phase-2 editor derivative. Kept in a per-user `.edits/` tree — device-downloadable via the file endpoint, but NOT part of the Plex library (these are user clips, not canonical episodes; there are no .nfo/poster @@ -161,7 +213,7 @@ def _clean_upload_ext(filename: str) -> str: def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str: - """On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file. + """On-disk path (relative to FILES_ROOT) for a locally-uploaded file. Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster @@ -250,24 +302,27 @@ def download_filename(display_name: str, container: str | None, path: Path) -> s return f"{base}.{ext}" -def abs_path(download_root: str, rel: str) -> Path: - return Path(download_root) / rel +def abs_path(files_root: str, rel: str) -> Path: + return Path(files_root) / rel -def safe_abs_path(download_root: str, rel: str) -> Path | None: - """The resolved absolute path for `rel`, but ONLY if it stays inside DOWNLOAD_ROOT and exists on - disk — otherwise None. Centralizes the path-traversal + existence guard every file-serving endpoint - shares, so the containment check lives in exactly one place.""" - root = Path(download_root).resolve() - path = abs_path(download_root, rel).resolve() +def safe_abs_path(files_root: str, rel: str | None) -> Path | None: + """The resolved absolute path for `rel`, but ONLY if it stays inside FILES_ROOT and exists on + disk — otherwise None (no `rel` at all included: an asset without a file has nothing to serve). + Centralizes the path-traversal + existence guard every file-serving endpoint shares, so the + containment check lives in exactly one place.""" + if not rel: + return None + root = Path(files_root).resolve() + path = abs_path(files_root, rel).resolve() if root not in path.parents or not path.exists(): return None return path -def place_file(staging_file: Path, download_root: str, rel: str) -> None: - """Move a completed staging file into its final location under DOWNLOAD_ROOT.""" - dest = abs_path(download_root, rel) +def place_file(staging_file: Path, files_root: str, rel: str) -> None: + """Move a completed staging file into its final location under FILES_ROOT.""" + dest = abs_path(files_root, rel) dest.parent.mkdir(parents=True, exist_ok=True) shutil.move(str(staging_file), str(dest)) @@ -301,9 +356,9 @@ def _sidecar(media: Path, ext: str) -> Path: return media.with_suffix(ext) -def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool: +def write_sidecars(files_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool: """Write `.nfo` + `.jpg` next to the media file. Returns True if written.""" - media = abs_path(download_root, rel) + media = abs_path(files_root, rel) try: _sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8") if thumb_src and thumb_src.exists(): @@ -319,9 +374,9 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat return False -def delete_asset_files(download_root: str, rel: str) -> None: +def delete_asset_files(files_root: str, rel: str) -> None: """Remove the media file and its sidecars; prune now-empty parent dirs (best effort).""" - media = abs_path(download_root, rel) + media = abs_path(files_root, rel) for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")): try: p.unlink() @@ -335,7 +390,7 @@ def delete_asset_files(download_root: str, rel: str) -> None: extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__), ) # Prune empty Season/Channel dirs up to (but not including) the root. - root = Path(download_root).resolve() + root = Path(files_root).resolve() parent = media.parent while parent != root and root in parent.parents: try: diff --git a/backend/app/downloads/upload.py b/backend/app/downloads/upload.py index 7612c4d3..7e382a19 100644 --- a/backend/app/downloads/upload.py +++ b/backend/app/downloads/upload.py @@ -31,9 +31,11 @@ from pathlib import Path from sqlalchemy.orm import Session +from app import audit +from app.audit import AuditAction from app.config import settings from app.downloads import edit as editmod -from app.downloads import mediakind, playable, probe, quota, service, storage +from app.downloads import folders, mediakind, playable, probe, quota, service, storage from app.downloads.status import JobKind, JobStatus from app.models import DownloadJob, MediaAsset, display_text @@ -53,14 +55,24 @@ def ingest_upload( src_path: Path, size: int, display_name: str | None = None, + sha256: str | None = None, + folder_id: int | None = None, ) -> DownloadJob: """Ingest an already-uploaded file at `src_path` into `user_id`'s library and return its job — `done`, or `queued` for the worker's phone-safe conversion (see the module docstring). `src_path` - is MOVED into place on success (the caller stages it to a temp file first). + is MOVED into place on success (the caller stages it to a temp file first). `sha256` is the + hash of those bytes, when the caller took it on the way in; `folder_id` the folder it asked for + (`folders.upload_target`), else its Uploads folder. + + Raises folders.FolderError when that folder is not one an upload may land in (or is gone). Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a rejected upload leaves nothing behind and the caller just cleans up its temp file.""" quota.check_upload(db, user_id, size) # the file's size is known → charge it up front + if folder_id is not None: + # Refused before any work; held only once the job is about to name it (below), not through + # the probe and the poster, which a rename or delete of the folder would wait out. + folders.upload_target(db, user_id, folder_id, hold=False) data = probe.probe_json(src_path) vcodec, acodec, width, height, duration = probe.facts(data) @@ -81,18 +93,21 @@ def ingest_upload( db.flush() # need asset.id for the on-disk path rel = storage.upload_rel_path(user_id, asset.id, filename) - storage.place_file(src_path, settings.download_root, rel) + storage.place_file(src_path, settings.files_root, rel) # From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the # asset/job rows roll back and nothing names the placed file: the per-user delete path keys off # asset.rel_path (which is gone), and the GC's `.uploads` sweep reaches it only an hour later. So # roll back the on-disk side too — unless the commit itself raised after it landed (see below). asset_id = asset.id try: - final = storage.abs_path(settings.download_root, rel) + final = storage.abs_path(settings.files_root, rel) actual_size = final.stat().st_size if final.exists() else size asset.status = "pending" if convert else "ready" asset.rel_path = rel + # The hash names the bytes that arrived: a conversion replaces them, so it is kept only + # when the file stays as it came. + asset.sha256 = None if convert else sha256 asset.size_bytes = actual_size asset.container = ext or None asset.vcodec = vcodec @@ -108,7 +123,7 @@ def ingest_upload( # A frame poster gives the library row a thumbnail + the share link an og:image. Audio has # no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes # server side even for a codec the browser can't play). - asset.poster_path = editmod.ensure_poster(asset, settings.download_root) + asset.poster_path = editmod.ensure_poster(asset, settings.files_root) job = DownloadJob( user_id=user_id, @@ -123,7 +138,14 @@ def ingest_upload( queue_pos=service.next_queue_pos(db), ) db.add(job) + folders.place(db, job) + if folder_id is not None: + job.folder_id = folders.upload_target(db, user_id, folder_id).id + if not convert: + folders.stamp_done(db, job) service.incr_ref_count(db, asset.id) + db.flush() + audit.record(db, user_id, AuditAction.FILE_UPLOAD, target_type="file", target_id=job.id) db.commit() except Exception: db.rollback() @@ -139,7 +161,7 @@ def ingest_upload( # return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op # when none was generated (audio/file, or a video whose poster never started). for leaked in (rel, str(Path(rel).with_suffix(".jpg"))): - storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True) + storage.abs_path(settings.files_root, leaked).unlink(missing_ok=True) raise db.refresh(job) return job diff --git a/backend/app/logs.py b/backend/app/logs.py index ecafc7d2..62ff3609 100644 --- a/backend/app/logs.py +++ b/backend/app/logs.py @@ -690,7 +690,9 @@ _POLLED_ROUTES = frozenset( "/api/me/notifications/unread_count", "/api/downloads", "/api/downloads/index", - "/api/downloads/usage", + "/api/files", + "/api/files/folders", + "/api/files/usage", "/api/sync/my-status", "/api/admin/scheduler", "/api/admin/downloads", diff --git a/backend/app/main.py b/backend/app/main.py index e06746b9..795456b7 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -28,6 +28,8 @@ log = logs.get_logger(__name__) from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first from app.config import settings # noqa: E402 — logging is configured first from app.db import SessionLocal, engine # noqa: E402 — logging is configured first +from app.downloads import service as download_service # noqa: E402 — logging is configured first +from app.downloads import storage # noqa: E402 — logging is configured first from app.plex import stream as plex_stream # noqa: E402 — logging is configured first from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first from app.routes import ( # noqa: E402 — logging is configured first @@ -42,6 +44,7 @@ from app.routes import ( # noqa: E402 — logging is configured first discover as discover_routes, downloads, feed, + files as files_routes, health, me, notifications, @@ -63,6 +66,10 @@ from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — @asynccontextmanager async def lifespan(app: FastAPI): log.info("Siftlode starting up") + problem = storage.files_root_problem(settings.files_root) + if problem: + # The rest of the app still serves; the files library cannot (the worker refuses to start). + log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem) # The stored copy of the log (log_events). The entrypoint has already migrated the schema. logsink.start() # First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print @@ -78,6 +85,18 @@ async def lifespan(app: FastAPI): # With the traceback: this catch spans a connection, a query and a coercion at boot, # and `%s` of the exception does not say which of them raised. log.exception("could not read the stored log level; staying at %s", current_level()) + if not problem: + try: + left = download_service.library_left_behind(db, settings.files_root) + except Exception: # a database not reachable yet: the check is a hint, never a gate + log.exception("could not check the files library against its database") + else: + if left: + log.critical( + "The files library is empty but the database names files: the volume " + "mounted at %s is not the one that holds them. See docs/self-hosting.md.", + settings.files_root, + ) if not state.is_configured(db): token = state.rotate_setup_token(db) url = f"{settings.app_base}/setup?token={token}" @@ -267,7 +286,8 @@ async def static_cache_headers(request, call_next): # Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning: # nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to # external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be -# iframed anywhere (we never embed our own pages; we embed OTHERS, which this does not affect), and +# iframed anywhere (we embed OTHERS, which this does not affect; the one own answer the app frames, a +# library file's preview, sets its own `frame-ancestors 'self'` — these are defaults, `setdefault`), and # Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates # the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and # suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy @@ -316,6 +336,7 @@ app.include_router(saved_views.router) app.include_router(plex_routes.router) app.include_router(downloads.router) app.include_router(downloads.admin_router) +app.include_router(files_routes.router) app.include_router(public_routes.router) app.include_router(admin.router) app.include_router(config_routes.router) diff --git a/backend/app/models.py b/backend/app/models.py index ec4c780c..6fe9726f 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,4 +1,4 @@ -from datetime import date, datetime +from datetime import date, datetime, timezone from sqlalchemy import ( BigInteger, @@ -18,6 +18,7 @@ from sqlalchemy import ( Text, UniqueConstraint, func, + text, ) from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR from sqlalchemy.orm import Mapped, mapped_column, relationship, validates @@ -38,7 +39,7 @@ def display_text(raw: str | None) -> str | None: blank -> None. One definition because there are SIX writers and they had drifted: `POST /api/downloads`, - `POST /api/downloads/edit`, `POST /api/downloads/upload`, the metadata PATCH's two fields, and + `POST /api/downloads/edit`, `POST /api/files/upload`, the library PATCH's two fields, and the Google sign-in callback. Only the PATCH truncated. Truncating is the right answer rather than refusing: the name is a label on work the user has already done — a file already transferred, an account already authenticated — and losing its tail is not worth failing the @@ -826,12 +827,13 @@ class MediaAsset(Base, TimestampMixin): links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is shared; per-user edited derivatives (phase 2) are never cached here. - `rel_path` is relative to `settings.download_root` (the mount), so the physical filename is + `rel_path` is relative to `settings.files_root` (the mount), so the physical filename is system-decided and bound to the source id — never renamed (the user's custom name is display metadata on `download_job`, applied only when downloading to their own machine). The naming fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built - uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) - plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job.""" + uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees + the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file + stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's).""" __tablename__ = "media_assets" __table_args__ = ( @@ -847,10 +849,10 @@ class MediaAsset(Base, TimestampMixin): status: Mapped[str] = mapped_column( String(16), default="pending", server_default="pending", index=True ) # "pending" | "ready" | "error" - rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root + rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic # A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the - # card / watch page / link preview can still show an image. Relative to download_root. + # card / watch page / link preview can still show an image. Relative to files_root. poster_path: Mapped[str | None] = mapped_column(Text) size_bytes: Mapped[int | None] = mapped_column(BigInteger) duration_s: Mapped[int | None] = mapped_column(Integer) @@ -866,8 +868,6 @@ class MediaAsset(Base, TimestampMixin): # earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here). media_kind: Mapped[str | None] = mapped_column(String(16)) nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - # Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run. - gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # Denormalized naming/metadata for the Plex tree + ad-hoc display. title: Mapped[str | None] = mapped_column(Text) uploader: Mapped[str | None] = mapped_column(String(255)) @@ -877,13 +877,21 @@ class MediaAsset(Base, TimestampMixin): upload_date: Mapped[date | None] = mapped_column(Date) thumbnail_url: Mapped[str | None] = mapped_column(Text) # yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from" - # reference shown on the Downloads page. Null until the worker extracts it (older/queued rows + # reference shown in the Files library. Null until the worker extracts it (older/queued rows # fall back to a URL derived from source_kind+source_ref). source_webpage_url: Mapped[str | None] = mapped_column(Text) error: Mapped[str | None] = mapped_column(Text) ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + # The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload + # streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet. + sha256: Mapped[str | None] = mapped_column(String(64)) + + @validates("rel_path") + def _forget_hash(self, _key: str, rel_path: str | None) -> str | None: + # A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them. + self.sha256 = None + return rel_path class DownloadProfile(Base, TimestampMixin): @@ -907,6 +915,47 @@ class DownloadProfile(Base, TimestampMixin): sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") +class FileFolder(Base, TimestampMixin): + """A folder of the Files library (`app.downloads.folders`): a row, never a directory. + + A SOURCE folder (`source_key` set: youtube / plex / upload / site:) is made the first time a + user gets a file from that origin; an OWN folder (`name` set) is one the user made. Every file — + a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a + file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it.""" + + __tablename__ = "file_folders" + __table_args__ = ( + UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"), + Index( + "uq_file_folder_own_name", + "user_id", + func.lower(text("name")), + unique=True, + postgresql_where=text("name IS NOT NULL"), + ), + CheckConstraint( + "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)" + " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)", + name="ck_file_folder_kind", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind + # "site:" + a host name (DNS caps one at 253 characters). + source_key: Mapped[str | None] = mapped_column(String(260)) + name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) + retention_days: Mapped[int | None] = mapped_column(Integer) + + +def _ready_on_insert(context) -> datetime | None: + """A job inserted already done (an upload kept as it came) was ready the moment it was made; one + that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`).""" + done = context.get_current_parameters().get("status") == "done" + return datetime.now(timezone.utc) if done else None + + class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): """A user's request to download one source — the per-user layer over `MediaAsset`. @@ -925,7 +974,13 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe.""" __tablename__ = "download_jobs" - __table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),) + __table_args__ = ( + Index("ix_download_jobs_claim", "status", "queue_pos"), + # Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`). + CheckConstraint( + "status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at" + ), + ) id: Mapped[int] = mapped_column(primary_key=True) user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) @@ -974,6 +1029,27 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): # The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL. error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + # The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own + # folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether + # the user keeps it past its folder's retention, and when it goes (stamped once done; NULL = + # never). `expiry_warned` marks the one warning the GC sends before it deletes the file. + folder_id: Mapped[int | None] = mapped_column( + ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True + ) + # The origin the file came from (the source folder's key), recorded where it is placed: an edit + # traces its origin through its parent, and that parent may be deleted or expire later — the + # file must still know which source folder it may go back to. NULL only for a file placed before + # the column existed outside a source folder; `folders.source_key_for` works those out. + source_key: Mapped[str | None] = mapped_column(String(260)) + kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) + expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + # When the file became done — what the library's "Added" means: a download that failed for days + # and then went through is new to its user, so it sorts as new. `created_at` stays when it was + # asked for (the queue's order). Set again when a file is made anew (a retry, a re-download). + ready_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), default=_ready_on_insert + ) class DownloadShare(Base, TimestampMixin): diff --git a/backend/app/plex/cache.py b/backend/app/plex/cache.py index b1aec046..1836f713 100644 --- a/backend/app/plex/cache.py +++ b/backend/app/plex/cache.py @@ -10,4 +10,4 @@ from pathlib import Path from app.config import settings -CACHE_DIR = Path(settings.download_root) / ".plex-img-cache" +CACHE_DIR = Path(settings.files_root) / ".plex-img-cache" diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index 1039f6af..4616bae0 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -41,7 +41,7 @@ log = logs.get_logger(__name__) _HLS_ROOT = ( Path(settings.plex_hls_dir) if settings.plex_hls_dir - else Path(settings.download_root) / ".plex-hls" + else Path(settings.files_root) / ".plex-hls" ) _SEG_SECONDS = 6 # Cold-start warm-up. A freshly-spawned ffmpeg has produced only the first segment (or two) by the diff --git a/backend/app/routes/_common.py b/backend/app/routes/_common.py index 51cd22c4..740f3f2d 100644 --- a/backend/app/routes/_common.py +++ b/backend/app/routes/_common.py @@ -6,7 +6,7 @@ and "fetch a row this user owns or 404" idioms have one definition, one status c one existence-leak policy. """ -from typing import IO, Any, TypeVar +from typing import Any, Protocol, TypeVar from fastapi import HTTPException, Request from pydantic import BaseModel, ConfigDict, Field @@ -122,8 +122,14 @@ async def capped_body(request: Request, limit: int) -> bytes: return bytes(body) +class ByteSink(Protocol): + """Where `capped_stream_to` writes: a file, or anything that takes the bytes on their way to one.""" + + def write(self, data: bytes, /) -> object: ... + + async def capped_stream_to( - request: Request, out: IO[bytes], limit: int, refusal: HTTPException + request: Request, out: ByteSink, limit: int, refusal: HTTPException ) -> int: """`capped_body`'s sibling for a body too large to hold: the same counting rule, written straight to `out` instead of accumulated, returning the size. diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py index 3e3af777..931df5e6 100644 --- a/backend/app/routes/downloads.py +++ b/backend/app/routes/downloads.py @@ -2,20 +2,16 @@ All routes are behind `current_user`; admin routes add `admin_user`. Live job state is polled by the frontend (the worker is a separate process); this module never downloads — it enqueues, manages job -state, serves finished files (range-aware, with the user's custom display name), and shares them. +state, and shares finished files. The library itself — listing, renaming, moving, deleting and +serving a done file, and uploading one — is `routes.files`. """ -import os import re -import tempfile from typing import Literal -from collections.abc import AsyncIterator, Sequence -from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from pathlib import Path -from urllib.parse import parse_qs, unquote, urlparse +from urllib.parse import parse_qs, urlparse -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import FileResponse from pydantic import BaseModel, Field from sqlalchemy import delete, func, select @@ -27,15 +23,14 @@ from app.collation import collated from app import audit, sysconfig from app.auth import admin_user, current_user from app.config import settings -from app.db import get_db, release_read_connection +from app.db import get_db from app.downloads import clip as clipmod from app.downloads import credentials as dlcreds from app.downloads import edit as editmod from app.downloads import links as linksmod from app.downloads import quota, service, storage -from app.downloads import upload as uploadmod from app.downloads import urlguard -from app.downloads.status import NOT_REFETCHABLE, JobStatus +from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus from app.models import ( DownloadJob, DownloadLink, @@ -49,15 +44,24 @@ from app.models import ( ) from app.routes._common import ( ProgressIn, - capped_stream_to, get_or_404, kept_position, not_found, owned_or_404, ) +from app.routes.files import ( + MAX_LIST, + accessible_job, + assets_for, + own_job_for_update, + positions_for, + quota_detail, + serialize, + serialize_job, + source_keys, +) from app.security import hash_password from app.userscope import is_messageable_user, messageable_clauses -from app.utils import iso from app.refusal import refused router = APIRouter(prefix="/api/downloads", tags=["downloads"]) @@ -65,11 +69,6 @@ admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"] _VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$") -# Cap the per-user job listings so a heavy library can't return an unbounded result set. The -# Download Center renders newest-first; 500 is far beyond any real manual-download history (and -# matches the admin listing's own cap). -_MAX_LIST = 500 - # --- source + serialization ---------------------------------------------------------------- @@ -113,115 +112,6 @@ def resolve_source(raw: str) -> tuple[str, str]: raise refused(400, "invalid_link") -def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict: - ready = asset is not None and asset.status == "ready" and bool(asset.rel_path) - return { - "id": job.id, - "status": job.status, - "progress": job.progress, - "speed_bps": job.speed_bps, - "eta_s": job.eta_s, - "phase": job.phase, - "error": job.error, - "error_code": job.error_code, - "error_detail": job.error_detail, - "queue_pos": job.queue_pos, - "created_at": iso(job.created_at), - "source_kind": job.source_kind, - "source_ref": job.source_ref, - "source_url": service.reference_url(job, asset), - # A locally-generated poster to fall back to when the source had no thumbnail. - "poster_url": f"/api/downloads/{job.id}/poster.jpg" - if (asset is not None and asset.poster_path) - else None, - "profile_id": job.profile_id, - "spec": job.profile_snapshot, - "display_name": job.display_name, - "display_uploader": job.display_uploader, - "display_uploader_url": job.display_uploader_url, - "extra_links": job.extra_links or [], - "job_kind": job.job_kind, - "source_job_id": job.source_job_id, - "edit_spec": job.edit_spec, - "can_download": ready and job.status == JobStatus.DONE, - "expired": job.asset_id is None and job.status == JobStatus.DONE, - "asset": None - if asset is None - else { - "id": asset.id, - "status": asset.status, - "title": asset.title, - "uploader": asset.uploader, - "upload_date": iso(asset.upload_date), - "duration_s": asset.duration_s, - "size_bytes": asset.size_bytes, - "width": asset.width, - "height": asset.height, - "container": asset.container, - # "video" | "audio" | "file" — the library gates play/edit on this (media only); a - # generic uploaded file is download-only. Null while still in flight (set at finalize). - "media_kind": asset.media_kind, - "uploader_url": asset.uploader_url, - "thumbnail_url": asset.thumbnail_url, - "expires_at": iso(asset.expires_at), - }, - } - - -def _assets_for(db: Session, jobs: list[DownloadJob]) -> dict[int, MediaAsset]: - ids = {j.asset_id for j in jobs if j.asset_id} - if not ids: - return {} - rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars() - return {a.id: a for a in rows} - - -def _positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]: - """`user`'s resume point in each of `jobs` that has one (the library player's `library_states`).""" - ids = [j.id for j in jobs] - if not ids: - return {} - rows = db.execute( - select(LibraryState.job_id, LibraryState.position_seconds).where( - LibraryState.user_id == user.id, LibraryState.job_id.in_(ids) - ) - ).all() - return {job_id: pos for job_id, pos in rows} - - -def _serialize_job(db: Session, job: DownloadJob) -> dict: - """Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step - every single-job handler shares.""" - asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None - return _serialize(job, asset) - - -def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob: - """The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload) - or writes the status those decide on (pause). The job is re-read under a row lock, so two - overlapping requests for one job run one after the other and the second decides on the first's - committed result — never both releasing the same hold, and never a pause turning a job a cancel - just released back into a holder; either can free a file another job still holds. - - The asset is locked BEFORE the job: the order a finishing worker child takes them in - (`service.finalize_if_running`), so the two can never deadlock.""" - while True: - asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id - if asset_id is not None: - db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update()) - job = db.execute( - select(DownloadJob) - .where(DownloadJob.id == job_id) - .with_for_update() - .execution_options(populate_existing=True) - ).scalar_one_or_none() - if job is None: - raise not_found("download") - if job.asset_id == asset_id: - return job - db.rollback() # another request moved the job to another asset meanwhile: lock that one - - # --- profiles ------------------------------------------------------------------------------ @@ -431,122 +321,8 @@ def enqueue_download( try: job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name) except quota.QuotaExceeded as e: - raise HTTPException(status_code=422, detail=_quota_detail(e)) from e - return _serialize_job(db, job) - - -# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a -# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by -# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous -# caller could make the process write the whole per-file allowance to temp, per request (SEC-20 -# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency -# declared AFTER the auth one, is the only shape that closes it — the same one -# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for -# every route with a raised ceiling rather than leaving it to a comment like this one. -# -# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a -# Hungarian filename would otherwise be unsendable. -UPLOAD_FILENAME_HEADER = "x-upload-filename" -UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name" - - -@dataclass(frozen=True) -class SpooledUpload: - """What the body dependency hands the handler: the bytes already on disk, plus the two names - that came alongside them.""" - - path: Path - size: int - user_id: int - filename: str - display_name: str | None - - -def _header_text(request: Request, name: str) -> str: - """One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the - caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's - own fallback names the file.""" - raw = request.headers.get(name, "") - try: - return unquote(raw, errors="strict").strip() - except (UnicodeDecodeError, ValueError): - # silent-ok: a mis-encoded header is the caller's own malformed request, and this one is - # cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on - # a label; logging it would hand any client a free log line per request. - return "" - - -async def spool_upload( - request: Request, - user: User = Depends(current_user), - db: Session = Depends(get_db), -) -> AsyncIterator[SpooledUpload]: - """Stream the request body to a staging temp file, enforcing the per-file cap as it arrives. - - `user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so - naming `current_user` here makes the auth check a precondition of reading the body at all — a - stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest - MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit, - empty, quota, probe or move error, a client that vanished mid-upload — it is removed.""" - cap = sysconfig.get_int(db, "download_upload_max_bytes") - user_id = user.id - # Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection` - # for why, and for the precondition it enforces. Everything this dependency still needs is - # already an int or a header in hand: touching an ORM attribute below would reload `user` and - # check a connection straight back out. - release_read_connection(db) - staging = Path(settings.download_root) / ".staging" / "uploads" - staging.mkdir(parents=True, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=str(staging)) - tmp_path = Path(tmp) - try: - with os.fdopen(fd, "wb") as out: - size = await capped_stream_to( - request, - out, - cap, - HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}), - ) - yield SpooledUpload( - path=tmp_path, - size=size, - user_id=user_id, - filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload", - display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)), - ) - finally: - tmp_path.unlink(missing_ok=True) - - -@router.post("/upload") -def upload_download( - upload: SpooledUpload = Depends(spool_upload), - db: Session = Depends(get_db), -) -> dict: - """Ingest a local file into the caller's library. Any file type is accepted; media - (video/audio) gets the full function set, a generic file is download-only. The body IS the file - (see the header note above); `spool_upload` has already authenticated the caller and streamed it - to a temp path under the download volume, so what is left here is the non-yt-dlp ingest which - probes + places it and returns its job — `done`, or `queued` for the worker's phone-safe - conversion (`downloads.upload`). - - A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s - plus sync DB writes — so it must NOT run on the event loop. The streaming half is async because - reading the body is; keeping them in separate callables is what lets each run where it belongs.""" - if upload.size == 0: - raise HTTPException(status_code=400, detail={"code": "upload_empty"}) - try: - job = uploadmod.ingest_upload( - db, - upload.user_id, - filename=upload.filename, - src_path=upload.path, - size=upload.size, - display_name=upload.display_name, - ) - except quota.QuotaExceeded as e: - raise HTTPException(status_code=422, detail=_quota_detail(e)) from e - return _serialize_job(db, job) + raise HTTPException(status_code=422, detail=quota_detail(e)) from e + return serialize_job(db, job) class EnqueueEditIn(BaseModel): @@ -564,7 +340,7 @@ def enqueue_edit( The source may be the user's own download OR one shared with them — either way the resulting clip is a NEW per-user derivative in the editor's own library (counts against their quota); the source file is only read, never modified, so editing a shared video is safe.""" - source_job = _accessible_job(db, user, body.source_job_id) + source_job = accessible_job(db, user, body.source_job_id) if source_job.status != JobStatus.DONE or source_job.asset_id is None: raise refused(409, "edit_unfinished") spec = body.edit_spec @@ -572,10 +348,10 @@ def enqueue_edit( try: job = service.enqueue_edit(db, user.id, source_job, spec, display_name) except quota.QuotaExceeded as e: - raise HTTPException(status_code=422, detail=_quota_detail(e)) from e + raise HTTPException(status_code=422, detail=quota_detail(e)) from e except service.EditError as e: raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e - return _serialize_job(db, job) + return serialize_job(db, job) class ClipCropIn(BaseModel): @@ -617,57 +393,49 @@ def enqueue_clip( detail = {"code": "clip", "reason": e.reason, **e.params} raise HTTPException(status_code=status, detail=detail) from e except quota.QuotaExceeded as e: - raise HTTPException(status_code=422, detail=_quota_detail(e)) from e - return _serialize_job(db, job) + raise HTTPException(status_code=422, detail=quota_detail(e)) from e + return serialize_job(db, job) -def _quota_detail(e: quota.QuotaExceeded) -> dict: - """Structured error the trilingual client localizes (reason key + numbers), instead of a - pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail().""" - return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current} +# How long a finished job stays in the queue: the tray polls it from every page and must see a job +# turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's. +# A done job is timed from when it became done, never from its last write: a rename, a move or the +# GC's expiry warning touches an old file's row, and the tray would announce it as just finished. +QUEUE_RECENT = timedelta(minutes=10) @router.get("") def list_downloads(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]: + """The queue: what is waiting, running, paused or failed, and what finished a moment ago.""" + recent = datetime.now(timezone.utc) - QUEUE_RECENT jobs = ( db.execute( select(DownloadJob) - .where(DownloadJob.user_id == user.id) + .where( + DownloadJob.user_id == user.id, + DownloadJob.status.not_in(FINISHED) + | ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent)) + | ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)), + ) .order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()) - .limit(_MAX_LIST) + .limit(MAX_LIST) ) .scalars() .all() ) - assets = _assets_for(db, jobs) - positions = _positions_for(db, user, jobs) + assets = assets_for(db, jobs) + positions = positions_for(db, user, jobs) + sources = source_keys(db, jobs) return [ { - **_serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None), + **serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None), "position_seconds": positions.get(j.id, 0), + "source_key": sources[j.id], } for j in jobs ] -@router.get("/usage") -def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict: - """The caller's download quota, plus the per-file upload cap. - - The cap rides here rather than in `quota.usage` because it is not a quota value — it is a - `sysconfig` knob about one route — and this is the route layer, where composing the two is the - job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side - caps answer too late to be pleasant (the edge ceiling refuses the request outright, see - `bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the - limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the - boundary: the caps stay where they are and still refuse a client that ignores this. - """ - return { - **quota.usage(db, user.id), - "upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"), - } - - # Priority when a source has several jobs (e.g. a done one plus a fresh queued one). _INDEX_RANK: dict[str, int] = { JobStatus.DONE: 4, @@ -694,106 +462,22 @@ def my_download_index(user: User = Depends(current_user), db: Session = Depends( return out -@router.get("/shared") -def shared_with_me(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]: - jobs = ( - db.execute( - select(DownloadJob) - .join(DownloadShare, DownloadShare.job_id == DownloadJob.id) - .where(DownloadShare.to_user_id == user.id) - .order_by(DownloadShare.created_at.desc(), DownloadJob.id.desc()) - .limit(_MAX_LIST) - ) - .scalars() - .all() - ) - assets = _assets_for(db, jobs) - positions = _positions_for(db, user, jobs) - out = [] - for j in jobs: - data = _serialize(j, assets.get(j.asset_id)) - data["shared"] = True - data["position_seconds"] = positions.get(j.id, 0) - out.append(data) - return out - - -_MAX_EXTRA_LINKS = 8 - - -def _clean_url(raw) -> str | None: - """Accept only a plausible http(s) URL (these get rendered as clickable links + embedded in a - public watch page); anything else is dropped rather than trusted.""" - if not isinstance(raw, str): - return None - url = raw.strip() - if not url: - return None - if not url.startswith(("http://", "https://")): - return None - # The third site in this module handing user input to `urlparse`, which raises on a malformed - # authority. This one already has a "not a plausible URL" answer, so the typo belongs in it. - try: - parsed = urlparse(url) - # silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL" - # answer, the caller already has one, and any line here would carry the URL itself. - except ValueError: - return None - if not parsed.netloc: - return None - return url[:2048] - - -class UpdateDownloadMetaIn(BaseModel): - # PATCH display-only overrides; presence (model_fields_set) decides what's touched. A field - # set to null/"" clears the override (falls back to the on-disk system value). - display_name: str | None = None - display_uploader: str | None = None - display_uploader_url: str | None = None - extra_links: list | None = None - - -@router.patch("/{job_id}") -def update_download_meta( - job_id: int, - body: UpdateDownloadMetaIn, - user: User = Depends(current_user), - db: Session = Depends(get_db), -) -> dict: - """Update a download's display metadata: title, channel name + link, and extra reference URLs. - All are display-only overrides — the on-disk file keeps its system-decided, id-bound name.""" - job = owned_or_404(db, DownloadJob, job_id, user, what="download") - sent = body.model_fields_set - if "display_name" in sent: - job.display_name = display_text(body.display_name) - if "display_uploader" in sent: - job.display_uploader = display_text(body.display_uploader) - if "display_uploader_url" in sent: - job.display_uploader_url = _clean_url(body.display_uploader_url) - if "extra_links" in sent: - raw = body.extra_links or [] - cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS] - job.extra_links = cleaned or None - db.commit() - return _serialize_job(db, job) - - @router.post("/{job_id}/pause") def pause_download( job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: - job = _own_job_for_update(db, user, job_id) + job = own_job_for_update(db, user, job_id) service.pause(db, job) - return _serialize_job(db, job) + return serialize_job(db, job) @router.post("/{job_id}/resume") def resume_download( job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: - job = _own_job_for_update(db, user, job_id) + job = own_job_for_update(db, user, job_id) service.resume(db, job) - return _serialize_job(db, job) + return serialize_job(db, job) @router.post("/{job_id}/redownload") @@ -812,7 +496,7 @@ def redownload_download( under the current format/naming rules — never silently reuses a shared cached file — and (b) never deletes a file another job/user still shares: the old asset is only removed if THIS job was its last holder.""" - job = _own_job_for_update(db, user, job_id) + job = own_job_for_update(db, user, job_id) if job.job_kind in NOT_REFETCHABLE: # None has a re-fetchable source: an editor clip is derived locally, an upload IS the # source, an ad-hoc clip is a range of a source the clip itself does not re-fetch. @@ -820,82 +504,16 @@ def redownload_download( if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED): raise refused(409, "still_downloading") service.redownload(db, job) - return _serialize_job(db, job) + return serialize_job(db, job) @router.post("/{job_id}/cancel") def cancel_download( job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: - job = _own_job_for_update(db, user, job_id) + job = own_job_for_update(db, user, job_id) service.cancel(db, job) - return _serialize_job(db, job) - - -@router.delete("/{job_id}") -def delete_download( - job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) -) -> dict: - job = _own_job_for_update(db, user, job_id) - service.delete(db, job) - return {"deleted": job_id} - - -# --- file download (range-aware, custom display name) -------------------------------------- - - -def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob: - job = db.get(DownloadJob, job_id) - if job is None: - raise not_found("download") - if job.user_id == user.id: - return job - shared = db.execute( - select(DownloadShare.id).where( - DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id - ) - ).first() - if shared is None: - raise not_found("download") - return job - - -@router.get("/{job_id}/file") -def download_file( - job_id: int, - request: Request, - user: User = Depends(current_user), - db: Session = Depends(get_db), -): - job = _accessible_job(db, user, job_id) - asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None - if asset is None or asset.status != "ready" or not asset.rel_path: - raise refused(409, "download_not_ready") - path = storage.safe_abs_path(settings.download_root, asset.rel_path) - if path is None: - raise not_found("file") - - filename = storage.download_filename( - job.display_name or asset.title or asset.source_ref, asset.container, path - ) - - asset.last_access_at = datetime.now(timezone.utc) - db.commit() - # The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list - # (storage.served_media_type), NEVER guessed from the file's own extension. A generic 'file' (and - # any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html") - # returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on - # this origin. Recognised media is served with its real type so the in-app player streams it. Both - # are range-aware (FileResponse → 206). - media_type = storage.served_media_type(asset.media_kind, asset.container) - if media_type is None: - return FileResponse( - path, - filename=filename, - media_type="application/octet-stream", - content_disposition_type="attachment", - ) - return FileResponse(path, filename=filename, media_type=media_type) + return serialize_job(db, job) # --- editor filmstrip (scrub track) -------------------------------------------------------- @@ -910,7 +528,7 @@ def set_library_progress( ) -> dict: """Checkpoint the library player's resume point in a download the caller can play (their own, or one shared with them). A position too early or too close to the end clears it instead.""" - _accessible_job(db, user, job_id) + accessible_job(db, user, job_id) position = kept_position(body) mine = (LibraryState.user_id == user.id) & (LibraryState.job_id == job_id) if not position: @@ -942,11 +560,11 @@ def storyboard_meta( Best-effort: for a very long/high-res source the sprite may not generate in time — then `available` is False and the editor shows a plain timeline.""" - job = _accessible_job(db, user, job_id) + job = accessible_job(db, user, job_id) asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None if asset is None or asset.status != "ready" or not asset.rel_path: raise refused(409, "download_not_ready") - meta = editmod.ensure_storyboard(asset, settings.download_root) + meta = editmod.ensure_storyboard(asset, settings.files_root) if meta is None: return {"available": False} db.commit() # persist storyboard_path if it was just generated @@ -963,11 +581,11 @@ def storyboard_meta( @router.get("/{job_id}/poster.jpg") def poster_image(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)): """The locally-generated poster frame for a download whose source had no thumbnail.""" - job = _accessible_job(db, user, job_id) + job = accessible_job(db, user, job_id) asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None if asset is None or not asset.poster_path: raise not_found("poster") - path = storage.safe_abs_path(settings.download_root, asset.poster_path) + path = storage.safe_abs_path(settings.files_root, asset.poster_path) if path is None: raise not_found("poster") return FileResponse(path, media_type="image/jpeg") @@ -977,11 +595,11 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session = def storyboard_image( job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ): - job = _accessible_job(db, user, job_id) + job = accessible_job(db, user, job_id) asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None if asset is None or not asset.storyboard_path: raise not_found("filmstrip") - path = storage.safe_abs_path(settings.download_root, asset.storyboard_path) + path = storage.safe_abs_path(settings.files_root, asset.storyboard_path) if path is None: raise not_found("filmstrip") return FileResponse(path, media_type="image/jpeg") @@ -1070,24 +688,6 @@ def unshare_download( return {"ok": True} -@router.delete("/shared/{job_id}") -def remove_shared_with_me( - job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) -) -> dict: - """Remove a download that was shared WITH this user from their 'Shared with me' list. Deletes - only the recipient's share grant — the owner's job and the physical file are untouched (this is - a per-user dismissal, not a delete).""" - row = db.execute( - select(DownloadShare).where( - DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id - ) - ).scalar_one_or_none() - if row is not None: - db.delete(row) - db.commit() - return {"ok": True, "removed": job_id} - - # --- recipients (for the internal "share with a user" picker) ------------------------------ @@ -1233,19 +833,20 @@ def admin_list_downloads( db.execute( select(DownloadJob) .order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()) - .limit(_MAX_LIST) + .limit(MAX_LIST) ) .scalars() .all() ) - assets = _assets_for(db, jobs) + assets = assets_for(db, jobs) emails = { u.id: u.email for u in db.execute(select(User).where(User.id.in_({j.user_id for j in jobs}))).scalars() } out = [] for j in jobs: - data = _serialize(j, assets.get(j.asset_id)) + data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None) + data["user_id"] = j.user_id data["user_email"] = emails.get(j.user_id) out.append(data) return out @@ -1259,24 +860,39 @@ def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_d select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK) ).one() total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar() - per_user = db.execute( - select(DownloadJob.user_id, func.count(func.distinct(DownloadJob.asset_id))) - .where(DownloadJob.asset_id.is_not(None)) - .group_by(DownloadJob.user_id) - ).all() - emails = { - u.id: u.email - for u in db.execute(select(User).where(User.id.in_([uid for uid, _ in per_user]))).scalars() + # EVERY user, not only those holding a file: a quota is set before anyone downloads. + emails: dict[int, str] = { + uid: email + for uid, email in db.execute(select(User.id, User.email).order_by(User.email)).tuples() } + uids = list(emails) + # What each one's Files lists: their finished files. + files: dict[int, int] = { + uid: n + for uid, n in db.execute( + select(DownloadJob.user_id, func.count()) + .where(DownloadJob.status == JobStatus.DONE) + .group_by(DownloadJob.user_id) + ).tuples() + } + + def row(uid: int) -> dict: + lim = quota.resolve(db, uid) + return { + "user_id": uid, + "email": emails[uid], + "footprint_bytes": quota.footprint(db, uid), + "files": files.get(uid, 0), + "max_bytes": lim.max_bytes, + "unlimited": lim.unlimited, + } + return { "files_on_disk": on_disk[0], "total_bytes": int(on_disk[1]), "total_assets": total_assets, "total_cap_bytes": sysconfig.get_int(db, "download_total_max_bytes"), - "per_user": [ - {"user_id": uid, "email": emails.get(uid), "footprint_bytes": quota.footprint(db, uid)} - for uid, _ in per_user - ], + "per_user": [row(uid) for uid in uids], } diff --git a/backend/app/routes/files.py b/backend/app/routes/files.py new file mode 100644 index 00000000..25be682b --- /dev/null +++ b/backend/app/routes/files.py @@ -0,0 +1,872 @@ +"""The Files library HTTP API — a user's files, the folders they sit in, and the file itself. + +A file in the library is a user's DONE job (`DownloadJob`) over its shared `MediaAsset`: the job is +the per-user layer (name, folder, kept, expiry, shares and links all hang off it), the asset the +bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk. + +In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above +the routes are shared with that module — serialization, the ownership locks, the upload spool — +because a download and a library file are one row at two moments. +""" + +import base64 +import hashlib +import os +import tempfile +import threading +from collections.abc import AsyncIterator, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import BinaryIO, Literal +from urllib.parse import unquote, urlparse + +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi.responses import FileResponse, Response +from pydantic import BaseModel, Field +from sqlalchemy import func, or_, select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app import audit, sysconfig +from app.audit import AuditAction +from app.auth import current_user +from app.collation import collated +from app.config import settings +from app.db import get_db, release_read_connection +from app.downloads import folders, preview, quota, service, storage +from app.downloads import upload as uploadmod +from app.downloads.status import JobStatus +from app.mailtext import lang_for_user +from app.models import ( + DISPLAY_TEXT_MAX, + DownloadJob, + DownloadShare, + FileFolder, + LibraryState, + MediaAsset, + User, + display_text, +) +from app.refusal import refused +from app.routes._common import capped_stream_to, not_found, owned_or_404 +from app.utils import iso, like_escape + +router = APIRouter(prefix="/api/files", tags=["files"]) + +# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's +# own cap; far beyond any real library). +MAX_LIST = 500 +# The longest a folder may keep a file, in days — ten years is "forever" in all but name. +_MAX_RETENTION_DAYS = 3650 +_HASH_CHUNK = 1 << 20 +# How many files the hash route reads at once: each holds a threadpool thread for the whole read. +_HASHERS = 2 +_HASHING = threading.BoundedSemaphore(_HASHERS) + + +# --- shared with routes.downloads ------------------------------------------------------- + + +def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict: + ready = asset is not None and asset.status == "ready" and bool(asset.rel_path) + return { + "id": job.id, + "status": job.status, + "progress": job.progress, + "speed_bps": job.speed_bps, + "eta_s": job.eta_s, + "phase": job.phase, + "error": job.error, + "error_code": job.error_code, + "error_detail": job.error_detail, + "queue_pos": job.queue_pos, + "created_at": iso(job.created_at), + "ready_at": iso(job.ready_at), + "source_kind": job.source_kind, + "source_ref": job.source_ref, + "source_url": service.reference_url(job, asset), + # A locally-generated poster to fall back to when the source had no thumbnail. + "poster_url": f"/api/downloads/{job.id}/poster.jpg" + if (asset is not None and asset.poster_path) + else None, + "profile_id": job.profile_id, + "spec": job.profile_snapshot, + "display_name": job.display_name, + "display_uploader": job.display_uploader, + "display_uploader_url": job.display_uploader_url, + "extra_links": job.extra_links or [], + "job_kind": job.job_kind, + "source_job_id": job.source_job_id, + "edit_spec": job.edit_spec, + "can_download": ready and job.status == JobStatus.DONE, + # The Files library: its folder, whether the user keeps it, and when it goes (NULL = never). + "folder_id": job.folder_id, + "kept": job.kept, + "expires_at": iso(job.expires_at), + "asset": None + if asset is None + else { + "id": asset.id, + "status": asset.status, + "title": asset.title, + "uploader": asset.uploader, + "upload_date": iso(asset.upload_date), + "duration_s": asset.duration_s, + "size_bytes": asset.size_bytes, + "width": asset.width, + "height": asset.height, + "container": asset.container, + # "video" | "audio" | "file" — the library gates play/edit on this (media only); a + # generic uploaded file is download-only. Null while still in flight (set at finalize). + "media_kind": asset.media_kind, + "uploader_url": asset.uploader_url, + "thumbnail_url": asset.thumbnail_url, + # "image" | "pdf" | "text" | null — what `/preview` shows (`downloads/preview.py`). + "preview": preview.kind(asset.media_kind, asset.container), + }, + } + + +def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]: + ids = {j.asset_id for j in jobs if j.asset_id} + if not ids: + return {} + rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars() + return {a.id: a for a in rows} + + +def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]: + """`user`'s resume point in each of `jobs` that has one (the library player's `library_states`).""" + ids = [j.id for j in jobs] + if not ids: + return {} + rows = db.execute( + select(LibraryState.job_id, LibraryState.position_seconds).where( + LibraryState.user_id == user.id, LibraryState.job_id.in_(ids) + ) + ).all() + return {job_id: pos for job_id, pos in rows} + + +def serialize_job(db: Session, job: DownloadJob) -> dict: + """Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step + every single-job handler shares.""" + asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None + return serialize(job, asset) + + +def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob: + """The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload) + or writes the status those decide on (pause). The job is re-read under a row lock, so two + overlapping requests for one job run one after the other and the second decides on the first's + committed result — never both releasing the same hold, and never a pause turning a job a cancel + just released back into a holder; either can free a file another job still holds. + + The asset is locked BEFORE the job: the order a finishing worker child takes them in + (`service.finalize_if_running`), so the two can never deadlock.""" + while True: + asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id + if asset_id is not None: + db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update()) + job = db.execute( + select(DownloadJob) + .where(DownloadJob.id == job_id) + .with_for_update() + .execution_options(populate_existing=True) + ).scalar_one_or_none() + if job is None: + raise not_found("download") + if job.asset_id == asset_id: + return job + db.rollback() # another request moved the job to another asset meanwhile: lock that one + + +def quota_detail(e: quota.QuotaExceeded) -> dict: + """Structured error the trilingual client localizes (reason key + numbers), instead of a + pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail().""" + return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current} + + +# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a +# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by +# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous +# caller could make the process write the whole per-file allowance to temp, per request (SEC-20 +# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency +# declared AFTER the auth one, is the only shape that closes it — the same one +# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for +# every route with a raised ceiling rather than leaving it to a comment like this one. +# +# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a +# Hungarian filename would otherwise be unsendable. +UPLOAD_FILENAME_HEADER = "x-upload-filename" +UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name" +# The folder the file goes in (its id); without it, the user's Uploads folder. +UPLOAD_FOLDER_HEADER = "x-upload-folder" + + +@dataclass(frozen=True) +class SpooledUpload: + """What the body dependency hands the handler: the bytes already on disk and their hash, plus + the names and the folder that came alongside them.""" + + path: Path + size: int + sha256: str + user_id: int + filename: str + display_name: str | None + folder_id: int | None + + +def _header_text(request: Request, name: str) -> str: + """One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the + caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's + own fallback names the file.""" + raw = request.headers.get(name, "") + try: + return unquote(raw, errors="strict").strip() + except (UnicodeDecodeError, ValueError): + # silent-ok: a mis-encoded header is the caller's own malformed request, and this one is + # cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on + # a label; logging it would hand any client a free log line per request. + return "" + + +def _upload_folder(db: Session, user_id: int, raw: str) -> int | None: + """The folder id an upload asked for (`UPLOAD_FOLDER_HEADER`), checked; None when it asked for + none.""" + if not raw.strip(): + return None + try: + return folders.upload_target(db, user_id, int(raw), hold=False).id + except ValueError as e: + raise refused(409, "file_folder_gone") from e + except folders.FolderError as e: + raise refused(409, e.reason) from e + + +class _HashingSink: + """The upload's file, hashing the bytes on their way in: the hash costs no second read.""" + + def __init__(self, out: BinaryIO) -> None: + self.out = out + self.hash = hashlib.sha256() + + def write(self, data: bytes, /) -> int: + self.hash.update(data) + return self.out.write(data) + + +async def spool_upload( + request: Request, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> AsyncIterator[SpooledUpload]: + """Stream the request body to a staging temp file, enforcing the per-file cap as it arrives. + + `user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so + naming `current_user` here makes the auth check a precondition of reading the body at all — a + stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest + MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit, + empty, quota, probe or move error, a client that vanished mid-upload — it is removed.""" + cap = sysconfig.get_int(db, "download_upload_max_bytes") + user_id = user.id + # A folder the file may not go in is refused before the transfer, not after it. + folder_id = _upload_folder(db, user_id, request.headers.get(UPLOAD_FOLDER_HEADER, "")) + # Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection` + # for why, and for the precondition it enforces. Everything this dependency still needs is + # already an int or a header in hand: touching an ORM attribute below would reload `user` and + # check a connection straight back out. + release_read_connection(db) + staging = Path(settings.files_root) / ".staging" / "uploads" + staging.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(staging)) + tmp_path = Path(tmp) + try: + with os.fdopen(fd, "wb") as out: + sink = _HashingSink(out) + size = await capped_stream_to( + request, + sink, + cap, + HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}), + ) + yield SpooledUpload( + path=tmp_path, + size=size, + sha256=sink.hash.hexdigest(), + user_id=user_id, + filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload", + display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)), + folder_id=folder_id, + ) + finally: + tmp_path.unlink(missing_ok=True) + + +def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob: + """A file the caller may read: their own, or one shared with them.""" + job = db.get(DownloadJob, job_id) + if job is None: + raise not_found("download") + if job.user_id == user.id: + return job + shared = db.execute( + select(DownloadShare.id).where( + DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id + ) + ).first() + if shared is None: + raise not_found("download") + return job + + +# --- folders ------------------------------------------------------------------------------- + + +def _serialize_folder(folder: FileFolder, count: int, size: int) -> dict: + return { + "id": folder.id, + "kind": folder.kind, + # A source folder's name is the client's to render from its key, in the reader's language. + "source_key": folder.source_key, + "name": folder.name, + "retention_days": folder.retention_days, + "count": count, + "size_bytes": size, + "created_at": iso(folder.created_at), + } + + +@router.get("/folders") +def list_folders(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]: + """The caller's folders with how many files each holds and their size. A source folder its + files have all left stays (count 0): the client decides whether to show it.""" + done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE) + rows = db.execute( + select( + FileFolder, + func.count(DownloadJob.id), + func.coalesce(func.sum(MediaAsset.size_bytes), 0), + ) + .outerjoin(DownloadJob, done) + .outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id) + .where(FileFolder.user_id == user.id) + .group_by(FileFolder.id) + .order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id) + ).all() + return [_serialize_folder(f, count, int(size)) for f, count, size in rows] + + +class FolderIn(BaseModel): + name: str = Field(min_length=1, max_length=DISPLAY_TEXT_MAX) + + +def _folder_name(raw: str) -> str: + name = display_text(raw) + if not name: + raise refused(400, "file_folder_name_required") + return name + + +@router.post("/folders") +def create_folder( + body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name)) + try: + with db.begin_nested(): + db.add(folder) + db.flush() + except IntegrityError as e: # the unique (user, lower(name)): one folder per name + raise refused(409, "file_folder_name_taken") from e + audit.record( + db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id + ) + db.commit() + return _serialize_folder(folder, 0, 0) + + +class UpdateFolderIn(BaseModel): + # Presence (model_fields_set) decides what is touched; retention_days null = keep forever. + name: str | None = Field(default=None, min_length=1, max_length=DISPLAY_TEXT_MAX) + retention_days: int | None = Field(default=None, ge=1, le=_MAX_RETENTION_DAYS) + + +@router.patch("/folders/{folder_id}") +def update_folder( + folder_id: int, + body: UpdateFolderIn, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + """Rename one of the caller's own folders (a source folder's name is its origin's), and/or set + how long any of their folders keeps a file (`folders.set_retention`).""" + folder = owned_or_404(db, FileFolder, folder_id, user, what="folder") + sent = body.model_fields_set + if "name" in sent: + if folder.kind != folders.FolderKind.OWN: + raise refused(409, "file_folder_not_own") + folder.name = _folder_name(body.name or "") + try: + with db.begin_nested(): + db.flush() + except IntegrityError as e: + raise refused(409, "file_folder_name_taken") from e + audit.record( + db, + user.id, + AuditAction.FILE_FOLDER_RENAME, + target_type="file_folder", + target_id=folder.id, + ) + if "retention_days" in sent: + # The value replaced is the one read under the folder's lock, not the one read above it. + try: + before = folders.set_retention(db, folder, body.retention_days) + except folders.FolderError as e: + raise refused(409, e.reason) from e + if before != body.retention_days: + audit.record( + db, + user.id, + AuditAction.FILE_FOLDER_RETENTION, + target_type="file_folder", + target_id=folder.id, + before={"retention_days": before}, + after={"retention_days": body.retention_days}, + ) + db.commit() + count, size = _folder_totals(db, folder.id) + return _serialize_folder(folder, count, size) + + +def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]: + count, size = db.execute( + select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0)) + .select_from(DownloadJob) + .outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id) + .where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE) + ).one() + return count, int(size) + + +@router.delete("/folders/{folder_id}") +def delete_folder( + folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + """Delete one of the caller's own folders; its files go back to their source folders.""" + folder = owned_or_404(db, FileFolder, folder_id, user, what="folder") + try: + moved = folders.delete_own(db, folder) + except folders.FolderError as e: + raise refused(409, e.reason) from e + audit.record( + db, + user.id, + AuditAction.FILE_FOLDER_DELETE, + target_type="file_folder", + target_id=folder_id, + after={"files_moved": moved}, + ) + db.commit() + return {"ok": True, "deleted": folder_id, "files_moved": moved} + + +@router.get("/usage") +def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict: + """The caller's storage quota, plus the per-file upload cap. + + The cap rides here rather than in `quota.usage` because it is not a quota value — it is a + `sysconfig` knob about one route — and this is the route layer, where composing the two is the + job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side + caps answer too late to be pleasant (the edge ceiling refuses the request outright, see + `bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the + limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the + boundary: the caps stay where they are and still refuse a client that ignores this. + """ + return { + **quota.usage(db, user.id), + "upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"), + } + + +@router.delete("/shared/{job_id}") +def remove_shared_with_me( + job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + """Remove a file that was shared WITH this user from their 'Shared with me' list. Deletes + only the recipient's share grant — the owner's job and the physical file are untouched (this is + a per-user dismissal, not a delete).""" + row = db.execute( + select(DownloadShare).where( + DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id + ) + ).scalar_one_or_none() + if row is not None: + db.delete(row) + db.commit() + return {"ok": True, "removed": job_id} + + +# --- files --------------------------------------------------------------------------------- + + +@router.get("") +def list_files( + folder_id: int | None = None, + q: str = Query("", max_length=200), + sort: Literal["added", "name", "size"] = "added", + order: Literal["asc", "desc"] = "desc", + scope: Literal["mine", "shared"] = "mine", + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> list[dict]: + """The caller's done files (`scope=mine`, optionally one folder's), or the ones shared with + them (`scope=shared`); searched by name and channel, sorted by when they were added, by name + (in the reader's alphabet) or by size.""" + stmt = ( + select(DownloadJob) + .join(MediaAsset, MediaAsset.id == DownloadJob.asset_id) + .where(DownloadJob.status == JobStatus.DONE) + ) + shared = scope == "shared" + if shared: + stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where( + DownloadShare.to_user_id == user.id + ) + else: + stmt = stmt.where(DownloadJob.user_id == user.id) + if folder_id is not None: + stmt = stmt.where(DownloadJob.folder_id == folder_id) + name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "") + term = q.strip() + if term: + like = f"%{like_escape(term)}%" + stmt = stmt.where( + or_( + name.ilike(like), + func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like), + ) + ) + # A file is added to its owner's list when it is done, not when it was asked for (a download that + # failed for days and then went through is new); one shared with the caller, when it was shared. + key = { + "added": DownloadShare.created_at if shared else DownloadJob.ready_at, + "name": collated(name, lang_for_user(user)), + "size": MediaAsset.size_bytes, + }[sort] + first = key.asc() if order == "asc" else key.desc() + tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc() + jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all() + assets = assets_for(db, jobs) + positions = positions_for(db, user, jobs) + shared_at = _shared_at(db, user, jobs) if shared else {} + sources = source_keys(db, jobs) + return [ + { + **serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None), + "position_seconds": positions.get(j.id, 0), + "shared": shared, + "shared_at": iso(shared_at.get(j.id)), + "source_key": sources[j.id], + } + for j in jobs + ] + + +def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]: + """Where each file came from (its source folder's key), for the list's Source column, the + "back to its source folder" move and the download tray's site: a file in a source folder has its folder's key; one moved into + an own folder (or someone else's, shared) is traced back by `folders.source_key_for`.""" + ids = {j.folder_id for j in jobs if j.folder_id is not None} + by_id = ( + {f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()} + if ids + else {} + ) + out: dict[int, str] = {} + for j in jobs: + folder = by_id.get(j.folder_id) if j.folder_id is not None else None + if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key: + out[j.id] = folder.source_key + else: + out[j.id] = folders.source_key_for(db, j) + return out + + +def _shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]: + rows = db.execute( + select(DownloadShare.job_id, DownloadShare.created_at).where( + DownloadShare.to_user_id == user.id, + DownloadShare.job_id.in_([j.id for j in jobs]), + ) + ).all() + return {job_id: at for job_id, at in rows} + + +_MAX_EXTRA_LINKS = 8 + + +def _clean_url(raw) -> str | None: + """Accept only a plausible http(s) URL (these get rendered as clickable links + embedded in a + public watch page); anything else is dropped rather than trusted.""" + if not isinstance(raw, str): + return None + url = raw.strip() + if not url: + return None + if not url.startswith(("http://", "https://")): + return None + # `urlparse` raises on a malformed authority; this function already has a "not a plausible URL" + # answer, so the typo belongs in it. + try: + parsed = urlparse(url) + # silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL" + # answer, the caller already has one, and any line here would carry the URL itself. + except ValueError: + return None + if not parsed.netloc: + return None + return url[:2048] + + +class UpdateFileIn(BaseModel): + # Presence (model_fields_set) decides what is touched; a null/"" text falls back to the system + # value (the title, the channel). All display-only: the file on disk keeps its own name. + name: str | None = None + uploader: str | None = None + uploader_url: str | None = None + extra_links: list | None = None + folder_id: int | None = None + kept: bool | None = None + + +@router.patch("/{job_id}") +def update_file( + job_id: int, + body: UpdateFileIn, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + """Rename a file and its channel, set its reference links (all display-only: the file on disk + keeps its own name), move it to another folder (`folders.move`), or keep it until deleted / + hand it back to its folder's retention.""" + job = owned_or_404(db, DownloadJob, job_id, user, what="download") + sent = body.model_fields_set + if "name" in sent: + job.display_name = display_text(body.name) + if "uploader" in sent: + job.display_uploader = display_text(body.uploader) + if "uploader_url" in sent: + job.display_uploader_url = _clean_url(body.uploader_url) + if "extra_links" in sent: + raw = body.extra_links or [] + cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS] + job.extra_links = cleaned or None + if "folder_id" in sent: + if body.folder_id is None: + raise refused(400, "file_folder_gone") + before = job.folder_id + try: + folders.move(db, job, folders.own_folder(db, user.id, body.folder_id)) + except folders.FolderError as e: + raise refused(409, e.reason) from e + if job.folder_id != before: + audit.record( + db, + user.id, + AuditAction.FILE_MOVE, + target_type="file", + target_id=job.id, + before={"folder_id": before}, + after={"folder_id": job.folder_id}, + ) + if "kept" in sent and body.kept is not None and body.kept != job.kept: + folders.set_kept(db, job, body.kept) + action = AuditAction.FILE_KEEP if body.kept else AuditAction.FILE_UNKEEP + audit.record(db, user.id, action, target_type="file", target_id=job.id) + db.commit() + return serialize_job(db, job) + + +@router.delete("/{job_id}") +def delete_file( + job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + """Delete a file from the caller's library; the bytes go with their last holder.""" + job = own_job_for_update(db, user, job_id) + audit.record(db, user.id, AuditAction.FILE_DELETE, target_type="file", target_id=job_id) + service.delete(db, job) # commits, the audit row with it + return {"ok": True, "deleted": job_id} + + +@router.post("/upload") +def upload_file( + upload: SpooledUpload = Depends(spool_upload), + db: Session = Depends(get_db), +) -> dict: + """Put a local file in the caller's library — in the folder `UPLOAD_FOLDER_HEADER` names, else + their Uploads folder. Any file type; media gets the player, a generic file is download-only. + The body IS the file (see the header note above `UPLOAD_FILENAME_HEADER`); `spool_upload` has + authenticated the caller, checked the folder and streamed it to disk already. + + A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s + plus sync DB writes — so it must NOT run on the event loop.""" + return serialize_job(db, ingest_spooled(db, upload)) + + +def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob: + """The spooled upload into the library (`upload.ingest_upload`, which commits its own rows), + its refusals answered as HTTP.""" + if upload.size == 0: + raise HTTPException(status_code=400, detail={"code": "upload_empty"}) + try: + job = uploadmod.ingest_upload( + db, + upload.user_id, + filename=upload.filename, + src_path=upload.path, + size=upload.size, + display_name=upload.display_name, + sha256=upload.sha256, + folder_id=upload.folder_id, + ) + except quota.QuotaExceeded as e: + raise HTTPException(status_code=422, detail=quota_detail(e)) from e + except folders.FolderError as e: # the folder went while the file was on its way + raise refused(409, e.reason) from e + return job + + +def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]: + asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None + if asset is None or asset.status != "ready" or not asset.rel_path: + raise refused(409, "download_not_ready") + path = storage.safe_abs_path(settings.files_root, asset.rel_path) + if path is None: + raise not_found("file") + return asset, path + + +def serve_file(db: Session, job: DownloadJob) -> FileResponse: + """The file of `job` (the caller may read it), range-aware, named by the user's display name. + + The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list + (`storage.served_media_type`), NEVER guessed from the file's own extension. A generic 'file' (and + any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html") + returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on + this origin. Recognised media is served with its real type so the in-app player streams it. Both + are range-aware (FileResponse → 206). A file whose hash is known says so (`Repr-Digest`, RFC + 9530: the digest of the whole file, whichever range is asked for).""" + asset, path = _ready_file(db, job) + filename = storage.download_filename( + job.display_name or asset.title or asset.source_ref, asset.container, path + ) + headers = {} + if asset.sha256: + digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode() + headers["Repr-Digest"] = f"sha-256=:{digest}:" + asset.last_access_at = datetime.now(timezone.utc) # the handler commits it + media_type = storage.served_media_type(asset.media_kind, asset.container) + if media_type is None: + return FileResponse( + path, + filename=filename, + media_type="application/octet-stream", + content_disposition_type="attachment", + headers=headers, + ) + return FileResponse(path, filename=filename, media_type=media_type, headers=headers) + + +@router.get("/{job_id}/file") +def get_file(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)): + response = serve_file(db, accessible_job(db, user, job_id)) + db.commit() + return response + + +# A preview's own headers. `sandbox` leaves the answer no script and an opaque origin, whatever the +# file turns out to hold; `frame-ancestors 'self'` lets the app's viewer frame a PDF (every other +# answer keeps the app-wide 'none', main.py), with X-Frame-Options saying the same for old engines. +PREVIEW_HEADERS = { + "Content-Security-Policy": "sandbox; frame-ancestors 'self'", + "X-Frame-Options": "SAMEORIGIN", +} +# Set on a text preview cut at `preview.TEXT_PREVIEW_BYTES`, so the viewer can say it shows the start. +TRUNCATED_HEADER = "X-Preview-Truncated" + + +@router.get("/{job_id}/preview") +def preview_file( + job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) +) -> Response: + """The file shown in the library's viewer: an image or a PDF inline, a text format as plain text + (its first `TEXT_PREVIEW_BYTES`). The type comes from `downloads/preview.py`'s allow-list, so + an HTML or SVG upload arrives as text, never as a page. Anything else: 415 `file_no_preview`.""" + job = accessible_job(db, user, job_id) + asset, path = _ready_file(db, job) + kind = preview.kind(asset.media_kind, asset.container) + if kind is None: + raise refused(415, "file_no_preview") + headers = dict(PREVIEW_HEADERS) + if kind == "text": + with path.open("rb") as f: + head = f.read(preview.TEXT_PREVIEW_BYTES + 1) + if len(head) > preview.TEXT_PREVIEW_BYTES: + head = head[: preview.TEXT_PREVIEW_BYTES] + headers[TRUNCATED_HEADER] = "1" + text = preview.text_utf8(head) + if text is None: + raise refused(415, "file_no_preview") + asset.last_access_at = datetime.now(timezone.utc) + db.commit() + return Response(text, media_type=preview.media_type(kind, asset.container), headers=headers) + filename = storage.download_filename( + job.display_name or asset.title or asset.source_ref, asset.container, path + ) + asset.last_access_at = datetime.now(timezone.utc) + db.commit() + return FileResponse( + path, + filename=filename, + media_type=preview.media_type(kind, asset.container), + content_disposition_type="inline", + headers=headers, + ) + + +@router.get("/{job_id}/sha256") +def get_sha256( + job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + """The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder. + A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool — and + every other sync route shares that pool, so only `_HASHERS` files are hashed at a time (a caller + past that is told to come back), and none of them holds a database connection while it reads.""" + asset, path = _ready_file(db, accessible_job(db, user, job_id)) + if asset.sha256: + return {"sha256": asset.sha256} + asset_id = asset.id + if not _HASHING.acquire(blocking=False): + raise refused(429, "file_hash_busy") + try: + release_read_connection(db) + h = hashlib.sha256() + with path.open("rb") as f: + while chunk := f.read(_HASH_CHUNK): + h.update(chunk) + except OSError as e: + raise not_found("file") from e + finally: + _HASHING.release() + digest = h.hexdigest() + db.execute( + update(MediaAsset) + .where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None)) + .values(sha256=digest) + ) + db.commit() + return {"sha256": digest} diff --git a/backend/app/routes/public.py b/backend/app/routes/public.py index 42099a93..ddb6e8da 100644 --- a/backend/app/routes/public.py +++ b/backend/app/routes/public.py @@ -73,7 +73,7 @@ def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAs raise refused(404, "video_gone") # Verify the file is actually on disk here (not just flagged ready), so the watch page never # loads a player for a missing file — meta and file agree. - path = storage.safe_abs_path(settings.download_root, asset.rel_path) + path = storage.safe_abs_path(settings.files_root, asset.rel_path) if path is None: raise refused(404, "video_gone") return job, asset @@ -158,7 +158,7 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)): if link.password_hash and not linksmod.check_grant(token, g, link.password_version): raise refused(403, "share_password_required") asset = _ready_asset(db, link) - path = storage.safe_abs_path(settings.download_root, asset.rel_path) + path = storage.safe_abs_path(settings.files_root, asset.rel_path) if path is None: raise refused(404, "video_gone") @@ -194,7 +194,7 @@ def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db) job, asset = _ready_target(db, link) if not asset.poster_path: raise not_found("poster") - path = storage.safe_abs_path(settings.download_root, asset.poster_path) + path = storage.safe_abs_path(settings.files_root, asset.poster_path) if path is None: raise not_found("poster") return FileResponse(path, media_type="image/jpeg") diff --git a/backend/app/worker.py b/backend/app/worker.py index dbff8e35..b8180456 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -37,7 +37,7 @@ from app import sysconfig from app.config import settings from app.db import SessionLocal from app.downloads import edit as editmod -from app.downloads import credentials, formats, quota, service +from app.downloads import credentials, formats, quota, service, storage from app.downloads import errors as dlerrors from app.downloads.jobdb import ( _edit_staging_dir, @@ -710,12 +710,18 @@ def main() -> None: # `log_events`, so on the deploy that first adds the table the worker may start before the API # has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks. logsink.start() + problem = storage.files_root_problem(settings.files_root) + if problem: + # Refuse, loudly: downloading into a root that is not the volume loses every file with the + # container. The queue waits until the mount is fixed (see docs/self-hosting.md). + log.critical("The files library cannot be written: %s. The worker will not start.", problem) + raise SystemExit(1) _recover_orphans() global _target _target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer threads: dict[int, threading.Thread] = {} level = _sync_knobs(threads, None) - log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.download_root) + log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.files_root) while not _stop.wait(_KNOB_SECONDS): level = _sync_knobs(threads, level) log.info("Download worker stopping…") diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index d4d142af..f2e1b127 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -254,20 +254,24 @@ def as_user(db, override_dep, _configured): @pytest.fixture def upload_headers(): - """Build the headers `POST /api/downloads/upload` takes (the file is the raw body). + """Build the headers the upload routes take (the file is the raw body). Shared, because two test modules drive that route and a second copy of this is exactly the drift that lets the wire shape change in one place only. The body IS the file now; the names ride as percent-encoded headers (`quote` mirrors the client's `encodeURIComponent`, since headers are latin-1 and a Hungarian filename would not survive raw).""" - def _headers(filename: str, display_name: str | None = None) -> dict[str, str]: + def _headers( + filename: str, display_name: str | None = None, folder: int | str | None = None + ) -> dict[str, str]: headers = { "content-type": "application/octet-stream", "x-upload-filename": quote(filename), } if display_name is not None: headers["x-upload-display-name"] = quote(display_name) + if folder is not None: + headers["x-upload-folder"] = str(folder) return headers return _headers diff --git a/backend/tests/test_body_limit.py b/backend/tests/test_body_limit.py index 4d55c4e7..dc308d01 100644 --- a/backend/tests/test_body_limit.py +++ b/backend/tests/test_body_limit.py @@ -221,7 +221,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes(): CAPPED_BODY_SITES = { ("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"), ("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"), - ("downloads.py", "spool_upload", "capped_stream_to", "cap"), + ("files.py", "spool_upload", "capped_stream_to", "cap"), } @@ -367,7 +367,7 @@ def test_every_body_reading_site_is_known_and_fits_under_its_ceiling(): assert ("POST", "/api/debug-sessions/{sid}/screenshots") in bodylimit.ROUTE_MAX_BODY_BYTES # The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather # than being a constant to compare against — that pairing is its own test below. - assert ("POST", "/api/downloads/upload") in bodylimit.ROUTE_MAX_BODY_BYTES + assert ("POST", "/api/files/upload") in bodylimit.ROUTE_MAX_BODY_BYTES def test_release_read_connection_ends_the_transaction_it_was_called_for(db): @@ -467,7 +467,7 @@ def test_every_body_reading_site_hands_its_pooled_connection_back_first(): assert not missing, ( "a route reads its own request body while still inside its read transaction, pinning a " f"pooled connection for the whole transfer: {missing}. Call " - "db.release_read_connection(db) before the await — see routes/downloads.py:spool_upload." + "db.release_read_connection(db) before the await — see routes/files.py:spool_upload." ) @@ -487,7 +487,7 @@ def test_the_upload_ceiling_and_the_admin_knob_cannot_drift_apart(): `errors.tooLarge` exists and why the dialog checks the file against `upload_max_bytes` first. """ spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes") - edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/downloads/upload")] + edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/files/upload")] assert spec.max == edge @@ -540,7 +540,7 @@ def test_no_route_with_a_raised_ceiling_declares_a_body_parameter(method, templa "its dependencies, so FastAPI reads it before the first dependency runs and the raised " "ceiling is all that stands between an anonymous caller and the whole allowance. Read the " "body yourself, from a dependency declared AFTER the auth one — see " - "routes/downloads.py:spool_upload." + "routes/files.py:spool_upload." ) diff --git a/backend/tests/test_clip.py b/backend/tests/test_clip.py index 8566ca0d..a0af274c 100644 --- a/backend/tests/test_clip.py +++ b/backend/tests/test_clip.py @@ -374,7 +374,7 @@ def test_a_clip_is_not_redownloadable(db, make_user, as_user, yt_video): c = as_user(make_user()) job = _clip(c, source="youtube", ref=yt_video).json() row = db.get(DownloadJob, job["id"]) - row.status = "done" + row.status, row.ready_at = "done", row.created_at db.commit() r = c.post(f"/api/downloads/{job['id']}/redownload") assert (r.status_code, r.json()["detail"]["reason"]) == (409, "not_redownloadable") @@ -395,7 +395,7 @@ def test_a_clip_job_reaches_the_clip_entry(monkeypatch): def worker(db, tmp_path, monkeypatch): """The worker path with its external programs replaced: ffmpeg writes the file it was given, the probe answers what the test says, the phone-safe step and the poster pass through.""" - monkeypatch.setattr(settings, "download_root", str(tmp_path / "root")) + monkeypatch.setattr(settings, "files_root", str(tmp_path / "root")) seen: dict = {} def ffmpeg(cmd, *_a, **_k): diff --git a/backend/tests/test_download_errors.py b/backend/tests/test_download_errors.py index 92738261..8a1535b9 100644 --- a/backend/tests/test_download_errors.py +++ b/backend/tests/test_download_errors.py @@ -5,6 +5,14 @@ from app.downloads import errors as dlerrors from app.downloads.errors import classify +# Verbatim from dev job 128 (2026-10-09, yt-dlp 2026.07.04); the same link downloads under 2026.08.19. +_TIKTOK_UNEXPECTED = ( + "ERROR: [TikTok] 7418778475890232609: Unexpected response from webpage request; please report " + "this issue on https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue " + "template. Confirm you are on the latest version using yt-dlp -U" +) + + def test_real_ytdlp_messages_map_to_categories(): # Verbatim-shaped yt-dlp / ffmpeg output → the category the UI explains. cases = { @@ -32,6 +40,7 @@ def test_real_ytdlp_messages_map_to_categories(): "The source file is missing.": "source_missing", "ERROR: unable to download video data: HTTP Error 403: Forbidden": "network", "ERROR: Unable to download webpage: ": "network", + _TIKTOK_UNEXPECTED: "site_changed", } for msg, expected in cases.items(): assert classify(msg) == expected, f"{msg!r} → {classify(msg)!r}, expected {expected!r}" @@ -211,13 +220,25 @@ def test_our_post_without_a_video_verdict_names_the_group_link(): def test_a_catalog_entry_never_leaves_its_host(): - # "Cannot parse data" is yt-dlp's wording for a broken extractor on every site. + # "Cannot parse data" is yt-dlp's wording for a broken extractor on every site: elsewhere it is + # only the category — the site changed under the extractor. other = _FB_PARSE.replace("[facebook]", "[vimeo]") - assert dlerrors.diagnose(other, url="https://vimeo.com/1") == ("unknown", None) + assert dlerrors.diagnose(other, url="https://vimeo.com/1") == ("site_changed", None) # The Facebook text for a link that is not Facebook's (a look-alike host) is not the Facebook wall. - assert dlerrors.diagnose(_FB_PARSE, url="https://facebook.com.evil/reel/1") == ("unknown", None) + assert dlerrors.diagnose(_FB_PARSE, url="https://facebook.com.evil/reel/1") == ( + "site_changed", + None, + ) # No link known: the category only. - assert dlerrors.diagnose(_FB_PARSE) == ("unknown", None) + assert dlerrors.diagnose(_FB_PARSE) == ("site_changed", None) + + +def test_a_bug_report_request_keeps_the_more_specific_category(): + # yt-dlp asks for the bug report on these too; what they say about the media wins. + assert classify(_FB_NO_FORMATS) == "format_unavailable" + assert classify("ERROR: [x] 1: Unable to extract title; please report this issue") == ( + "format_unavailable" + ) def test_diagnose_falls_back_to_the_category(): diff --git a/backend/tests/test_download_failure_catalog.py b/backend/tests/test_download_failure_catalog.py index ed585e64..10ea902d 100644 --- a/backend/tests/test_download_failure_catalog.py +++ b/backend/tests/test_download_failure_catalog.py @@ -80,7 +80,7 @@ def _asset_and_job(db, user, *, sig, asset_status, job_status, error=None): def _fail_the_child(monkeypatch, tmp_path, *, cookie: bool): - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) cookiefile = tmp_path / "cookies.txt" cookiefile.write_text("# Netscape HTTP Cookie File\n") @@ -230,7 +230,7 @@ def test_a_download_yt_dlp_gave_no_duration_stores_the_files_duration(db, monkey # row lost its trim action. Through the real finalize, with a real 3-second file. u = _user(db, "fb-duration@example.com") a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running") - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs): @@ -350,7 +350,7 @@ def test_a_signed_in_facebook_download_is_titled_with_its_posts_text(db, monkeyp # file name and the share preview all said "Video". The post's text is in the description. u = _user(db, "fb-title@example.com") a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running") - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs): @@ -401,7 +401,7 @@ def test_an_mp3_download_is_stored_as_an_mp3(db, monkeypatch, tmp_path): # video/mp4, which iPad's WebKit will not play. It stays an .mp3 (served as audio/mpeg). u = _user(db, "fb-mp3@example.com") a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running") - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs): @@ -430,7 +430,7 @@ def test_a_video_download_is_still_made_browser_playable(db, monkeypatch, tmp_pa a, j = _asset_and_job( db, u, sig=f"mkv-{mode}", asset_status="downloading", job_status="running" ) - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs): diff --git a/backend/tests/test_download_job_locks.py b/backend/tests/test_download_job_locks.py index a4f13d6c..ada7e477 100644 --- a/backend/tests/test_download_job_locks.py +++ b/backend/tests/test_download_job_locks.py @@ -5,7 +5,7 @@ unlocked read: two overlapping requests for one job (a double-click, a second ta gateway timeout) both saw it holding its asset and both released the hold. The count then reaches 0 while another job still holds the asset, and GC or the ready-file delete removes it under that job. A pause racing a cancel did the same one step later: its unlocked write turned the released job back -into a holder, which a later delete releases again. `_own_job_for_update` makes the second request +into a holder, which a later delete releases again. `own_job_for_update` makes the second request wait and decide on the first one's committed result. """ @@ -18,12 +18,8 @@ from sqlalchemy import select from app.db import SessionLocal from app.downloads import service from app.models import DownloadJob, MediaAsset, User -from app.routes.downloads import ( - _own_job_for_update, - cancel_download, - delete_download, - pause_download, -) +from app.routes.downloads import cancel_download, pause_download +from app.routes.files import delete_file, own_job_for_update _REF = "https://example.com/video" @@ -58,7 +54,7 @@ def _asset_held_by_two_jobs(db, status="error"): ("status", "transition", "route", "left"), [ ("error", service.cancel, cancel_download, "canceled"), - ("error", service.delete, delete_download, None), + ("error", service.delete, delete_file, None), ("queued", service.cancel, pause_download, "canceled"), ], ) @@ -80,7 +76,7 @@ def test_overlapping_requests_for_one_job_release_its_hold_once( finished.set() try: - locked = _own_job_for_update(first, user, job_id) + locked = own_job_for_update(first, user, job_id) other = threading.Thread(target=the_other_request) other.start() # The second request waits for the first one's locks instead of deciding on the same row. diff --git a/backend/tests/test_download_limits.py b/backend/tests/test_download_limits.py index d2dfdc32..e2d97fe9 100644 --- a/backend/tests/test_download_limits.py +++ b/backend/tests/test_download_limits.py @@ -188,7 +188,7 @@ def test_list_downloads_is_capped(db, monkeypatch): _job(db, u.id, queue_pos=i, status="done") db.commit() - monkeypatch.setattr(dl, "_MAX_LIST", 2) + monkeypatch.setattr(dl, "MAX_LIST", 2) rows = dl.list_downloads(user=u, db=db) assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk @@ -259,7 +259,7 @@ def test_a_refusal_for_one_user_does_not_poison_the_shared_asset(db, monkeypatch that shared row — correct for an upstream failure, wrong for "YOUR storage is full", which `worker._process_job` would then hand verbatim to someone with an empty library. """ - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) over, other = User(email="over@example.com"), User(email="other@example.com") db.add_all([over, other]) db.flush() @@ -306,7 +306,7 @@ def test_a_download_over_the_space_left_says_so_and_releases_the_shared_asset( # yt-dlp refuses a file over `max_filesize` by saying so and returning with no file (pinned in # test_formats). That is the owner's storage, not a processing fault: their refusal, in words # true of a cap with space still left, and the shared asset goes back for the next job. - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) u = User(email="nearcap@example.com") db.add(u) db.flush() @@ -352,7 +352,7 @@ def test_a_download_that_outgrows_the_space_left_unannounced_is_refused_and_not_ # yt-dlp's `max_filesize` only stops a plain HTTP file that announces its size: YouTube's # fragmented formats never meet it (measured on localdev: 718 MB fetched with 1 MB left). The # file it produced is held to what is left by its real size, as a clip is. - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) u = User(email="unannounced@example.com") db.add(u) db.flush() diff --git a/backend/tests/test_download_transitions.py b/backend/tests/test_download_transitions.py index 67aab12f..212e22d6 100644 --- a/backend/tests/test_download_transitions.py +++ b/backend/tests/test_download_transitions.py @@ -77,7 +77,7 @@ def test_resume_resets_a_failed_shared_asset_to_pending(db, as_user): def test_delete_releases_and_removes_the_job(db, as_user): u, asset, job = _seed(db, job_status="done", asset_status="ready", ref_count=1, rel_path=None) - r = as_user(u).request("DELETE", f"/api/downloads/{job.id}") + r = as_user(u).request("DELETE", f"/api/files/{job.id}") assert r.status_code == 200, r.text db.expire_all() assert db.get(DownloadJob, job.id) is None diff --git a/backend/tests/test_edit_cap.py b/backend/tests/test_edit_cap.py index 8f78c28d..fbc52efa 100644 --- a/backend/tests/test_edit_cap.py +++ b/backend/tests/test_edit_cap.py @@ -21,7 +21,7 @@ pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg n def test_an_editor_clip_larger_than_what_is_left_is_refused_and_not_placed( db, make_user, tmp_path, monkeypatch ): - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) src = tmp_path / "in.mp4" subprocess.run( [ diff --git a/backend/tests/test_failure_lines.py b/backend/tests/test_failure_lines.py index 431fc8d8..3ade4eee 100644 --- a/backend/tests/test_failure_lines.py +++ b/backend/tests/test_failure_lines.py @@ -258,7 +258,7 @@ def test_an_edit_that_crashes_is_said_once(sink, finalized, monkeypatch, tmp_pat """The line `_process_edit`'s handler used to log before `_fail_edit` logged it again.""" (tmp_path / "src.mp4").write_bytes(b"x") monkeypatch.setattr(pipeline, "SessionLocal", _EditDb) - monkeypatch.setattr(pipeline.settings, "download_root", str(tmp_path)) + monkeypatch.setattr(pipeline.settings, "files_root", str(tmp_path)) monkeypatch.setattr(pipeline, "_edit_staging_dir", lambda asset_id: tmp_path / "staging") def odd_spec(*a, **k): @@ -300,7 +300,7 @@ def test_a_failed_download_line_does_not_put_back_what_the_adapter_took_out(sink RuntimeError("dictionary changed size during iteration"), RecursionError("maximum recursion depth exceeded"), NotImplementedError(), - FileNotFoundError(2, "No such file", "/downloads/Some_Channel/Season_2024/A_Private_Title"), + FileNotFoundError(2, "No such file", "/files/Some_Channel/Season_2024/A_Private_Title"), ], ids=lambda e: type(e).__name__, ) @@ -499,11 +499,9 @@ def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path): @pytest.mark.parametrize( "err", [ - subprocess.TimeoutExpired( - ["ffmpeg", "-i", "/downloads/Some_Channel/A_Private_Title.mp4"], 30 - ), - FileNotFoundError(2, "No such file", "/downloads/Some_Channel/A_Private_Title.jpg"), - subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/downloads/Some_Channel/x.mp4"]), + subprocess.TimeoutExpired(["ffmpeg", "-i", "/files/Some_Channel/A_Private_Title.mp4"], 30), + FileNotFoundError(2, "No such file", "/files/Some_Channel/A_Private_Title.jpg"), + subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/files/Some_Channel/x.mp4"]), ], ids=lambda e: type(e).__name__, ) @@ -523,9 +521,7 @@ def test_a_thumbnail_that_cannot_be_copied_names_no_file(sink, monkeypatch, tmp_ src.write_bytes(b"\xff\xd8\xff rest") def refuse(a, b): - raise PermissionError( - 13, "Permission denied", "/downloads/Some_Channel/A_Private_Title.jpg" - ) + raise PermissionError(13, "Permission denied", "/files/Some_Channel/A_Private_Title.jpg") monkeypatch.setattr(editmod.shutil, "copyfile", refuse) assert editmod.thumbnail_to_jpg(src, tmp_path / "out.jpg") is None diff --git a/backend/tests/test_file_folders.py b/backend/tests/test_file_folders.py new file mode 100644 index 00000000..a3b71fc1 --- /dev/null +++ b/backend/tests/test_file_folders.py @@ -0,0 +1,807 @@ +"""The Files library's folders and a file's retention: which folder a new file lands in, when a done +file goes, and the GC that takes it — warning once, never a file its user kept, never a file another +user still holds. + +Pure-logic for the source-key rule; DB-backed for the rest. +""" + +import hashlib +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import func, select + +from app import sysconfig +from app.config import settings +from app.downloads import folders, formats, service +from app.downloads import gc as gcmod +from app.db import SessionLocal +from app.downloads.status import JobStatus +from app.models import DownloadJob, FileFolder, MediaAsset, Notification + +NOW = datetime.now(timezone.utc) + + +@pytest.mark.parametrize( + ("url", "key"), + [ + ("https://www.tiktok.com/@a/video/1", "site:tiktok.com"), + ("https://m.facebook.com/groups/x/posts/2", "site:facebook.com"), + ("https://vimeo.com/123", "site:vimeo.com"), + ("https://youtu.be/abcdefghijk", "youtube"), + ("https://music.youtube.com/watch?v=abcdefghijk", "youtube"), + ("https://www.youtube.com/watch?v=abcdefghijk", "youtube"), + # A front door is only stripped when a real host remains behind it. + ("https://m.co/x", "site:m.co"), + ("not a url", "site:unknown"), + ], +) +def test_a_url_lands_in_its_sites_folder(url, key): + assert folders.site_key(url) == key + + +@pytest.fixture +def _root(tmp_path, monkeypatch): + monkeypatch.setattr(settings, "files_root", str(tmp_path)) + return tmp_path + + +def _asset(db, kind="youtube", ref="abcdefghijk", status="ready", rel="a.mp4", size=10): + asset = MediaAsset( + source_kind=kind, + source_ref=ref, + format_sig=hashlib.sha256(f"{kind}:{ref}".encode()).hexdigest()[:32], + status=status, + rel_path=rel, + size_bytes=size, + ref_count=0, + title="A title", + ) + db.add(asset) + db.flush() + return asset + + +def _job(db, user_id, asset, *, kind="youtube", ref=None, status=JobStatus.DONE, **kw): + job = DownloadJob( + user_id=user_id, + asset_id=asset.id if asset else None, + source_kind=kind, + source_ref=ref or (asset.source_ref if asset else "x"), + profile_snapshot={}, + status=status, + **kw, + ) + db.add(job) + db.flush() + folders.place(db, job) + if asset is not None: + service.incr_ref_count(db, asset.id) + db.commit() + return job + + +def _folder(db, job) -> FileFolder: + return db.get(FileFolder, job.folder_id) + + +def test_every_kind_of_job_lands_in_its_source_folder(db, make_user): + uid = make_user().id + yt = _job(db, uid, _asset(db)) + site = _job(db, uid, _asset(db, "url", "https://www.tiktok.com/@a/video/1"), kind="url") + up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload") + plex = _job( + db, uid, _asset(db, "clip", "c1"), kind="clip", edit_spec={"source": "plex", "ref": "9"} + ) + ytclip = _job( + db, uid, _asset(db, "clip", "c2"), kind="clip", edit_spec={"source": "youtube", "ref": "v"} + ) + # An edit of a Plex clip: only its parent knows it came from Plex (the clip's asset says "clip"). + edit = _job( + db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=plex.id, + source_asset_id=plex.asset_id, + ) # fmt: skip + keys = {j.id: _folder(db, j).source_key for j in (yt, site, up, plex, ytclip, edit)} + assert keys == { + yt.id: "youtube", + site.id: "site:tiktok.com", + up.id: "upload", + plex.id: "plex", + ytclip.id: "youtube", + edit.id: "plex", # an edit sits with the file it was cut from + } + # One folder per origin per user: the two YouTube files share theirs. + assert _folder(db, yt).id == _folder(db, ytclip).id + + +def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user): + uid = make_user().id + src = _asset(db, "url", "https://vimeo.com/1") + edit = DownloadJob( + user_id=uid, source_kind="edit", source_ref="e", profile_snapshot={}, + source_job_id=None, source_asset_id=src.id, status=JobStatus.DONE, + ) # fmt: skip + assert folders.source_key_for(db, edit) == "site:vimeo.com" + + +def test_an_edit_keeps_its_origin_after_its_parent_and_its_file_are_gone(db, make_user): + # The parent download expires, its file goes with its last holder, and both of the edit's + # pointers to it are cleared: the edit must still go back to the YouTube folder it came from. + uid = make_user().id + parent = _job(db, uid, _asset(db)) + edit = _job( + db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=parent.id, + source_asset_id=parent.asset_id, + ) # fmt: skip + home = _folder(db, edit) + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine") + db.add(own) + db.flush() + folders.move(db, edit, own) + edit.source_job_id = edit.source_asset_id = None # what ON DELETE SET NULL leaves + db.delete(parent) + db.commit() + folders.move(db, edit, home) + assert edit.folder_id == home.id + folders.move(db, edit, own) + folders.delete_own(db, own) + assert edit.folder_id == home.id + + +def test_a_folder_deleted_while_a_file_goes_in_takes_that_file_home_too(db, make_user): + # Before, the delete missed the file still on its way in and failed on the foreign key (a 500), + # or the file's insert did after the folder went. + import threading + + uid = make_user().id + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine") + db.add(own) + db.commit() + own_id = own.id + job = _job(db, uid, _asset(db)) + home = job.folder_id + target = folders.own_folder(db, uid, own_id) # held from the check until this commits + outcome: list[object] = [] + + def delete_it(): + other = SessionLocal() + try: + outcome.append(folders.delete_own(other, other.get(FileFolder, own_id))) + other.commit() + except Exception as e: # noqa: BLE001 — the assertion below names what went wrong + outcome.append(e) + finally: + other.close() + + t = threading.Thread(target=delete_it) + t.start() + t.join(timeout=1) + assert t.is_alive() # the delete waits for the file, even before it is written + folders.move(db, job, target) + db.commit() + t.join(timeout=10) + assert outcome == [1] + db.expire_all() + assert db.get(DownloadJob, job.id).folder_id == home + assert db.get(FileFolder, own_id) is None + + +def test_folders_are_per_user(db, make_user): + a, b = make_user().id, make_user().id + asset = _asset(db) + ja, jb = _job(db, a, asset), _job(db, b, asset) + assert ja.folder_id != jb.folder_id + assert (_folder(db, ja).user_id, _folder(db, jb).user_id) == (a, b) + + +def test_a_source_folder_keeps_files_the_retention_days_and_uploads_forever(db, make_user): + sysconfig.set_value(db, "download_retention_days", 12) + db.commit() + uid = make_user().id + yt = _job(db, uid, _asset(db)) + up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload") + assert _folder(db, yt).retention_days == 12 + assert _folder(db, up).retention_days is None + + +def test_enqueue_places_the_job_and_a_cache_hit_is_stamped_at_once(db, make_user): + uid = make_user().id + spec = formats.normalize({}) + first = service.enqueue(db, uid, "youtube", "abcdefghijk", spec) + assert _folder(db, first).source_key == "youtube" + assert first.expires_at is None # queued: its retention starts when it is done + asset = db.get(MediaAsset, first.asset_id) + asset.status, asset.rel_path = "ready", "a.mp4" + db.commit() + other = make_user().id + hit = service.enqueue(db, other, "youtube", "abcdefghijk", spec) + assert hit.status == JobStatus.DONE + days = _folder(db, hit).retention_days + assert hit.expires_at is not None + assert abs(hit.expires_at - (NOW + timedelta(days=days))) < timedelta(minutes=5) + + +def test_a_finishing_worker_stamps_the_expiry_unless_kept(db, make_user): + uid = make_user().id + asset = _asset(db) + plain = _job(db, uid, asset, status=JobStatus.RUNNING) + kept = _job(db, uid, asset, status=JobStatus.RUNNING, kept=True) + for job in (plain, kept): + assert service.finalize_if_running(db, job.id, status=JobStatus.DONE, progress=100) + db.commit() + db.refresh(plain) + db.refresh(kept) + assert plain.expires_at is not None + assert kept.expires_at is None + + +def _gc(db): + return gcmod.run_download_gc(db) + + +def _notices(db, uid, kind): + return db.execute( + select(func.count()) + .select_from(Notification) + .where(Notification.user_id == uid, Notification.type == kind) + ).scalar() + + +def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user, _root): + (_root / "a.mp4").write_bytes(b"x") + uid = make_user().id + job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12)) + _gc(db) + _gc(db) + assert _notices(db, uid, "download_expiring") == 1 + job.expires_at = NOW - timedelta(minutes=1) + db.commit() + assert _gc(db)["expired"] == 1 + assert db.get(DownloadJob, job.id) is None + assert _notices(db, uid, "download_expired") == 1 + # Its last holder gone, the shared file and its row go too. + assert not (_root / "a.mp4").exists() + assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0 + + +@pytest.mark.parametrize("restart", ["unkeep", "move", "retention"]) +def test_a_file_whose_clock_starts_again_is_warned_again(db, make_user, _root, restart): + # The GC warns once per expiry; a file whose expiry was moved must be warned before the new one. + (_root / "a.mp4").write_bytes(b"x") + uid = make_user().id + job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12)) + _gc(db) + assert job.expiry_warned + if restart == "unkeep": + folders.set_kept(db, job, True) + folders.set_kept(db, job, False) + elif restart == "move": + home = _folder(db, job) + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine") + db.add(own) + db.flush() + folders.move(db, job, own) + folders.move(db, job, home) + else: + folders.set_retention(db, _folder(db, job), 90) + db.commit() + assert not job.expiry_warned + + +def test_the_gc_never_takes_a_kept_file(db, make_user, _root): + (_root / "a.mp4").write_bytes(b"x") + uid = make_user().id + job = _job(db, uid, _asset(db), kept=True, expires_at=NOW - timedelta(days=1)) + assert _gc(db)["expired"] == 0 + assert db.get(DownloadJob, job.id) is not None + + +def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root): + (_root / "a.mp4").write_bytes(b"x") + a, b = make_user().id, make_user().id + asset = _asset(db) + gone = _job(db, a, asset, expires_at=NOW - timedelta(minutes=1)) + stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20)) + _gc(db) + assert db.get(DownloadJob, gone.id) is None + assert db.get(DownloadJob, stays.id) is not None + assert (_root / "a.mp4").exists() + + +def _meanwhile(monkeypatch, act, step="_relock_expired"): + """Run `act` in another session (another request) between a GC pass's SELECT and its handling of + each row it found — the window a user's click can land in. `step` is the pass's per-row re-read + (expiry, warning, eviction); `act` gets that row's id.""" + real = getattr(gcmod, step) + + def wrapped(db, key, *rest): + other = SessionLocal() + try: + act(other, key) + other.commit() + finally: + other.close() + return real(db, key, *rest) + + monkeypatch.setattr(gcmod, step, wrapped) + + +def test_a_file_kept_while_the_gc_runs_stays(db, make_user, _root, monkeypatch): + (_root / "a.mp4").write_bytes(b"x") + job = _job(db, make_user().id, _asset(db), expires_at=NOW - timedelta(minutes=1)) + _meanwhile(monkeypatch, lambda other, jid: setattr(other.get(DownloadJob, jid), "kept", True)) + assert _gc(db)["expired"] == 0 + db.expire_all() + assert db.get(DownloadJob, job.id) is not None + assert (_root / "a.mp4").exists() + + +def test_a_file_deleted_while_the_gc_runs_is_released_once(db, make_user, _root, monkeypatch): + # Released twice, the shared file would go from under the other user's copy. + (_root / "a.mp4").write_bytes(b"x") + a, b = make_user().id, make_user().id + asset = _asset(db) + _job(db, a, asset, expires_at=NOW - timedelta(minutes=1)) + stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20)) + + def user_deletes(other, jid): + service.delete(other, other.get(DownloadJob, jid)) + + _meanwhile(monkeypatch, user_deletes) + _gc(db) + db.expire_all() + assert db.get(DownloadJob, stays.id).asset_id == asset.id + assert db.get(MediaAsset, asset.id).ref_count == 1 + assert (_root / "a.mp4").exists() + + +def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library( + db, make_user, _root +): + sysconfig.set_value(db, "download_total_max_bytes", 15) + db.commit() + a, b = make_user().id, make_user().id + for name in ("old.mp4", "new.mp4"): + (_root / name).write_bytes(b"x") + old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10) + new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10) + old.last_access_at = NOW - timedelta(days=9) + new.last_access_at = NOW - timedelta(days=1) + kept = _job(db, a, old, kept=True) # the older file, but kept: the newer one goes instead + ja, jb = _job(db, a, new), _job(db, b, new) + assert _gc(db)["evicted"] == 1 + assert db.get(DownloadJob, kept.id) is not None + assert db.get(DownloadJob, ja.id) is None and db.get(DownloadJob, jb.id) is None + assert _notices(db, a, "download_evicted") == 1 + assert _notices(db, b, "download_evicted") == 1 + assert (_root / "old.mp4").exists() and not (_root / "new.mp4").exists() + + +def test_a_file_deleted_while_the_gc_warns_does_not_stop_the_run(db, make_user, _root, monkeypatch): + # Written back from a stale read, the warning of a deleted file failed the whole run: nothing + # expired or was evicted that cycle, nor the next while the race lasted. + (_root / "a.mp4").write_bytes(b"x") + (_root / "b.mp4").write_bytes(b"x") + uid = make_user().id + _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=1)) + gone = _job( + db, uid, _asset(db, ref="bbbbbbbbbbb", rel="b.mp4"), expires_at=NOW - timedelta(minutes=1) + ) + + def user_deletes(other, jid): + service.delete(other, other.get(DownloadJob, jid)) + + _meanwhile(monkeypatch, user_deletes, step="_mark_warned") + result = _gc(db) + assert (result["warned"], result["expired"]) == (0, 1) + db.expire_all() + assert db.get(DownloadJob, gone.id) is None + assert _notices(db, uid, "download_expiring") == 0 + + +def test_a_file_re_timed_while_the_gc_warns_keeps_its_new_expiry_unwarned( + db, make_user, _root, monkeypatch +): + # Moved, un-kept or its folder's retention changed: a fresh expiry, warned when it comes — not + # marked warned now for a date it no longer has. + uid = make_user().id + job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=1)) + later = NOW + timedelta(days=30) + + def re_timed(other, jid): + moved = other.get(DownloadJob, jid) + moved.expires_at = later + moved.expiry_warned = False + + _meanwhile(monkeypatch, re_timed, step="_mark_warned") + assert _gc(db)["warned"] == 0 + db.expire_all() + fresh = db.get(DownloadJob, job.id) + assert (fresh.expires_at, fresh.expiry_warned) == (later, False) + assert _notices(db, uid, "download_expiring") == 0 + + +def _two_over_the_cap(db, make_user, _root): + sysconfig.set_value(db, "download_total_max_bytes", 15) + db.commit() + a, b = make_user().id, make_user().id + for name in ("old.mp4", "new.mp4"): + (_root / name).write_bytes(b"x") + old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10) + new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10) + old.last_access_at = NOW - timedelta(days=9) + new.last_access_at = NOW - timedelta(days=1) + return _job(db, a, old), _job(db, b, new) + + +def test_a_file_deleted_while_the_cache_is_over_its_cap_frees_that_space_and_no_more( + db, make_user, _root, monkeypatch +): + # Its bytes still counted, the pass went on to evict another user's file for space already free. + jold, jnew = _two_over_the_cap(db, make_user, _root) + + def holder_deletes(other, asset_id): + if asset_id == jold.asset_id: + service.delete(other, other.get(DownloadJob, jold.id)) + + _meanwhile(monkeypatch, holder_deletes, step="_relock_evictable") + assert _gc(db)["evicted"] == 0 + db.expire_all() + assert db.get(DownloadJob, jnew.id) is not None + assert (_root / "new.mp4").exists() + + +def test_a_file_kept_while_the_cache_is_over_its_cap_stays(db, make_user, _root, monkeypatch): + jold, jnew = _two_over_the_cap(db, make_user, _root) + + def holder_keeps(other, asset_id): + if asset_id == jold.asset_id: + other.get(DownloadJob, jold.id).kept = True + + _meanwhile(monkeypatch, holder_keeps, step="_relock_evictable") + assert _gc(db)["evicted"] == 1 # the newer one goes in its place + db.expire_all() + assert db.get(DownloadJob, jold.id) is not None + assert (_root / "old.mp4").exists() + assert db.get(DownloadJob, jnew.id) is None + + +def _in_thread(work): + """Run `work(other_session)` in another thread and session; returns (thread, outcome list).""" + import threading + + outcome: list[object] = [] + + def run(): + other = SessionLocal() + try: + outcome.append(work(other)) + other.commit() + except Exception as e: # noqa: BLE001 — the assertion below names what went wrong + outcome.append(e) + finally: + other.close() + + t = threading.Thread(target=run) + t.start() + return t, outcome + + +def test_a_retention_changed_while_a_file_goes_in_times_that_file_too(db, make_user): + # The change used to list the folder's files before the arriving one was in, so that one kept the + # old retention — deleted later from a folder now set to keep files forever. + uid = make_user().id + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7) + db.add(own) + db.commit() + own_id = own.id + job = _job(db, uid, _asset(db)) + target = folders.own_folder(db, uid, own_id) # held from the check until this commits + t, outcome = _in_thread( + lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None) + ) + t.join(timeout=1) + assert t.is_alive() # the change waits for the file being put in + folders.move(db, job, target) # stamped with the 7 days it sees + db.commit() + t.join(timeout=10) + assert outcome == [7] # the retention it replaced + db.expire_all() + assert db.get(DownloadJob, job.id).expires_at is None + + +def test_a_retention_changed_while_a_file_finishes_there_times_that_file_too(db, make_user): + # The worker's finish read the folder's old retention unlocked, while the change listed the done + # files before this one was among them. + uid = make_user().id + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7) + db.add(own) + db.commit() + own_id = own.id + job = _job(db, uid, _asset(db), status=JobStatus.RUNNING) + folders.move(db, job, own) + db.commit() + job.status = JobStatus.DONE + folders.stamp_done(db, job) # the finish: stamped with the 7 days it reads + db.flush() + t, outcome = _in_thread( + lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None) + ) + t.join(timeout=1) + assert t.is_alive() # the change waits for the file being stamped + db.commit() + t.join(timeout=10) + assert outcome == [7] # the retention it replaced + db.expire_all() + assert db.get(DownloadJob, job.id).expires_at is None + + +def _deleted_uncommitted(db, job): + """`service.delete` up to its commit: the file is gone in this session, its row still locked.""" + service.release_asset(db, job) + db.delete(job) + db.flush() + + +def test_a_file_moved_out_while_its_folder_is_deleted_stays_where_it_went(db, make_user): + uid = make_user().id + f = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="F") + g = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="G") + db.add_all([f, g]) + db.commit() + f_id, g_id = f.id, g.id + job = _job(db, uid, _asset(db)) + folders.move(db, job, f) + db.commit() + folders.move(db, job, folders.own_folder(db, uid, g_id)) + db.flush() # the move is written, not yet committed + t, outcome = _in_thread(lambda other: folders.delete_own(other, other.get(FileFolder, f_id))) + t.join(timeout=1) + assert t.is_alive() # the delete waits for the file it would take home + db.commit() + t.join(timeout=10) + assert outcome == [0] + db.expire_all() + assert db.get(DownloadJob, job.id).folder_id == g_id + + +def test_a_file_deleted_while_its_folder_is_deleted_does_not_fail_the_delete(db, make_user, _root): + (_root / "a.mp4").write_bytes(b"x") + uid = make_user().id + f = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="F") + db.add(f) + db.commit() + f_id = f.id + job = _job(db, uid, _asset(db)) + folders.move(db, job, f) + db.commit() + _deleted_uncommitted(db, db.get(DownloadJob, job.id)) + t, outcome = _in_thread(lambda other: folders.delete_own(other, other.get(FileFolder, f_id))) + t.join(timeout=1) + assert t.is_alive() # the delete waits for the file being deleted + db.commit() + t.join(timeout=10) + assert outcome == [0] + db.expire_all() + assert db.get(FileFolder, f_id) is None + + +def _own_with_a_file(db, uid, retention=30): + """A user's own folder keeping files `retention` days, holding one done file due in as many.""" + own = FileFolder( + user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=retention + ) + db.add(own) + db.flush() + job = _job(db, uid, _asset(db)) + job.folder_id = own.id + job.expires_at = NOW + timedelta(days=retention) + db.commit() + return own.id, job.id + + +def _elsewhere(act): + other = SessionLocal() + try: + act(other) + other.commit() + finally: + other.close() + + +def test_a_retention_change_works_from_what_a_change_before_it_left(db, make_user): + # The locked re-read was never fetched, so the change shifted the files from the value read before + # its wait: 30 -> 10 -> 60 left them due in 40 days under a folder saying 60. + own_id, job_id = _own_with_a_file(db, make_user().id) + stale = db.get(FileFolder, own_id) # read before the other change lands + _elsewhere(lambda other: folders.set_retention(other, other.get(FileFolder, own_id), 10)) + assert folders.set_retention(db, stale, 60) == 10 + db.commit() + db.expire_all() + due = db.get(DownloadJob, job_id).expires_at - NOW + assert timedelta(days=59) < due < timedelta(days=61) + + +def test_a_retention_change_to_a_folder_deleted_meanwhile_is_refused(db, make_user): + own_id, _ = _own_with_a_file(db, make_user().id) + stale = db.get(FileFolder, own_id) + _elsewhere(lambda other: folders.delete_own(other, other.get(FileFolder, own_id))) + with pytest.raises(folders.FolderError, match="file_folder_gone"): + folders.set_retention(db, stale, 60) + + +def test_a_file_moved_out_while_its_folder_retention_changes_keeps_its_new_folders(db, make_user): + uid = make_user().id + own_id, job_id = _own_with_a_file(db, uid) + g = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="G", retention_days=7) + db.add(g) + db.commit() + g_id = g.id + folders.move(db, db.get(DownloadJob, job_id), folders.own_folder(db, uid, g_id)) + db.flush() # the move is written, not yet committed + t, outcome = _in_thread( + lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None) + ) + t.join(timeout=1) + assert t.is_alive() # the change waits for the file it would time + db.commit() + t.join(timeout=10) + assert outcome == [30] + db.expire_all() + job = db.get(DownloadJob, job_id) + assert job.folder_id == g_id + assert job.expires_at is not None # G's 7 days, not the forever of the folder it left + + +def test_a_file_deleted_while_its_folder_retention_changes_does_not_fail_the_change( + db, make_user, _root +): + (_root / "a.mp4").write_bytes(b"x") + own_id, job_id = _own_with_a_file(db, make_user().id) + _deleted_uncommitted(db, db.get(DownloadJob, job_id)) + t, outcome = _in_thread( + lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None) + ) + t.join(timeout=1) + assert t.is_alive() # the change waits for the file it would time + db.commit() + t.join(timeout=10) + assert outcome == [30] + + +def test_a_file_finishing_while_its_folder_is_deleted_gets_its_new_folders_retention(db, make_user): + # The finish read the folder unlocked, then waited out the delete and found no folder: the file + # was done with no expiry at all, in a source folder that keeps files 30 days. + uid = make_user().id + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7) + db.add(own) + db.commit() + own_id = own.id + job = _job(db, uid, _asset(db), status=JobStatus.RUNNING) + job_id = job.id + folders.move(db, job, own) + db.commit() + folders.delete_own(db, db.get(FileFolder, own_id)) + db.flush() # the delete, not yet committed + t, outcome = _in_thread( + lambda other: service.finalize_if_running(other, job_id, status=JobStatus.DONE) + ) + t.join(timeout=1) + assert t.is_alive() # the finish waits for the delete + db.commit() + t.join(timeout=10) + assert outcome == [True] + db.expire_all() + done = db.get(DownloadJob, job_id) + assert done.folder_id != own_id + assert done.expires_at is not None + + +def _an_upload_over_the_cap_too(db, uid, _root): + (_root / "u.bin").write_bytes(b"x") + return _job(db, uid, _asset(db, "upload", "u1", rel="u.bin", size=10), kind="upload") + + +def test_space_freed_elsewhere_while_the_cache_is_over_its_cap_evicts_no_more( + db, make_user, _root, monkeypatch +): + # The pass counted the disk once: an upload deleted meanwhile still counted, so a second user's + # file went for space already free. + jold, jnew = _two_over_the_cap(db, make_user, _root) + up = _an_upload_over_the_cap_too(db, make_user().id, _root) + + def upload_deleted(other, asset_id): + if asset_id == jold.asset_id: + service.delete(other, other.get(DownloadJob, up.id)) + + _meanwhile(monkeypatch, upload_deleted, step="_relock_evictable") + assert _gc(db)["evicted"] == 1 + db.expire_all() + assert db.get(DownloadJob, jnew.id) is not None + assert (_root / "new.mp4").exists() + + +def test_a_file_asked_for_again_while_the_cache_is_over_its_cap_stays( + db, make_user, _root, monkeypatch +): + # A cache hit between the pass's SELECT and its lock: the file the user just asked for went. + jold, jnew = _two_over_the_cap(db, make_user, _root) + + def asked_again(other, asset_id): + if asset_id == jold.asset_id: + other.get(MediaAsset, asset_id).last_access_at = datetime.now(timezone.utc) + + _meanwhile(monkeypatch, asked_again, step="_relock_evictable") + _gc(db) + db.expire_all() + assert db.get(DownloadJob, jold.id) is not None + assert (_root / "old.mp4").exists() + + +def test_a_file_asked_for_while_its_cached_copy_is_freed_gets_a_fresh_one(db, make_user): + # The enqueue read the cached copy unlocked: freed meanwhile, its job pointed at a row that was gone + # (a foreign-key failure, a 500). + asset = _asset(db) + asset_id, sig = asset.id, asset.format_sig + db.commit() + db.execute(select(MediaAsset).where(MediaAsset.id == asset_id).with_for_update()) + db.delete(db.get(MediaAsset, asset_id)) + db.flush() # the release, not yet committed + t, outcome = _in_thread( + lambda other: service.get_or_create_asset(other, "youtube", "abcdefghijk", sig).id + ) + t.join(timeout=1) + assert t.is_alive() # it waits for the release + db.commit() + t.join(timeout=10) + [fresh] = outcome + assert isinstance(fresh, int) and fresh != asset_id + + +def test_an_empty_volume_in_the_librarys_place_is_noticed(db, make_user, _root): + # A fresh volume is a real, writable mount: only the files the library names tell it is the wrong one. + assert service.library_left_behind(db, str(_root)) is False # nothing named yet + _asset(db, rel="a.mp4") + db.commit() + assert service.library_left_behind(db, str(_root)) is True + (_root / "a.mp4").write_bytes(b"x") + assert service.library_left_behind(db, str(_root)) is False + + +def test_deleting_a_user_takes_their_folders_and_files_with_them(db, make_user): + # Both tables cascade from users; the files' folder FK refuses deleting a folder that still holds + # files, which must not stop the account's own deletion. + user = make_user() + _job(db, user.id, _asset(db)) + db.delete(user) + db.commit() + assert db.execute(select(func.count()).select_from(FileFolder)).scalar() == 0 + assert db.execute(select(func.count()).select_from(DownloadJob)).scalar() == 0 + + +def test_the_longest_host_has_room_for_its_folder(db, make_user): + label = "a" * 63 + host = ".".join([label, label, label, "b" * 58]) + ".hu" # 253 characters, the DNS maximum + assert len(host) == 253 + uid = make_user().id + job = _job(db, uid, _asset(db, "url", f"https://{host}/v"), kind="url") + assert _folder(db, job).source_key == f"site:{host}" + + +def test_the_ready_notice_names_the_file_and_its_folder(db, make_user): + # The notice's "Open" goes to the file in its folder, so both ride along as data. + from app.downloads import pipeline + + user = make_user() + job = _job(db, user.id, _asset(db)) + pipeline._notify_download_done(db, user.id, "A title", job.id, kind="download") + db.commit() + note = db.scalars(select(Notification).where(Notification.user_id == user.id)).one() + assert note.type == "download_done" + assert note.data == { + "title": "A title", + "kind": "download", + "job_id": job.id, + "folder_id": job.folder_id, + } + assert job.folder_id is not None diff --git a/backend/tests/test_files_api.py b/backend/tests/test_files_api.py new file mode 100644 index 00000000..1f4f4395 --- /dev/null +++ b/backend/tests/test_files_api.py @@ -0,0 +1,670 @@ +"""The Files library API (`routes.files`), DB-lane: folders (list, own create/rename/delete, a +folder's retention), the file list (done only, by folder, searched, sorted, shared with me), a +file's rename / move / keep / delete, the upload into a folder, and the file's hash. + +Every route is also held to the caller's own rows: another user's folder or file answers like a +missing one. +""" + +import base64 +import hashlib +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select + +from app import sysconfig +from app.config import settings +from app.downloads import folders, service +from app.downloads import upload as uploadmod +from app.downloads.status import JobStatus +from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset + +NOW = datetime.now(timezone.utc) + + +@pytest.fixture +def _root(tmp_path, monkeypatch): + monkeypatch.setattr(settings, "files_root", str(tmp_path)) + return tmp_path + + +def _file(db, user_id, *, kind="youtube", ref=None, title="A title", size=10, rel=None, **kw): + """A done library file: an asset with bytes on disk-path terms, and the user's job over it.""" + n = db.execute(select(MediaAsset.id)).all() + ref = ref or f"vid{len(n):08d}" + asset = MediaAsset( + source_kind=kind, + source_ref=ref, + format_sig=f"sig-{ref}", + status="ready", + rel_path=rel or f"{ref}.mp4", + size_bytes=size, + ref_count=0, + title=title, + media_kind="video", + container="mp4", + ) + db.add(asset) + db.flush() + job = DownloadJob( + user_id=user_id, + asset_id=asset.id, + source_kind=kind, + source_ref=ref, + profile_snapshot={}, + status=kw.pop("status", JobStatus.DONE), + **kw, + ) + db.add(job) + db.flush() + folders.place(db, job) + folders.stamp_done(db, job) + service.incr_ref_count(db, asset.id) + db.commit() + return job + + +def _own(db, user_id, name="Saját"): + folder = FileFolder(user_id=user_id, kind=folders.FolderKind.OWN, name=name) + db.add(folder) + db.commit() + return folder + + +def _audits(db, action): + return db.execute(select(AuditLog).where(AuditLog.action == action)).scalars().all() + + +# --- folders ------------------------------------------------------------------------------- + + +def test_the_folder_list_counts_each_folders_done_files_and_is_the_callers_own( + db, make_user, as_user +): + alice, bob = make_user(), make_user() + _file(db, alice.id, size=10) + _file(db, alice.id, size=5) + _file(db, alice.id, kind="url", ref="https://vimeo.com/1", size=7) + _file(db, alice.id, status=JobStatus.RUNNING) # not in the library yet + _file(db, bob.id) + got = { + (f["kind"], f["source_key"]): (f["count"], f["size_bytes"]) + for f in as_user(alice).get("/api/files/folders").json() + } + assert got == {("source", "youtube"): (2, 15), ("source", "site:vimeo.com"): (1, 7)} + + +def test_an_own_folder_is_made_once_per_name_whatever_its_case(db, make_user, as_user): + alice = make_user() + c = as_user(alice) + made = c.post("/api/files/folders", json={"name": "Nyaralás"}) + assert made.status_code == 200, made.text + assert (made.json()["kind"], made.json()["name"]) == ("own", "Nyaralás") + again = c.post("/api/files/folders", json={"name": "NYARALÁS"}) + assert again.status_code == 409 + assert again.json()["detail"]["reason"] == "file_folder_name_taken" + blank = c.post("/api/files/folders", json={"name": " "}) + assert blank.json()["detail"]["reason"] == "file_folder_name_required" + # Another user may have a folder of the same name. + assert as_user(make_user()).post("/api/files/folders", json={"name": "Nyaralás"}).is_success + assert len(_audits(db, "file_folder_create")) == 2 + + +def test_only_an_own_folder_is_renamed_and_only_by_its_user(db, make_user, as_user): + alice = make_user() + c = as_user(alice) + src = db.get(FileFolder, _file(db, alice.id).folder_id) + own = _own(db, alice.id, "Régi") + taken = _own(db, alice.id, "Foglalt") + ok = c.patch(f"/api/files/folders/{own.id}", json={"name": "Új"}) + assert ok.json()["name"] == "Új" + clash = c.patch(f"/api/files/folders/{own.id}", json={"name": taken.name.lower()}) + assert clash.json()["detail"]["reason"] == "file_folder_name_taken" + no = c.patch(f"/api/files/folders/{src.id}", json={"name": "YouTube 2"}) + assert no.status_code == 409 and no.json()["detail"]["reason"] == "file_folder_not_own" + stranger = as_user(make_user()).patch(f"/api/files/folders/{own.id}", json={"name": "X"}) + assert stranger.status_code == 404 + db.refresh(own) + assert own.name == "Új" + + +def test_a_shorter_retention_moves_each_files_expiry_but_never_inside_the_warning_window( + db, make_user, as_user +): + sysconfig.set_value(db, "download_gc_grace_days", 3) + db.commit() + alice = make_user() + fresh = _file(db, alice.id) # expires in 30 days + old = _file(db, alice.id) + old.expires_at = NOW + timedelta(days=10) # 20 days old already + kept = _file(db, alice.id, kept=True) + db.commit() + folder_id = fresh.folder_id + r = as_user(alice).patch(f"/api/files/folders/{folder_id}", json={"retention_days": 20}) + assert r.json()["retention_days"] == 20 + for j in (fresh, old, kept): + db.refresh(j) + # 30 → 20 days: each loses 10 — the one with 10 left would be due now, so it gets the window. + assert abs(fresh.expires_at - (NOW + timedelta(days=20))) < timedelta(minutes=5) + assert abs(old.expires_at - (NOW + timedelta(days=3))) < timedelta(minutes=5) + assert kept.expires_at is None + forever = as_user(alice).patch(f"/api/files/folders/{folder_id}", json={"retention_days": None}) + assert forever.json()["retention_days"] is None + db.refresh(fresh) + assert fresh.expires_at is None + rows = _audits(db, "file_folder_retention") + assert [(a.before, a.after) for a in rows] == [ + ({"retention_days": 30}, {"retention_days": 20}), + ({"retention_days": 20}, {"retention_days": None}), + ] + + +def test_a_retention_set_on_a_folder_that_kept_forever_starts_its_files_clock_now( + db, make_user, as_user +): + alice = make_user() + up = _file(db, alice.id, kind="upload", ref="u1") + assert up.expires_at is None + r = as_user(alice).patch(f"/api/files/folders/{up.folder_id}", json={"retention_days": 7}) + assert r.status_code == 200, r.text + db.refresh(up) + assert abs(up.expires_at - (NOW + timedelta(days=7))) < timedelta(minutes=5) + + +def test_deleting_an_own_folder_sends_its_files_home_and_a_source_folder_cannot_be_deleted( + db, make_user, as_user +): + alice = make_user() + c = as_user(alice) + yt = _file(db, alice.id) + site = _file(db, alice.id, kind="url", ref="https://vimeo.com/2") + home = {yt.id: yt.folder_id, site.id: site.folder_id} + own = _own(db, alice.id) + for j in (yt, site): + assert c.patch(f"/api/files/{j.id}", json={"folder_id": own.id}).is_success + r = c.delete(f"/api/files/folders/{own.id}") + assert r.json() == {"ok": True, "deleted": own.id, "files_moved": 2} + assert db.get(FileFolder, own.id) is None + for j in (yt, site): + db.refresh(j) + assert j.folder_id == home[j.id] + # Back in a source folder, its clock runs again from now. + assert abs(j.expires_at - (NOW + timedelta(days=30))) < timedelta(minutes=5) + src = c.delete(f"/api/files/folders/{yt.folder_id}") + assert src.status_code == 409 and src.json()["detail"]["reason"] == "file_folder_not_own" + other = _own(db, alice.id, "Más") + assert as_user(make_user()).delete(f"/api/files/folders/{other.id}").status_code == 404 + assert len(_audits(db, "file_folder_delete")) == 1 + + +# --- the file list ------------------------------------------------------------------------- + + +def test_the_list_holds_the_callers_done_files_by_folder(db, make_user, as_user): + alice, bob = make_user(), make_user() + yt = _file(db, alice.id) + site = _file(db, alice.id, kind="url", ref="https://vimeo.com/3") + _file(db, alice.id, status=JobStatus.QUEUED) + _file(db, bob.id) + c = as_user(alice) + assert {f["id"] for f in c.get("/api/files").json()} == {yt.id, site.id} + only = c.get("/api/files", params={"folder_id": site.folder_id}).json() + assert [f["id"] for f in only] == [site.id] + assert only[0]["folder_id"] == site.folder_id and only[0]["shared"] is False + # Bob's folder id names nothing of Alice's. + bobs = db.execute(select(FileFolder.id).where(FileFolder.user_id == bob.id)).scalar() + assert c.get("/api/files", params={"folder_id": bobs}).json() == [] + + +def test_the_list_is_searched_by_name_and_channel_with_wildcards_taken_literally( + db, make_user, as_user +): + alice = make_user() + hit = _file(db, alice.id, title="Balaton 100% nyár") + by_channel = _file(db, alice.id, title="Más", display_uploader="Balatoni csatorna") + _file(db, alice.id, title="Balaton 1000 nyár") + c = as_user(alice) + assert [f["id"] for f in c.get("/api/files", params={"q": "100%"}).json()] == [hit.id] + found = {f["id"] for f in c.get("/api/files", params={"q": "balatoni"}).json()} + assert found == {by_channel.id} + + +def test_the_list_sorts_by_name_size_or_when_added(db, make_user, as_user): + alice = make_user() + b = _file(db, alice.id, title="Béka", size=30) + a = _file(db, alice.id, title="alma", size=10) + z = _file(db, alice.id, title="Zebra", size=20) + c = as_user(alice) + + def ids(**params): + return [f["id"] for f in c.get("/api/files", params=params).json()] + + assert ids() == [z.id, a.id, b.id] # newest first + assert ids(sort="name", order="asc") == [a.id, b.id, z.id] # the reader's alphabet, any case + assert ids(sort="size", order="desc") == [b.id, z.id, a.id] + + +def test_a_file_is_added_when_it_is_done_not_when_it_was_asked_for(db, make_user, as_user): + """A download that failed for days and then went through is new: it lists first, dated today.""" + alice = make_user() + early = _file(db, alice.id, created_at=NOW - timedelta(days=1)) + early.ready_at = NOW - timedelta(days=1) + late = _file(db, alice.id, status=JobStatus.RUNNING, created_at=NOW - timedelta(days=5)) + late.ready_at = None + db.commit() + assert service.finalize_if_running(db, late.id, status=JobStatus.DONE) + db.commit() + db.expire_all() # the finalize is a Core UPDATE; the route reads this same session + got = as_user(alice).get("/api/files").json() + assert [f["id"] for f in got] == [late.id, early.id] + ready = datetime.fromisoformat(got[0]["ready_at"]) + assert ready > NOW - timedelta(minutes=5) + assert datetime.fromisoformat(got[0]["created_at"]) < NOW - timedelta(days=4) + + +def test_a_job_made_done_is_ready_as_it_is_made(db, make_user): + # An upload kept as it came is inserted done; the database refuses a done job without the moment. + job = DownloadJob( + user_id=make_user().id, + source_kind="upload", + source_ref="r", + profile_snapshot={}, + status=JobStatus.DONE, + ) + db.add(job) + db.commit() + assert job.ready_at is not None + + +def test_shared_with_me_lists_what_others_shared_and_nothing_else(db, make_user, as_user): + alice, bob, carol = make_user(), make_user(), make_user() + theirs = _file(db, bob.id) + _file(db, bob.id) # not shared + for_carol = _file(db, bob.id) + db.add(DownloadShare(job_id=theirs.id, from_user_id=bob.id, to_user_id=alice.id)) + db.add(DownloadShare(job_id=for_carol.id, from_user_id=bob.id, to_user_id=carol.id)) + db.commit() + got = as_user(alice).get("/api/files", params={"scope": "shared"}).json() + assert [(f["id"], f["shared"]) for f in got] == [(theirs.id, True)] + + +def test_shared_with_me_is_ordered_by_when_it_was_shared_and_can_be_dismissed( + db, make_user, as_user +): + """A friend's old file shared today is new to me: "added" is the share's date there. Dismissing + one drops only my grant; the owner's file stays.""" + alice, bob = make_user(), make_user() + old = _file(db, bob.id, created_at=NOW - timedelta(days=30)) + new = _file(db, bob.id, created_at=NOW - timedelta(days=1)) + db.add( + DownloadShare( + job_id=new.id, + from_user_id=bob.id, + to_user_id=alice.id, + created_at=NOW - timedelta(hours=2), + ) + ) + db.add( + DownloadShare( + job_id=old.id, + from_user_id=bob.id, + to_user_id=alice.id, + created_at=NOW - timedelta(hours=1), + ) + ) + db.commit() + c = as_user(alice) + got = c.get("/api/files", params={"scope": "shared"}).json() + assert [f["id"] for f in got] == [old.id, new.id] + assert got[0]["shared_at"] is not None and got[0]["shared_at"] > got[1]["shared_at"] + r = c.delete(f"/api/files/shared/{old.id}") + assert r.json() == {"ok": True, "removed": old.id} + assert [f["id"] for f in c.get("/api/files", params={"scope": "shared"}).json()] == [new.id] + assert db.get(DownloadJob, old.id) is not None + + +def test_each_listed_file_says_where_it_came_from_wherever_it_sits(db, make_user, as_user): + """The Source column: a file's own source folder, also after it moved into an own folder.""" + alice = make_user() + yt = _file(db, alice.id) + site = _file(db, alice.id, kind="url", ref="https://www.tiktok.com/@x/video/1") + moved = _file(db, alice.id) + own = _own(db, alice.id) + c = as_user(alice) + assert c.patch(f"/api/files/{moved.id}", json={"folder_id": own.id}).status_code == 200 + got = {f["id"]: f["source_key"] for f in c.get("/api/files").json()} + assert got == {yt.id: "youtube", site.id: "site:tiktok.com", moved.id: "youtube"} + + +# --- one file ------------------------------------------------------------------------------ + + +def test_a_file_is_renamed_kept_and_given_back_to_its_folders_retention(db, make_user, as_user): + alice = make_user() + job = _file(db, alice.id) + c = as_user(alice) + assert c.patch(f"/api/files/{job.id}", json={"name": "Új név"}).json()["display_name"] == ( + "Új név" + ) + kept = c.patch(f"/api/files/{job.id}", json={"kept": True}).json() + assert (kept["kept"], kept["expires_at"]) == (True, None) + back = c.patch(f"/api/files/{job.id}", json={"kept": False}).json() + assert back["kept"] is False and back["expires_at"] is not None + # Setting what it already is records nothing. + assert c.patch(f"/api/files/{job.id}", json={"kept": False}).status_code == 200 + assert len(_audits(db, "file_keep")) == 1 and len(_audits(db, "file_unkeep")) == 1 + assert as_user(make_user()).patch(f"/api/files/{job.id}", json={"kept": True}).status_code == ( + 404 + ) + + +def test_a_files_channel_and_links_are_set_and_only_plausible_links_are_kept( + db, make_user, as_user +): + alice = make_user() + job = _file(db, alice.id) + c = as_user(alice) + got = c.patch( + f"/api/files/{job.id}", + json={ + "uploader": " Saját csatorna ", + "uploader_url": "mailto:someone@example.com", + "extra_links": ["https://a.example/x", "ftp://b.example", "", 7, "http://[::1/x"] + + [f"https://c.example/{i}" for i in range(10)], + }, + ).json() + assert got["display_uploader"] == "Saját csatorna" + assert got["display_uploader_url"] is None + assert got["extra_links"][0] == "https://a.example/x" and len(got["extra_links"]) == 8 + cleared = c.patch(f"/api/files/{job.id}", json={"uploader": "", "extra_links": []}).json() + assert (cleared["display_uploader"], cleared["extra_links"]) == (None, []) + + +def test_the_usage_carries_the_quota_and_the_upload_cap(db, make_user, as_user): + got = as_user(make_user()).get("/api/files/usage").json() + assert got["upload_max_bytes"] == sysconfig.get_int(db, "download_upload_max_bytes") + assert "footprint_bytes" in got and "max_bytes" in got + + +def test_a_file_moves_to_an_own_folder_or_home_and_nowhere_else(db, make_user, as_user): + alice, bob = make_user(), make_user() + job = _file(db, alice.id) + home = job.folder_id + vimeo = db.get(FileFolder, _file(db, alice.id, kind="url", ref="https://vimeo.com/4").folder_id) + own = _own(db, alice.id) + bobs = _own(db, bob.id, "Bobé") + c = as_user(alice) + moved = c.patch(f"/api/files/{job.id}", json={"folder_id": own.id}).json() + assert (moved["folder_id"], moved["expires_at"]) == (own.id, None) # an own folder keeps it + wrong = c.patch(f"/api/files/{job.id}", json={"folder_id": vimeo.id}) + assert wrong.json()["detail"]["reason"] == "file_folder_not_its_source" + foreign = c.patch(f"/api/files/{job.id}", json={"folder_id": bobs.id}) + assert foreign.json()["detail"]["reason"] == "file_folder_gone" + home_again = c.patch(f"/api/files/{job.id}", json={"folder_id": home}).json() + assert home_again["folder_id"] == home and home_again["expires_at"] is not None + assert [(a.before, a.after) for a in _audits(db, "file_move")] == [ + ({"folder_id": home}, {"folder_id": own.id}), + ({"folder_id": own.id}, {"folder_id": home}), + ] + + +def test_a_deleted_file_leaves_the_library_and_the_log_says_so(db, make_user, as_user, _root): + alice = make_user() + job = _file(db, alice.id) + assert as_user(make_user()).delete(f"/api/files/{job.id}").status_code == 404 + r = as_user(alice).delete(f"/api/files/{job.id}") + assert r.json() == {"ok": True, "deleted": job.id} + db.expire_all() + assert db.get(DownloadJob, job.id) is None + [row] = _audits(db, "file_delete") + assert (row.actor_id, row.target_id) == (alice.id, str(job.id)) + + +def test_a_file_shared_with_someone_is_theirs_to_watch_not_to_change(db, make_user, as_user, _root): + # The read routes let a share's recipient in; the write routes must not. + alice, friend = make_user(), make_user() + job = _file(db, alice.id, title="Mine") + db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id)) + db.commit() + c = as_user(friend) + assert c.patch(f"/api/files/{job.id}", json={"name": "Theirs"}).status_code == 404 + assert c.patch(f"/api/files/{job.id}", json={"kept": True}).status_code == 404 + assert c.delete(f"/api/files/{job.id}").status_code == 404 + db.expire_all() + still = db.get(DownloadJob, job.id) + assert still is not None and still.display_name is None and not still.kept + + +# --- upload, the file, its hash ------------------------------------------------------------ + + +@pytest.fixture +def _no_probe(monkeypatch): + # A file ffprobe cannot read: a generic file, done at once (no conversion). + monkeypatch.setattr(uploadmod.probe, "probe_json", lambda _p: None) + + +def test_an_upload_lands_in_the_folder_it_asked_for_with_its_hash( + db, make_user, as_user, _root, _no_probe, upload_headers +): + alice = make_user() + own = _own(db, alice.id) + body = b"hello files" + c = as_user(alice) + r = c.post("/api/files/upload", content=body, headers=upload_headers("a.txt", folder=own.id)) + assert r.status_code == 200, r.text + job = db.get(DownloadJob, r.json()["id"]) + assert job.folder_id == own.id and job.expires_at is None + asset = db.get(MediaAsset, job.asset_id) + assert asset.sha256 == hashlib.sha256(body).hexdigest() + plain = c.post("/api/files/upload", content=b"x", headers=upload_headers("b.txt")).json() + assert db.get(FileFolder, plain["folder_id"]).source_key == "upload" + assert len(_audits(db, "file_upload")) == 2 + + +@pytest.mark.parametrize( + ("which", "reason"), + [ + ("site", "file_folder_not_its_source"), # YouTube's folder: no upload came from there + ("foreign", "file_folder_gone"), + ("garbage", "file_folder_gone"), + ], +) +def test_an_upload_to_a_folder_it_may_not_use_is_refused_before_its_body_is_read( + db, make_user, as_user, _root, _no_probe, upload_headers, monkeypatch, which, reason +): + from app.routes import files as files_routes + + alice, bob = make_user(), make_user() + folder = { + "site": lambda: _file(db, alice.id).folder_id, + "foreign": lambda: _own(db, bob.id).id, + "garbage": lambda: "12abc", + }[which]() + streamed = [] + + async def spy(*args, **kwargs): + streamed.append(True) + raise AssertionError("the body was read") + + monkeypatch.setattr(files_routes, "capped_stream_to", spy) + before = db.execute(select(DownloadJob.id)).all() + r = as_user(alice).post( + "/api/files/upload", content=b"data", headers=upload_headers("a.txt", folder=folder) + ) + assert (r.status_code, r.json()["detail"]["reason"]) == (409, reason) + assert streamed == [] + assert db.execute(select(DownloadJob.id)).all() == before + + +def test_an_upload_holds_its_folder_only_once_its_file_is_about_to_be_in_it( + db, make_user, as_user, _root, upload_headers, monkeypatch +): + # Held from the start, the folder waited out the probe and the poster (up to a minute) before a + # rename or a delete of it could go on; not held at all, a delete could take it from under the file. + alice = make_user() + own = _own(db, alice.id) + steps: list[object] = [] + real = folders.own_folder + + def own_folder(db_, user_id, folder_id, *, hold=True): + steps.append(("folder", hold)) + return real(db_, user_id, folder_id, hold=hold) + + def probe(_p): + steps.append("probe") + + monkeypatch.setattr(folders, "own_folder", own_folder) + monkeypatch.setattr(uploadmod.probe, "probe_json", probe) + r = as_user(alice).post( + "/api/files/upload", content=b"x", headers=upload_headers("a.txt", folder=own.id) + ) + assert r.status_code == 200, r.text + # The route's own look before the body, the ingest's look, the probe, then the hold. + assert steps[-3:] == [("folder", False), "probe", ("folder", True)] + + +def test_a_file_moved_holds_the_folder_it_goes_to(db, make_user, as_user, monkeypatch): + alice = make_user() + job = _file(db, alice.id) + own = _own(db, alice.id) + holds: list[bool] = [] + real = folders.own_folder + + def own_folder(db_, user_id, folder_id, *, hold=True): + holds.append(hold) + return real(db_, user_id, folder_id, hold=hold) + + monkeypatch.setattr(folders, "own_folder", own_folder) + r = as_user(alice).patch(f"/api/files/{job.id}", json={"folder_id": own.id}) + assert r.status_code == 200, r.text + assert holds == [True] + + +def test_a_folder_deleted_while_its_upload_streams_refuses_the_upload( + db, make_user, as_user, _root, _no_probe, upload_headers, monkeypatch +): + from app.routes import files as files_routes + + alice = make_user() + own = _own(db, alice.id) + real = files_routes.capped_stream_to + + async def deleting_meanwhile(*args, **kwargs): + size = await real(*args, **kwargs) + db.delete(db.get(FileFolder, own.id)) + db.commit() + return size + + monkeypatch.setattr(files_routes, "capped_stream_to", deleting_meanwhile) + r = as_user(alice).post( + "/api/files/upload", content=b"data", headers=upload_headers("a.txt", folder=own.id) + ) + assert (r.status_code, r.json()["detail"]["reason"]) == (409, "file_folder_gone") + assert db.execute(select(DownloadJob.id).where(DownloadJob.user_id == alice.id)).all() == [] + assert not (_root / ".uploads").exists() # checked before the file was placed + + +def test_the_file_carries_its_hash_once_known_and_the_hash_is_taken_on_first_ask( + db, make_user, as_user, _root +): + alice, friend = make_user(), make_user() + body = b"\x00\x01video bytes" + job = _file(db, alice.id, rel="v.mp4") + (_root / "v.mp4").write_bytes(body) + c = as_user(alice) + first = c.get(f"/api/files/{job.id}/file") + assert first.status_code == 200 and first.content == body + assert "repr-digest" not in first.headers # not hashed yet: nothing claimed + want = hashlib.sha256(body).hexdigest() + assert c.get(f"/api/files/{job.id}/sha256").json() == {"sha256": want} + assert db.get(MediaAsset, job.asset_id).sha256 == want + ranged = c.get(f"/api/files/{job.id}/file", headers={"Range": "bytes=0-3"}) + assert ranged.status_code == 206 + digest = base64.b64encode(bytes.fromhex(want)).decode() + assert ranged.headers["repr-digest"] == f"sha-256=:{digest}:" + # A friend it was shared with may read it too; a stranger is told it does not exist. + db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id)) + db.commit() + assert as_user(friend).get(f"/api/files/{job.id}/sha256").json() == {"sha256": want} + assert as_user(make_user()).get(f"/api/files/{job.id}/sha256").status_code == 404 + + +def test_only_a_few_files_are_hashed_at_once(db, make_user, as_user, _root): + # Each hash holds a thread of the pool every sync route shares for the whole read; a caller + # past the few that may run is told to come back, not queued behind them. + from app.routes import files as filesroute + + alice = make_user() + job = _file(db, alice.id, rel="v.mp4") + (_root / "v.mp4").write_bytes(b"bytes") + held = [filesroute._HASHING.acquire(blocking=False) for _ in range(filesroute._HASHERS)] + try: + assert all(held) + busy = as_user(alice).get(f"/api/files/{job.id}/sha256") + finally: + for _ in held: + filesroute._HASHING.release() + assert busy.status_code == 429 + assert busy.json()["detail"]["reason"] == "file_hash_busy" + assert as_user(alice).get(f"/api/files/{job.id}/sha256").status_code == 200 + + +def test_a_new_file_at_an_assets_path_forgets_the_old_hash(db, make_user): + job = _file(db, make_user().id) + asset = db.get(MediaAsset, job.asset_id) + asset.sha256 = "ab" * 32 + asset.rel_path = "converted.mp4" + assert asset.sha256 is None + + +# --- the queue (the tray's poll) ------------------------------------------------------------------ + + +def test_the_queue_is_what_is_in_flight_and_what_just_finished_not_the_library( + db, make_user, as_user +): + # The tray polls this on every page: a finished file is the library's, so it is listed only + # while the tray may still be showing it done. + from sqlalchemy import update + + user = make_user() + old = _file(db, user.id, title="Old") + fresh = _file(db, user.id, title="Fresh") + waiting = _file(db, user.id, title="Waiting", status=JobStatus.QUEUED) + stuck = _file(db, user.id, title="Stuck", status=JobStatus.ERROR) + paused = _file(db, user.id, title="Paused", status=JobStatus.PAUSED) + stopped = _file(db, user.id, title="Stopped", status=JobStatus.ERROR) + dropped = _file(db, user.id, title="Dropped", status=JobStatus.CANCELED) + for job in (old, stopped, dropped): + db.execute( + update(DownloadJob) + .where(DownloadJob.id == job.id) + .values(updated_at=NOW - timedelta(hours=1), ready_at=NOW - timedelta(hours=1)) + ) + db.commit() + rows = as_user(user).get("/api/downloads").json() + # An old failure still waits for the user; an old finished or cancelled job does not. + assert {row["id"] for row in rows} == {fresh.id, waiting.id, stuck.id, paused.id, stopped.id} + # The tray names where each came from, as the library does. + assert {row["source_key"] for row in rows} == {"youtube"} + + +def test_an_old_file_written_to_does_not_come_back_as_just_finished(db, make_user, as_user): + # Renaming, keeping or moving touches the row; the tray would announce the file as ready again. + from sqlalchemy import update + + user = make_user() + old = _file(db, user.id, title="Old") + db.execute( + update(DownloadJob) + .where(DownloadJob.id == old.id) + .values(updated_at=NOW - timedelta(days=30), ready_at=NOW - timedelta(days=30)) + ) + db.commit() + c = as_user(user) + assert ( + c.patch(f"/api/files/{old.id}", json={"name": "Renamed", "kept": True}).status_code == 200 + ) + assert [row["id"] for row in c.get("/api/downloads").json()] == [] diff --git a/backend/tests/test_files_preview.py b/backend/tests/test_files_preview.py new file mode 100644 index 00000000..904ede98 --- /dev/null +++ b/backend/tests/test_files_preview.py @@ -0,0 +1,174 @@ +"""A library file's preview (`GET /api/files/{id}/preview`, `downloads/preview.py`), DB-lane. + +What it may show is an allow-list on the stored extension: an image or a PDF inline, a text format +(HTML and SVG among them) as plain text, its head only. Every answer is sandboxed and may be framed +by the app alone; anything else, a binary "text" included, has no preview. +""" + +import pytest +from sqlalchemy import select + +from app.config import settings +from app.downloads import folders, preview, service +from app.downloads.status import JobStatus +from app.models import DownloadJob, DownloadShare, MediaAsset +from app.routes.files import TRUNCATED_HEADER + + +@pytest.fixture +def _root(tmp_path, monkeypatch): + monkeypatch.setattr(settings, "files_root", str(tmp_path)) + return tmp_path + + +def _upload(db, root, user_id, ext, body, *, media_kind="file", title="notes"): + """A done uploaded file named `title.ext` holding `body`.""" + n = len(db.execute(select(MediaAsset.id)).all()) + rel = f"up{n}.{ext}" if ext else f"up{n}" + (root / rel).write_bytes(body) + asset = MediaAsset( + source_kind="upload", + source_ref=f"up{n}", + format_sig=f"sig-up{n}", + status="ready", + rel_path=rel, + size_bytes=len(body), + ref_count=0, + title=title, + media_kind=media_kind, + container=ext, + ) + db.add(asset) + db.flush() + job = DownloadJob( + user_id=user_id, + asset_id=asset.id, + source_kind="upload", + source_ref=f"up{n}", + profile_snapshot={}, + status=JobStatus.DONE, + ) + db.add(job) + db.flush() + folders.place(db, job) + service.incr_ref_count(db, asset.id) + db.commit() + return job + + +def _assert_sandboxed(r): + assert r.headers["content-security-policy"] == "sandbox; frame-ancestors 'self'" + assert r.headers["x-frame-options"] == "SAMEORIGIN" + assert r.headers["x-content-type-options"] == "nosniff" + + +@pytest.mark.parametrize( + "ext,ctype", + [ + ("png", "image/png"), + ("JPG", "image/jpeg"), + ("webp", "image/webp"), + ("pdf", "application/pdf"), + ], +) +def test_an_image_or_a_pdf_is_shown_inline_with_its_listed_type_in_a_sandbox( + db, make_user, as_user, _root, ext, ctype +): + alice = make_user() + job = _upload(db, _root, alice.id, ext, b"bytes of it") + r = as_user(alice).get(f"/api/files/{job.id}/preview") + assert r.status_code == 200 and r.content == b"bytes of it" + assert r.headers["content-type"] == ctype + assert r.headers["content-disposition"].startswith("inline;") + assert f'filename="notes.{ext}"' in r.headers["content-disposition"] + _assert_sandboxed(r) + + +@pytest.mark.parametrize("ext", ["html", "svg", "txt", "py"]) +def test_html_svg_and_code_arrive_as_plain_text_never_as_a_page(db, make_user, as_user, _root, ext): + alice = make_user() + body = b"" + job = _upload(db, _root, alice.id, ext, body) + r = as_user(alice).get(f"/api/files/{job.id}/preview") + assert r.status_code == 200 and r.content == body + assert r.headers["content-type"] == "text/plain; charset=utf-8" + assert TRUNCATED_HEADER.lower() not in r.headers + _assert_sandboxed(r) + + +def test_a_long_text_is_cut_to_its_head_and_says_so(db, make_user, as_user, _root): + alice = make_user() + body = b"a" * preview.TEXT_PREVIEW_BYTES + b"tail" + job = _upload(db, _root, alice.id, "log", body) + r = as_user(alice).get(f"/api/files/{job.id}/preview") + assert r.status_code == 200 and len(r.content) == preview.TEXT_PREVIEW_BYTES + assert r.headers[TRUNCATED_HEADER] == "1" + exact = _upload(db, _root, alice.id, "log", b"a" * preview.TEXT_PREVIEW_BYTES) + whole = as_user(alice).get(f"/api/files/{exact.id}/preview") + assert len(whole.content) == preview.TEXT_PREVIEW_BYTES + assert TRUNCATED_HEADER.lower() not in whole.headers + + +@pytest.mark.parametrize( + "ext,body,media_kind", + [ + ("zip", b"PK\x03\x04", "file"), # not on the list + ("docx", b"PK\x03\x04", "file"), + (None, b"plain", "file"), # no extension + ("txt", b"text\x00with a nul", "file"), # a "text" that is binary + ("mp4", b"video", "video"), # media plays in the player instead + ("png", b"png", "video"), # an extension alone never makes media a preview + ], +) +def test_anything_else_has_no_preview(db, make_user, as_user, _root, ext, body, media_kind): + alice = make_user() + job = _upload(db, _root, alice.id, ext, body, media_kind=media_kind) + r = as_user(alice).get(f"/api/files/{job.id}/preview") + assert r.status_code == 415 + assert r.json()["detail"] == {"code": "refused", "reason": "file_no_preview"} + + +@pytest.mark.parametrize( + "body", + [ + "Árvíztűrő tükörfúrógép".encode("cp1250"), # an old Hungarian subtitle + "Árvíztűrő tükörfúrógép".encode("utf-8-sig"), + "Árvíztűrő tükörfúrógép".encode("utf-16"), # a BOM, then NULs: text all the same + "Árvíztűrő tükörfúrógép".encode("utf-8"), + ], +) +def test_a_text_arrives_as_utf8_whatever_it_was_written_in(db, make_user, as_user, _root, body): + alice = make_user() + job = _upload(db, _root, alice.id, "srt", body) + r = as_user(alice).get(f"/api/files/{job.id}/preview") + assert r.status_code == 200 + assert r.content.decode("utf-8") == "Árvíztűrő tükörfúrógép" + + +def test_a_character_cut_at_the_cap_keeps_the_text_utf8(db, make_user, as_user, _root): + alice = make_user() + body = b"a" * (preview.TEXT_PREVIEW_BYTES - 1) + "ő".encode() # its 2nd byte past the cap + job = _upload(db, _root, alice.id, "txt", body) + r = as_user(alice).get(f"/api/files/{job.id}/preview") + assert r.content == body[: preview.TEXT_PREVIEW_BYTES] # not read as cp1250 + + +def test_the_list_says_which_preview_each_file_gets(db, make_user, as_user, _root): + alice = make_user() + for ext in ["png", "pdf", "html", "zip"]: + _upload(db, _root, alice.id, ext, b"x", title=ext) + _upload(db, _root, alice.id, "mp4", b"x", media_kind="video", title="mp4") + got = { + f["asset"]["container"]: f["asset"]["preview"] + for f in as_user(alice).get("/api/files").json() + } + assert got == {"png": "image", "pdf": "pdf", "html": "text", "zip": None, "mp4": None} + + +def test_a_preview_is_the_holders_or_a_friends_it_was_shared_with(db, make_user, as_user, _root): + alice, friend = make_user(), make_user() + job = _upload(db, _root, alice.id, "png", b"img") + assert as_user(make_user()).get(f"/api/files/{job.id}/preview").status_code == 404 + db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id)) + db.commit() + assert as_user(friend).get(f"/api/files/{job.id}/preview").status_code == 200 diff --git a/backend/tests/test_library_progress.py b/backend/tests/test_library_progress.py index 12228e39..3c033f05 100644 --- a/backend/tests/test_library_progress.py +++ b/backend/tests/test_library_progress.py @@ -9,13 +9,26 @@ cannot reach, is ever read or written. DB-backed (real Postgres). import pytest from sqlalchemy import select -from app.models import DownloadJob, DownloadShare, LibraryState +from app.models import DownloadJob, DownloadShare, LibraryState, MediaAsset from app.routes._common import ProgressIn, kept_position def _done_job(db, user) -> DownloadJob: + asset = MediaAsset( + source_kind="url", + source_ref="https://example.com/a.mp4", + format_sig=f"sig-{user.id}", + status="ready", + rel_path=f"a-{user.id}.mp4", + ref_count=1, + media_kind="video", + container="mp4", + ) + db.add(asset) + db.flush() job = DownloadJob( user_id=user.id, + asset_id=asset.id, source_kind="url", source_ref="https://example.com/a.mp4", status="done", @@ -59,7 +72,7 @@ def test_a_checkpoint_is_listed_back_on_the_item(db, make_user, as_user): "position_seconds": 754, } - [item] = c.get("/api/downloads").json() + [item] = c.get("/api/files").json() assert item["position_seconds"] == 754 # A second checkpoint moves it rather than adding a row. assert _checkpoint(c, job.id, 900) == 900 @@ -69,7 +82,7 @@ def test_a_checkpoint_is_listed_back_on_the_item(db, make_user, as_user): def test_an_item_never_played_lists_zero(db, make_user, as_user): me = make_user() _done_job(db, me) - [item] = as_user(me).get("/api/downloads").json() + [item] = as_user(me).get("/api/files").json() assert item["position_seconds"] == 0 @@ -110,8 +123,11 @@ def test_a_recipient_keeps_their_own_place_in_a_shared_item(db, make_user, as_us assert _checkpoint(as_user(owner), job.id, 100) == 100 assert _checkpoint(as_user(friend), job.id, 2000) == 2000 - assert as_user(owner).get("/api/downloads").json()[0]["position_seconds"] == 100 - assert as_user(friend).get("/api/downloads/shared").json()[0]["position_seconds"] == 2000 + assert as_user(owner).get("/api/files").json()[0]["position_seconds"] == 100 + assert ( + as_user(friend).get("/api/files", params={"scope": "shared"}).json()[0]["position_seconds"] + == 2000 + ) def test_an_item_the_caller_cannot_play_is_a_404_and_writes_nothing(db, make_user, as_user): diff --git a/backend/tests/test_migration_0102.py b/backend/tests/test_migration_0102.py new file mode 100644 index 00000000..a2a40512 --- /dev/null +++ b/backend/tests/test_migration_0102.py @@ -0,0 +1,141 @@ +"""Migration 0102's data step, DB-lane: every existing job lands in its source folder, and a done +file's expiry moves from the shared asset to the user's job. + +The schema half is checked by test_migrations.py; a row placed in the wrong folder is invisible there. +""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def test_existing_jobs_are_placed_and_keep_their_expiry(db, make_user): + alice, bob = make_user().id, make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0101_plex_text_plain") + try: + with engine.begin() as conn: + + def asset(kind, ref, expires=None): + return conn.execute( + text( + "INSERT INTO media_assets (source_kind, source_ref, format_sig, status," + " ref_count, expires_at) VALUES (:k, :r, :sig, 'ready', 1, :e)" + " RETURNING id" + ), + {"k": kind, "r": ref, "sig": kind + ref, "e": expires}, + ).scalar_one() + + def job(user, kind, ref, asset_id, status="done", spec=None, parent=None): + return conn.execute( + text( + "INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref," + " profile_snapshot, status, edit_spec, source_job_id)" + " VALUES (:u, :a, :k, :r, '{}', :s, CAST(:spec AS json), :p) RETURNING id" + ), + {"u": user, "a": asset_id, "k": kind, "r": ref, "s": status, + "spec": spec, "p": parent}, + ).scalar_one() # fmt: skip + + yt = asset("youtube", "abcdefghijk", "2030-01-02T00:00:00+00:00") + tt = asset("url", "https://www.tiktok.com/@a/video/1", "2030-01-03T00:00:00+00:00") + up = asset("upload", "u1") + cl = asset("clip", "c1", "2030-01-04T00:00:00+00:00") + j_yt = job(alice, "youtube", "abcdefghijk", yt) + j_yt_bob = job(bob, "youtube", "abcdefghijk", yt, status="queued") + j_tt = job(alice, "url", "https://www.tiktok.com/@a/video/1", tt) + j_up = job(alice, "upload", "u1", up) + j_cl = job(alice, "clip", "c1", cl, spec='{"source": "plex", "ref": "9"}') + j_ed = job(alice, "edit", "e1", cl, parent=j_tt) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + rows = conn.execute( + text( + "SELECT j.id, f.user_id, f.source_key, f.retention_days, j.expires_at::date" + " FROM download_jobs j JOIN file_folders f ON f.id = j.folder_id" + ) + ).all() + folders = conn.execute(text("SELECT count(*) FROM file_folders")).scalar() + got = {r[0]: (r[1], r[2], r[3], str(r[4]) if r[4] else None) for r in rows} + assert got == { + j_yt: (alice, "youtube", 30, "2030-01-02"), + j_yt_bob: (bob, "youtube", 30, None), # not done yet: no expiry until it is + j_tt: (alice, "site:tiktok.com", 30, "2030-01-03"), + j_up: (alice, "upload", None, None), # an upload never expires + j_cl: (alice, "plex", 30, "2030-01-04"), + j_ed: (alice, "site:tiktok.com", 30, "2030-01-04"), + } + # alice: youtube, tiktok, upload, plex; bob: youtube. + assert folders == 5 + + +def test_the_admins_retention_is_kept_and_the_downgrade_gives_the_asset_back_its_expiry( + db, make_user +): + alice, bob = make_user().id, make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0101_plex_text_plain") + try: + with engine.begin() as conn: + conn.execute( + text( + "INSERT INTO system_config (key, value) VALUES ('download_retention_days', '7')" + " ON CONFLICT (key) DO UPDATE SET value = '7'" + ) + ) + yt = conn.execute( + text( + "INSERT INTO media_assets (source_kind, source_ref, format_sig, status," + " ref_count, expires_at) VALUES ('youtube', 'abcdefghijk', 'sig', 'ready', 2," + " '2030-01-02T00:00:00+00:00') RETURNING id" + ) + ).scalar_one() + for user in (alice, bob): + conn.execute( + text( + "INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref," + " profile_snapshot, status) VALUES (:u, :a, 'youtube', 'abcdefghijk'," + " '{}', 'done')" + ), + {"u": user, "a": yt}, + ) + command.upgrade(cfg, "head") + with engine.connect() as conn: + retention = conn.execute( + text( + "SELECT DISTINCT retention_days FROM file_folders WHERE source_key = 'youtube'" + ) + ).all() + + def asset_expiry_after_downgrade(alice_at, bob_at): + with engine.begin() as conn: + for user, at in ((alice, alice_at), (bob, bob_at)): + conn.execute( + text( + "UPDATE download_jobs SET expires_at = CAST(:at AS timestamptz)" + " WHERE asset_id = :a AND user_id = :u" + ), + {"at": at, "a": yt, "u": user}, + ) + command.downgrade(cfg, "0101_plex_text_plain") + with engine.connect() as conn: + got = conn.execute( + text("SELECT expires_at::date FROM media_assets WHERE id = :a"), {"a": yt} + ).scalar() + command.upgrade(cfg, "head") + return str(got) if got else None + + # The shared file goes when its last holder's copy would; one holder keeping it keeps it. + latest = asset_expiry_after_downgrade( + "2030-02-01T00:00:00+00:00", "2030-03-01T00:00:00+00:00" + ) + kept_by_one = asset_expiry_after_downgrade("2030-02-01T00:00:00+00:00", None) + finally: + command.upgrade(cfg, "head") + assert retention == [(7,)] + assert latest == "2030-03-01" + assert kept_by_one is None diff --git a/backend/tests/test_migration_0103.py b/backend/tests/test_migration_0103.py new file mode 100644 index 00000000..0514b45d --- /dev/null +++ b/backend/tests/test_migration_0103.py @@ -0,0 +1,62 @@ +"""Migration 0103's data step, DB-lane: the fileless "expired" rows of the old retention go, every +other job stays. + +An expired file now leaves the library (`downloads.gc`); before, it stayed as a done job with no +asset. Left in place, those rows would sit unseen in no listing forever. +""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def test_the_fileless_expired_rows_go_and_every_other_job_stays(db, make_user): + uid = make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0102_file_folders") + try: + with engine.begin() as conn: + folder = conn.execute( + text( + "INSERT INTO file_folders (user_id, kind, source_key, retention_days)" + " VALUES (:u, 'source', 'youtube', 30) RETURNING id" + ), + {"u": uid}, + ).scalar_one() + asset = conn.execute( + text( + "INSERT INTO media_assets (source_kind, source_ref, format_sig, status, ref_count)" + " VALUES ('youtube', 'abcdefghijk', 'sig', 'ready', 1) RETURNING id" + ) + ).scalar_one() + + def job(status, asset_id): + return conn.execute( + text( + "INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref," + " profile_snapshot, status, folder_id) VALUES (:u, :a, 'youtube'," + " 'abcdefghijk', '{}', :s, :f) RETURNING id" + ), + {"u": uid, "a": asset_id, "s": status, "f": folder}, + ).scalar_one() + + expired = job("done", None) + alive = job("done", asset) + failed = job("error", None) # a failed job has no file either, and is not expired + queued = job("queued", None) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + left = set(conn.execute(text("SELECT id FROM download_jobs")).scalars()) + has_hash = conn.execute( + text( + "SELECT count(*) FROM information_schema.columns" + " WHERE table_name = 'media_assets' AND column_name = 'sha256'" + ) + ).scalar() + assert left == {alive, failed, queued} + assert expired not in left + assert has_hash == 1 diff --git a/backend/tests/test_migration_0104.py b/backend/tests/test_migration_0104.py new file mode 100644 index 00000000..86f5ac2a --- /dev/null +++ b/backend/tests/test_migration_0104.py @@ -0,0 +1,42 @@ +"""Migration 0104's data step, DB-lane: a done file from before is dated by when it was asked for +(where it already sat); a job not done has no date.""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def test_a_done_file_takes_its_request_time(db, make_user): + uid = make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0103_files_sha256") + try: + with engine.begin() as conn: + + def job(status, created): + return conn.execute( + text( + "INSERT INTO download_jobs (user_id, source_kind, source_ref," + " profile_snapshot, status, created_at) VALUES (:u, 'url', 'x', '{}', :s," + " CAST(:c AS timestamptz)) RETURNING id" + ), + {"u": uid, "s": status, "c": created}, + ).scalar_one() + + done = job("done", "2026-10-02T10:00:00+00:00") + failed = job("error", "2026-10-03T10:00:00+00:00") + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + rows = dict( + conn.execute( + text( + "SELECT id, to_char(ready_at AT TIME ZONE 'UTC', 'MM-DD HH24') FROM download_jobs" + ) + ).all() + ) + assert rows[done] == "10-02 10" + assert rows[failed] is None diff --git a/backend/tests/test_migration_0105.py b/backend/tests/test_migration_0105.py new file mode 100644 index 00000000..937d6d01 --- /dev/null +++ b/backend/tests/test_migration_0105.py @@ -0,0 +1,45 @@ +"""Migration 0105's data step, DB-lane: a file in a source folder records that folder's key as its +origin; a file in an own folder is left to be worked out, as before.""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def test_a_file_in_a_source_folder_records_its_origin(db, make_user): + uid = make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0104_job_ready_at") + try: + with engine.begin() as conn: + + def folder(kind, key, name): + return conn.execute( + text( + "INSERT INTO file_folders (user_id, kind, source_key, name) VALUES" + " (:u, :k, :s, :n) RETURNING id" + ), + {"u": uid, "k": kind, "s": key, "n": name}, + ).scalar_one() + + def job(folder_id): + return conn.execute( + text( + "INSERT INTO download_jobs (user_id, source_kind, source_ref," + " profile_snapshot, status, folder_id) VALUES (:u, 'url', 'x', '{}'," + " 'queued', :f) RETURNING id" + ), + {"u": uid, "f": folder_id}, + ).scalar_one() + + in_source = job(folder("source", "site:vimeo.com", None)) + in_own = job(folder("own", None, "Mine")) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + rows = dict(conn.execute(text("SELECT id, source_key FROM download_jobs")).all()) + assert rows[in_source] == "site:vimeo.com" + assert rows[in_own] is None diff --git a/backend/tests/test_odd_frame.py b/backend/tests/test_odd_frame.py index 13f13561..78b1cffe 100644 --- a/backend/tests/test_odd_frame.py +++ b/backend/tests/test_odd_frame.py @@ -46,7 +46,7 @@ def test_an_odd_sized_4_4_4_video_converts_to_the_even_frame_below_it(tmp_path): def test_an_odd_crop_makes_a_clip_and_stores_the_frame_it_has(db, make_user, tmp_path, monkeypatch): - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) src = tmp_path / "in.mp4" _ffmpeg( "-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=2", diff --git a/backend/tests/test_phone_safe_contract.py b/backend/tests/test_phone_safe_contract.py index a15e1476..1a0f659c 100644 --- a/backend/tests/test_phone_safe_contract.py +++ b/backend/tests/test_phone_safe_contract.py @@ -127,7 +127,7 @@ _FILES = { def media_root(tmp_path, monkeypatch): root = tmp_path / "media" root.mkdir() - monkeypatch.setattr(settings, "download_root", str(root)) + monkeypatch.setattr(settings, "files_root", str(root)) return root @@ -141,9 +141,7 @@ def test_what_a_link_serves_passes_the_rule( subprocess.run(["ffmpeg", "-y", "-loglevel", "error", *args, str(src)], check=True) owner = as_user(make_user()) - r = owner.post( - "/api/downloads/upload", content=src.read_bytes(), headers=upload_headers(filename) - ) + r = owner.post("/api/files/upload", content=src.read_bytes(), headers=upload_headers(filename)) assert r.status_code == 200, r.text job_id = r.json()["id"] assert r.json()["status"] == "queued" # every one of these fails the rule as it came diff --git a/backend/tests/test_public_unlock_limit.py b/backend/tests/test_public_unlock_limit.py index d4fc12d6..ab14fe61 100644 --- a/backend/tests/test_public_unlock_limit.py +++ b/backend/tests/test_public_unlock_limit.py @@ -19,15 +19,13 @@ _PW = "open sesame" @pytest.fixture def unlock_url(as_user, make_user, tmp_path, monkeypatch, upload_headers): - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) video = {"codec_type": "video", "codec_name": "h264", "pix_fmt": "yuv420p"} audio = {"codec_type": "audio", "codec_name": "aac"} data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}} monkeypatch.setattr(probemod, "probe_json", lambda _p: data) owner = as_user(make_user()) - job = owner.post( - "/api/downloads/upload", content=b"fake-video", headers=upload_headers("clip.mp4") - ) + job = owner.post("/api/files/upload", content=b"fake-video", headers=upload_headers("clip.mp4")) assert job.status_code == 200, job.text link = owner.post(f"/api/downloads/{job.json()['id']}/links", json={"password": _PW}) assert link.status_code == 200, link.text diff --git a/backend/tests/test_route_conventions.py b/backend/tests/test_route_conventions.py index 819232b9..4ea0e820 100644 --- a/backend/tests/test_route_conventions.py +++ b/backend/tests/test_route_conventions.py @@ -46,7 +46,9 @@ SERIALIZERS = { "_collection_card", "_playlist_card", "_serialize", - "_serialize_job", + "_serialize_folder", + "serialize", + "serialize_job", "_serialize_profile", "_serialize_user", "_summary", diff --git a/backend/tests/test_spa_entries.py b/backend/tests/test_spa_entries.py index 7fab4d24..22e46475 100644 --- a/backend/tests/test_spa_entries.py +++ b/backend/tests/test_spa_entries.py @@ -27,14 +27,14 @@ def pages(tmp_path, monkeypatch): @pytest.fixture def link_token(as_user, make_user, tmp_path, monkeypatch, upload_headers): - monkeypatch.setattr(settings, "download_root", str(tmp_path / "media")) + monkeypatch.setattr(settings, "files_root", str(tmp_path / "media")) video = {"codec_type": "video", "codec_name": "h264", "pix_fmt": "yuv420p"} audio = {"codec_type": "audio", "codec_name": "aac"} data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}} monkeypatch.setattr(probemod, "probe_json", lambda _p: data) owner = as_user(make_user()) job = owner.post( - "/api/downloads/upload", + "/api/files/upload", content=b"fake-video", headers=upload_headers("clip.mp4", display_name="Holiday clip"), ) diff --git a/backend/tests/test_storage.py b/backend/tests/test_storage.py index 108c26bc..53d92b27 100644 --- a/backend/tests/test_storage.py +++ b/backend/tests/test_storage.py @@ -6,6 +6,7 @@ from datetime import date from app.downloads.storage import ( MediaMeta, download_filename, + files_root_problem, rel_path, safe_abs_path, sanitize, @@ -100,7 +101,7 @@ class TestDownloadFilename: class TestSafeAbsPath: - """The path-traversal guard: only paths that resolve INSIDE download_root AND exist are returned.""" + """The path-traversal guard: only paths that resolve INSIDE files_root AND exist are returned.""" def test_returns_the_path_for_a_file_inside_the_root(self, tmp_path: Path): (tmp_path / "sub").mkdir() @@ -174,3 +175,38 @@ def test_sidecars_keep_the_full_basename_when_the_title_has_a_dot(tmp_path: Path assert f"{stem}.nfo" in names, names assert f"{stem}.jpg" in names, names assert "Chan_-_2026-08-12_-_Sonka_Hazilag.nfo" not in names # the old truncated orphan + + +def _mounts(*points): + """A /proc/self/mountinfo table with these mount points (the fields around them are filler).""" + return "\n".join(f"{i} 1 0:{i} / {p} rw - ext4 /dev/x rw" for i, p in enumerate(points, 22)) + + +def test_a_files_root_on_the_containers_own_disk_is_refused(tmp_path): + # The image's empty /files, left unmounted by an old compose file: every file would 404 and + # every download would go with the container. + root = str(tmp_path) + plain = _mounts("/", "/proc", "/etc/hosts") + assert "not a mounted volume" in files_root_problem(root, plain, in_container=True) + assert "does not exist" in files_root_problem(str(tmp_path / "gone")) + # Its own mount, or inside one: the volume — whatever device it reports. On a storage driver that + # keeps the container's root on the host's disk (vfs) a volume has the same device as /. + assert files_root_problem(root, _mounts("/", root), in_container=True) is None + assert files_root_problem(root, _mounts("/", str(tmp_path.parent)), in_container=True) is None + # A mount point with a space in it, as the kernel escapes it. + spaced = tmp_path / "my files" + spaced.mkdir() + escaped = str(spaced).replace(" ", "\\040") + assert files_root_problem(str(spaced), _mounts("/", escaped), in_container=True) is None + # Outside a container a plain directory is a fine root. + assert files_root_problem(root, plain, in_container=False) is None + + +def test_the_old_name_of_the_files_root_still_finds_the_files(monkeypatch): + from app.config import Settings + + monkeypatch.delenv("FILES_ROOT", raising=False) + monkeypatch.setenv("DOWNLOAD_ROOT", "/downloads") + assert Settings().files_root == "/downloads" + monkeypatch.setenv("FILES_ROOT", "/files") + assert Settings().files_root == "/files" diff --git a/backend/tests/test_upload.py b/backend/tests/test_upload.py index a240e67f..90f11046 100644 --- a/backend/tests/test_upload.py +++ b/backend/tests/test_upload.py @@ -5,7 +5,7 @@ generic file (force-download so uploaded HTML/SVG can't execute inline), quota a size known up front, the per-file cap enforced while streaming, filename sanitization, and the two guards that keep an upload out of paths that assume a re-fetchable source (edit / redownload). -DB-backed (real Postgres) — skipped when no database is reachable. `settings.download_root` is +DB-backed (real Postgres) — skipped when no database is reachable. `settings.files_root` is pointed at a per-test tmp dir so ingest's on-disk moves are isolated. The ingest/route tests monkeypatch the shared `probe.probe_json` so they don't shell out to ffprobe; the classifier's OWN parsing (image → file, cover-art skip, dims, duration) is covered directly by `TestProbeClassifier` @@ -51,7 +51,7 @@ def test_derive_media_kind(): @pytest.fixture def _root(tmp_path, monkeypatch): - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) return tmp_path @@ -212,7 +212,7 @@ def test_an_anonymous_upload_is_refused_before_a_byte_of_it_is_read(client): multipart the old route would have spooled, not the raw body the new one expects.""" body, pulled = _counting_multipart() r = client.post( - "/api/downloads/upload", + "/api/files/upload", content=body, headers={"content-type": f"multipart/form-data; boundary={_BOUNDARY}"}, ) @@ -227,7 +227,7 @@ def test_an_authenticated_upload_still_has_its_body_read(as_user, make_user, _ro body, pulled = _counting_body() c = as_user(make_user()) r = c.post( - "/api/downloads/upload", + "/api/files/upload", content=body, headers={"content-type": "application/octet-stream", "x-upload-filename": "clip.bin"}, ) @@ -281,7 +281,7 @@ def test_upload_route_generic_forces_download( _probe(monkeypatch, (None, None, None, None, None)) c = as_user(make_user()) r = c.post( - "/api/downloads/upload", + "/api/files/upload", content=b"", headers=upload_headers("report.html"), ) @@ -290,7 +290,7 @@ def test_upload_route_generic_forces_download( assert job["asset"]["media_kind"] == "file" assert job["can_download"] is True # The served file must be a neutral force-download attachment — never inline text/html. - fr = c.get(f"/api/downloads/{job['id']}/file") + fr = c.get(f"/api/files/{job['id']}/file") assert fr.status_code == 200 assert fr.headers["content-type"].startswith("application/octet-stream") assert "attachment" in fr.headers["content-disposition"].lower() @@ -307,7 +307,7 @@ def test_upload_route_media_html_extension_not_served_as_html( _probe(monkeypatch, ("h264", "aac", 320, 240, 5)) # ffprobe says it's video… c = as_user(make_user()) r = c.post( - "/api/downloads/upload", + "/api/files/upload", content=b"", headers=upload_headers("evil.html"), ) @@ -320,11 +320,11 @@ def test_upload_route_media_html_extension_not_served_as_html( assert job["status"] == "queued" row = db.get(DownloadJob, job["id"]) db.get(MediaAsset, row.asset_id).status = "ready" - row.status = "done" + row.status, row.ready_at = "done", row.created_at db.commit() # Authenticated endpoint: never text/html. - fr = c.get(f"/api/downloads/{job['id']}/file") + fr = c.get(f"/api/files/{job['id']}/file") assert fr.status_code == 200 assert fr.headers["content-type"].startswith("application/octet-stream") assert "attachment" in fr.headers["content-disposition"].lower() @@ -348,7 +348,7 @@ def test_public_watch_file_forces_download_for_generic_file( _probe(monkeypatch, (None, None, None, None, None)) c = as_user(make_user()) job = c.post( - "/api/downloads/upload", + "/api/files/upload", content=b"", headers=upload_headers("notes.html"), ).json() @@ -369,7 +369,7 @@ def test_public_watch_file_serves_media_inline_with_real_type( _probe(monkeypatch, ("h264", "aac", 640, 360, 12)) c = as_user(make_user()) job = c.post( - "/api/downloads/upload", + "/api/files/upload", content=b"fake-mp4-bytes", headers=upload_headers("clip.mp4"), ).json() @@ -389,7 +389,7 @@ def test_upload_route_enforces_per_file_cap( db.commit() c = as_user(make_user()) r = c.post( - "/api/downloads/upload", + "/api/files/upload", content=b"way over the cap", headers=upload_headers("big.bin"), ) @@ -412,14 +412,10 @@ def test_the_cap_admits_a_body_of_EXACTLY_the_cap_and_refuses_one_byte_more( db.commit() c = as_user(make_user()) - at_the_cap = c.post( - "/api/downloads/upload", content=payload, headers=upload_headers("exact.bin") - ) + at_the_cap = c.post("/api/files/upload", content=payload, headers=upload_headers("exact.bin")) assert at_the_cap.status_code == 200, at_the_cap.text - over = c.post( - "/api/downloads/upload", content=payload + b"x", headers=upload_headers("over.bin") - ) + over = c.post("/api/files/upload", content=payload + b"x", headers=upload_headers("over.bin")) assert over.status_code == 422 assert over.json()["detail"] == {"code": "upload_too_large", "limit": len(payload)} @@ -437,9 +433,7 @@ def test_a_non_ascii_filename_survives_the_header_round_trip( """ _probe(monkeypatch, (None, None, None, None, None)) c = as_user(make_user()) - r = c.post( - "/api/downloads/upload", content=b"data", headers=upload_headers("nyaralás ősszel.bin") - ) + r = c.post("/api/files/upload", content=b"data", headers=upload_headers("nyaralás ősszel.bin")) assert r.status_code == 200, r.text asset = db.execute(select(MediaAsset).order_by(MediaAsset.id.desc())).scalars().first() assert asset is not None @@ -467,9 +461,7 @@ def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one( c = as_user(make_user()) # 1. the upload route — the name arrives in a request header. - up = c.post( - "/api/downloads/upload", content=b"video", headers=upload_headers("a.mp4", long_name) - ) + up = c.post("/api/files/upload", content=b"video", headers=upload_headers("a.mp4", long_name)) assert up.status_code == 200, up.text assert up.json()["display_name"] == bound @@ -494,10 +486,10 @@ def test_a_long_display_name_is_truncated_on_EVERY_route_that_takes_one( assert edit.status_code == 200, edit.text assert edit.json()["display_name"] == bound - # 4 + 5. the metadata PATCH's two fields. + # 4 + 5. the library PATCH's two fields. patched = c.patch( - f"/api/downloads/{up.json()['id']}", - json={"display_name": long_name, "display_uploader": long_name}, + f"/api/files/{up.json()['id']}", + json={"name": long_name, "uploader": long_name}, ) assert patched.status_code == 200, patched.text assert patched.json()["display_name"] == bound @@ -584,7 +576,7 @@ def _display_write_sites() -> dict[tuple[str, str], set[str]]: # one value that needs no bound, and a clear is still a write. # service.enqueue / enqueue_edit / `display_name` is already bounded by its caller; the fallback # enqueue_clip slices the source's title to the same width. -# downloads.upload_download passes through `spool_upload`'s value, bounded there. +# files.ingest_spooled passes through `spool_upload`'s value, bounded there. # routes enqueue_clip the `display_name=` keyword passes on the name bound above it. # pipeline._download / _process_clip / worker._finish_from_cache # slice a fetched or stored title to the column's width. @@ -610,12 +602,12 @@ DISPLAY_WRITE_SITES = { ("routes/downloads.py", "enqueue_download"): {"display_text(body.display_name)"}, ("routes/downloads.py", "enqueue_edit"): {"display_text(body.display_name)"}, ("routes/downloads.py", "enqueue_clip"): {"display_text(body.display_name)", "display_name"}, - ("routes/downloads.py", "update_download_meta"): { - "display_text(body.display_name)", - "display_text(body.display_uploader)", + ("routes/files.py", "ingest_spooled"): {"upload.display_name"}, + ("routes/files.py", "update_file"): { + "display_text(body.name)", + "display_text(body.uploader)", }, - ("routes/downloads.py", "upload_download"): {"upload.display_name"}, - ("routes/downloads.py", "spool_upload"): { + ("routes/files.py", "spool_upload"): { "display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER))" }, } @@ -665,7 +657,7 @@ def test_the_display_name_header_reaches_the_job_and_is_bounded( c = as_user(make_user()) ok = c.post( - "/api/downloads/upload", + "/api/files/upload", content=b"data", headers=upload_headers("clip.bin", "Nyári vakáció"), ) @@ -674,7 +666,7 @@ def test_the_display_name_header_reaches_the_job_and_is_bounded( long_name = "n" * 300 bounded = c.post( - "/api/downloads/upload", content=b"data2", headers=upload_headers("clip2.bin", long_name) + "/api/files/upload", content=b"data2", headers=upload_headers("clip2.bin", long_name) ) assert bounded.status_code == 200, bounded.text assert bounded.json()["display_name"] == "n" * DISPLAY_TEXT_MAX @@ -694,12 +686,12 @@ def test_the_upload_holds_no_db_transaction_while_the_body_streams( the checkout is its consequence (measured), and the pool counter is shared with whatever else the test process is holding. """ - from app.routes import downloads as downloads_routes + from app.routes import files as files_routes _probe(monkeypatch, (None, None, None, None, None)) seen: dict[str, bool] = {} - real_stream = downloads_routes.capped_stream_to - real_release = downloads_routes.release_read_connection + real_stream = files_routes.capped_stream_to + real_release = files_routes.release_read_connection def spy_release(session): # The PRECONDITION, so this test cannot pass vacuously. Asserting only "no transaction @@ -714,10 +706,10 @@ def test_the_upload_holds_no_db_transaction_while_the_body_streams( seen["while_streaming"] = bool(db.in_transaction()) return await real_stream(request, out, limit, refusal) - monkeypatch.setattr(downloads_routes, "release_read_connection", spy_release) - monkeypatch.setattr(downloads_routes, "capped_stream_to", spy_stream) + monkeypatch.setattr(files_routes, "release_read_connection", spy_release) + monkeypatch.setattr(files_routes, "capped_stream_to", spy_stream) c = as_user(make_user()) - r = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("held.bin")) + r = c.post("/api/files/upload", content=b"data", headers=upload_headers("held.bin")) assert r.status_code == 200, r.text assert seen["before"] is True, ( "nothing had opened a transaction before the release, so this test proves nothing — the " @@ -739,14 +731,14 @@ def test_the_usage_route_reports_the_per_file_upload_cap(as_user, make_user, db) from app import sysconfig c = as_user(make_user()) - r = c.get("/api/downloads/usage") + r = c.get("/api/files/usage") assert r.status_code == 200, r.text assert r.json()["upload_max_bytes"] == sysconfig.get_int(db, "download_upload_max_bytes") # And it is the KNOB, not a constant: a lowered override must show through. sysconfig.set_value(db, "download_upload_max_bytes", 4096) db.commit() - assert c.get("/api/downloads/usage").json()["upload_max_bytes"] == 4096 + assert c.get("/api/files/usage").json()["upload_max_bytes"] == 4096 def test_a_refused_upload_leaves_nothing_in_the_staging_directory( @@ -761,10 +753,10 @@ def test_a_refused_upload_leaves_nothing_in_the_staging_directory( sysconfig.set_value(db, "download_upload_max_bytes", 8) db.commit() - staging = Path(settings.download_root) / ".staging" / "uploads" + staging = Path(settings.files_root) / ".staging" / "uploads" c = as_user(make_user()) - r = c.post("/api/downloads/upload", content=b"x" * 4096, headers=upload_headers("leaky.bin")) + r = c.post("/api/files/upload", content=b"x" * 4096, headers=upload_headers("leaky.bin")) assert r.status_code == 422 assert staging.exists(), "the dependency did not get as far as creating the staging dir" assert list(staging.iterdir()) == [], "a refused upload left its partial file behind" @@ -786,7 +778,7 @@ def test_upload_cap_cannot_be_disabled_with_zero(db): def test_upload_route_rejects_empty_file(as_user, make_user, _root, monkeypatch, upload_headers): c = as_user(make_user()) - r = c.post("/api/downloads/upload", content=b"", headers=upload_headers("empty.bin")) + r = c.post("/api/files/upload", content=b"", headers=upload_headers("empty.bin")) assert r.status_code == 400 assert r.json()["detail"]["code"] == "upload_empty" @@ -794,7 +786,7 @@ def test_upload_route_rejects_empty_file(as_user, make_user, _root, monkeypatch, def test_redownload_rejects_an_upload(as_user, make_user, _root, monkeypatch, upload_headers): _probe(monkeypatch, (None, None, None, None, None)) c = as_user(make_user()) - job = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("a.bin")).json() + job = c.post("/api/files/upload", content=b"data", headers=upload_headers("a.bin")).json() r = c.post(f"/api/downloads/{job['id']}/redownload") assert r.status_code == 409 @@ -804,7 +796,7 @@ def test_edit_rejects_a_generic_upload_source( ): _probe(monkeypatch, (None, None, None, None, None)) c = as_user(make_user()) - job = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("a.bin")).json() + job = c.post("/api/files/upload", content=b"data", headers=upload_headers("a.bin")).json() r = c.post( "/api/downloads/edit", json={"source_job_id": job["id"], "edit_spec": {"trim": {"start_s": 0, "end_s": 1}}}, diff --git a/backend/tests/test_upload_conversion.py b/backend/tests/test_upload_conversion.py index 8a8407f0..aeb850b9 100644 --- a/backend/tests/test_upload_conversion.py +++ b/backend/tests/test_upload_conversion.py @@ -29,7 +29,7 @@ from sqlalchemy.orm import Session @pytest.fixture def _root(tmp_path, monkeypatch): - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) return tmp_path @@ -407,7 +407,7 @@ def test_the_sweep_keeps_an_aged_upload_file_its_row_names_in_any_state( _probe(monkeypatch) job, asset = _ingest(db, _root, make_user().id) path = storage.abs_path(str(_root), asset.rel_path) - job.status, asset.status = job_status, asset_status + job.status, asset.status, job.ready_at = job_status, asset_status, job.created_at db.commit() _aged(path) @@ -427,7 +427,11 @@ def _route_moved(db, job, asset, job_status: str, ref_count: int) -> None: """What the cancel/pause/delete route leaves behind, written as SQL: the session's copy of `ref_count` is stale (the routes count it in SQL), so an ORM assignment of the same value would write nothing.""" - db.execute(update(DownloadJob).where(DownloadJob.id == job.id).values(status=job_status)) + db.execute( + update(DownloadJob) + .where(DownloadJob.id == job.id) + .values(status=job_status, ready_at=DownloadJob.created_at) + ) db.execute(update(MediaAsset).where(MediaAsset.id == asset.id).values(ref_count=ref_count)) db.commit() @@ -549,7 +553,7 @@ def test_a_link_says_preparing_only_while_its_video_is_being_made( _probe(monkeypatch, vcodec="h264") c = as_user(make_user()) job = c.post( - "/api/downloads/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4") + "/api/files/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4") ).json() link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json() assert c.get(f"/api/public/watch/{link['token']}").status_code == 200 @@ -570,7 +574,7 @@ def test_a_link_to_a_suspended_owners_waiting_rebuild_says_it_is_gone( owner = make_user() c = as_user(owner) job = c.post( - "/api/downloads/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4") + "/api/files/upload", content=b"h264 bytes", headers=upload_headers("clip.mp4") ).json() link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json() db.get(MediaAsset, job["asset"]["id"]).status = "pending" @@ -593,9 +597,45 @@ def test_the_admin_storage_view_counts_a_waiting_upload(db, as_user, make_user, _probe(monkeypatch) u = make_user() _ingest(db, _root, u.id, size=10) - r = as_user(make_user(role="admin")).get("/api/admin/downloads/storage").json() + admin = make_user(role="admin") + r = as_user(admin).get("/api/admin/downloads/storage").json() assert (r["files_on_disk"], r["total_bytes"]) == (1, 10) - assert {p["user_id"]: p["footprint_bytes"] for p in r["per_user"]} == {u.id: 10} + assert {p["user_id"]: p["footprint_bytes"] for p in r["per_user"]} == {u.id: 10, admin.id: 0} + + +def test_the_admin_storage_view_names_each_users_quota_and_files( + db, as_user, make_user, _root, monkeypatch +): + # A row says how full the user is against THEIR limit (a custom one, here), and how many files + # their Files lists — the finished ones, not one still waiting for its conversion. + _probe(monkeypatch) + u = make_user() + done, _ = _ingest(db, _root, u.id, size=10) + waiting, _ = _ingest(db, _root, u.id, name="b.mov", size=5) + done.status, waiting.status = "done", "queued" + done.ready_at = done.created_at + db.commit() + admin = as_user(make_user(role="admin")) + put = admin.put(f"/api/admin/downloads/quota/{u.id}", json={"max_bytes": 1000}) + assert put.status_code == 200 + rows = admin.get("/api/admin/downloads/storage").json()["per_user"] + mine = [(p["files"], p["max_bytes"], p["unlimited"]) for p in rows if p["user_id"] == u.id] + assert mine == [(1, 1000, False)] + # Every download names its owner, so its row can open THEIR quota. + jobs = admin.get("/api/admin/downloads").json() + assert {(j["id"], j["user_id"]) for j in jobs} == {(done.id, u.id), (waiting.id, u.id)} + + +def test_the_admin_storage_view_lists_every_user_even_one_without_a_file(db, as_user, make_user): + # The admin sets a quota BEFORE anyone downloads: a user holding nothing is still a row. + empty = make_user() + admin_user = make_user(role="admin") + admin = as_user(admin_user) + rows = admin.get("/api/admin/downloads/storage").json()["per_user"] + by_id = {p["user_id"]: p for p in rows} + assert {empty.id, admin_user.id} <= by_id.keys() + assert (by_id[empty.id]["email"], by_id[empty.id]["files"]) == (empty.email, 0) + assert by_id[empty.id]["footprint_bytes"] == 0 @pytest.mark.parametrize( diff --git a/backend/tests/test_worker_dispatch.py b/backend/tests/test_worker_dispatch.py index 8dc74b96..2593a306 100644 --- a/backend/tests/test_worker_dispatch.py +++ b/backend/tests/test_worker_dispatch.py @@ -32,7 +32,7 @@ def test_a_clip_the_phone_safe_step_changed_is_stored_under_its_new_extension( ): # A fast trim stream-copies its source's container; when the phone-safe step remuxes it (an # audio clip out of an .mp4 becomes an .m4a), the stored path and container follow the new file. - monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(settings, "files_root", str(tmp_path)) data = { "streams": [{"codec_type": "audio", "codec_name": "aac"}], "format": {"format_name": "mov,mp4", "duration": "10"}, diff --git a/backend/tests/test_worker_knobs.py b/backend/tests/test_worker_knobs.py index b423510d..c8cb192e 100644 --- a/backend/tests/test_worker_knobs.py +++ b/backend/tests/test_worker_knobs.py @@ -67,6 +67,7 @@ def test_worker_boots_with_the_stored_concurrency_and_log_level(monkeypatch, sto monkeypatch.setattr(worker, "_wait_for_schema", lambda: True) monkeypatch.setattr(worker.logsink, "start", lambda: None) monkeypatch.setattr(worker, "_recover_orphans", lambda: None) + monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: None) threading.Timer(0.5, worker._stop.set).start() worker.main() @@ -107,3 +108,17 @@ def test_an_unreadable_database_keeps_what_runs(monkeypatch, levels, loops): assert worker._target == 2 assert sorted(threads) == [0, 1] # the loops still run on the boot value assert levels == [] + + +def test_the_worker_will_not_start_on_a_files_root_that_is_not_the_volume(monkeypatch, loops): + # Files downloaded into the container's own empty /files go with the container: refuse, loudly. + monkeypatch.setattr(worker.settings, "worker_enabled", True) + monkeypatch.setattr(worker.logs, "configure_logging", lambda _source: None) + monkeypatch.setattr(worker.signal, "signal", lambda *_a: None) + monkeypatch.setattr(worker, "_wait_for_schema", lambda: True) + monkeypatch.setattr(worker.logsink, "start", lambda: None) + monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: "not a mounted volume") + with pytest.raises(SystemExit) as exit_: + worker.main() + assert exit_.value.code == 1 + assert loops == [] diff --git a/docker-compose.home.yml.example b/docker-compose.home.yml.example index 69c40b84..7ac42b60 100644 --- a/docker-compose.home.yml.example +++ b/docker-compose.home.yml.example @@ -17,7 +17,7 @@ # TOKEN_ENCRYPTION_KEY (Fernet), GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, # OAUTH_REDIRECT_URL=https://siftlode.example.com/auth/callback, ALLOWED_EMAILS, ADMIN_EMAILS, # YOUTUBE_API_KEY. APP_VERSION is exported by the deploy step (selects the image tag). -# Optional: DOWNLOAD_HOST_PATH (host dir for the download tree) and PLEX_MEDIA_HOST_PATH (host dir +# Optional: FILES_HOST_PATH (host dir for the download tree) and PLEX_MEDIA_HOST_PATH (host dir # holding the Plex library, mounted read-only for the optional Plex module). name: siftlode @@ -72,8 +72,8 @@ services: # The public instance owns the background scheduler (single writer). SCHEDULER_ENABLED: "true" # Download center: the API serves finished files + builds filmstrips (read-only root is fine — - # it only writes to the /downloads mount + tmpfs). The yt-dlp job loop runs in `worker` below. - DOWNLOAD_ROOT: /downloads + # it only writes to the /files mount + tmpfs). The yt-dlp job loop runs in `worker` below. + FILES_ROOT: /files WORKER_ENABLED: "false" depends_on: db: @@ -84,9 +84,9 @@ services: # (10.x.x.x:8080) and on the trusted home LAN. Never forwarded to the public internet. - "8080:8000" volumes: - # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env + # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env # to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there. - - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads + - ${FILES_HOST_PATH:-${DOWNLOAD_HOST_PATH:-siftlode_files}}:/files # Plex media, read-only. Only used when the optional Plex module is enabled: set # PLEX_MEDIA_HOST_PATH in .env to the host directory holding the Plex library (the local side # of plex_path_map, default /plex-media). Left unset → an empty named volume (harmless; the @@ -113,7 +113,7 @@ services: restart: unless-stopped # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the - # DB (job queue) and the downloads mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to + # DB (job queue) and the /files mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to # $HOME caches + the staging dir); heavier limits since it does the video encoding. worker: image: registry.example.com/you/siftlode:${APP_VERSION:-latest} @@ -124,7 +124,7 @@ services: DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-siftlode} # The API owns the scheduler; the worker must not also run it. SCHEDULER_ENABLED: "false" - DOWNLOAD_ROOT: /downloads + FILES_ROOT: /files WORKER_ENABLED: "true" depends_on: db: @@ -137,7 +137,7 @@ services: condition: service_started networks: [internal] volumes: - - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads + - ${FILES_HOST_PATH:-${DOWNLOAD_HOST_PATH:-siftlode_files}}:/files # Plex media (read-only) — same mount as the api; used by Plex clip jobs (a later phase). - ${PLEX_MEDIA_HOST_PATH:-siftlode_plex_media}:/plex-media:ro mem_limit: 2g @@ -167,7 +167,7 @@ services: volumes: siftlode_pgdata: - siftlode_downloads: + siftlode_files: # Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off). siftlode_plex_media: diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml index 202ca5b8..c2be617e 100644 --- a/docker-compose.localdev.yml +++ b/docker-compose.localdev.yml @@ -21,21 +21,21 @@ services: environment: # This instance owns its scheduler (its own DB, so no double-write concern). SCHEDULER_ENABLED: "true" - # Plex HLS remux scratch on fast container-local storage (the /downloads bind-mount is a slow + # Plex HLS remux scratch on fast container-local storage (the /files bind-mount is a slow # Windows Docker Desktop mount in dev; prod uses fast local disk so it keeps the default). PLEX_HLS_DIR: /var/tmp/plex-hls security_opt: - apparmor:unconfined volumes: # Downloaded media lives on the host so you can inspect the generated Plex tree. Gitignored. - - ./downloads:/downloads + - ./files:/files - plex_media:/plex-media:ro worker: security_opt: - apparmor:unconfined volumes: - - ./downloads:/downloads + - ./files:/files - plex_media:/plex-media:ro bgutil-pot: diff --git a/docker-compose.selfhost.yml b/docker-compose.selfhost.yml index d3087d9d..596266b9 100644 --- a/docker-compose.selfhost.yml +++ b/docker-compose.selfhost.yml @@ -48,8 +48,8 @@ services: # This instance owns the background scheduler (single writer). SCHEDULER_ENABLED: "true" # Download center: the API serves finished files + builds filmstrips; the worker (below) runs - # the yt-dlp job loop. Read-only root is fine — it only writes to the /downloads mount + tmpfs. - DOWNLOAD_ROOT: /downloads + # the yt-dlp job loop. Read-only root is fine — it only writes to the /files mount + tmpfs. + FILES_ROOT: /files WORKER_ENABLED: "false" depends_on: db: @@ -60,9 +60,9 @@ services: # front for HTTPS / public exposure — and set OAUTH_REDIRECT_URL to the https URL. - "${HTTP_PORT:-8080}:8000" volumes: - # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env + # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env # to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there. - - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads + - ${FILES_HOST_PATH:-${DOWNLOAD_HOST_PATH:-siftlode_files}}:/files # Plex library, read-only. The Plex module streams files itself, so it needs to SEE them: set # PLEX_MEDIA_HOST_PATH in .env to the host directory holding the library (the local side of the # admin "path map", whose default is /data=/plex-media). Left unset it falls back to an empty @@ -85,7 +85,7 @@ services: restart: unless-stopped # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the - # DB (job queue) and the downloads mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to + # DB (job queue) and the /files mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to # $HOME caches + the staging dir). worker: image: git.b1fr0st.eu/peter/siftlode:${IMAGE_TAG:-latest} @@ -98,7 +98,7 @@ services: environment: DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-siftlode} SCHEDULER_ENABLED: "false" - DOWNLOAD_ROOT: /downloads + FILES_ROOT: /files WORKER_ENABLED: "true" depends_on: db: @@ -111,7 +111,7 @@ services: condition: service_started networks: [internal] volumes: - - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads + - ${FILES_HOST_PATH:-${DOWNLOAD_HOST_PATH:-siftlode_files}}:/files # Same read-only Plex library as the API — the worker resolves library paths too. - ${PLEX_MEDIA_HOST_PATH:-siftlode_plex_media}:/plex-media:ro security_opt: @@ -136,7 +136,7 @@ services: volumes: siftlode_pgdata: - siftlode_downloads: + siftlode_files: # Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off). siftlode_plex_media: diff --git a/docker-compose.yml b/docker-compose.yml index fe760858..af7a1e59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode} # Download center: the API serves finished files + builds filmstrips; the worker (below) runs # the yt-dlp job loop. - DOWNLOAD_ROOT: /downloads + FILES_ROOT: /files WORKER_ENABLED: "false" depends_on: db: @@ -45,9 +45,9 @@ services: ports: - "${APP_PORT:-8080}:8000" volumes: - # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env + # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env # to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there. - - ${DOWNLOAD_HOST_PATH:-downloads}:/downloads + - ${FILES_HOST_PATH:-${DOWNLOAD_HOST_PATH:-files}}:/files # Plex library, read-only. The Plex module streams files itself, so it needs to SEE them: set # PLEX_MEDIA_HOST_PATH in .env to the host directory holding the library (the local side of the # admin "path map", whose default is /data=/plex-media). Left unset it falls back to an empty @@ -63,7 +63,7 @@ services: restart: unless-stopped # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the - # DB (job queue) + the downloads mount with the API. + # DB (job queue) + the /files mount with the API. worker: build: context: . @@ -80,7 +80,7 @@ services: environment: DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode} SCHEDULER_ENABLED: "false" - DOWNLOAD_ROOT: /downloads + FILES_ROOT: /files WORKER_ENABLED: "true" depends_on: db: @@ -91,7 +91,7 @@ services: bgutil-pot: condition: service_started volumes: - - ${DOWNLOAD_HOST_PATH:-downloads}:/downloads + - ${FILES_HOST_PATH:-${DOWNLOAD_HOST_PATH:-files}}:/files # Same read-only Plex library as the API — the worker resolves library paths too. - ${PLEX_MEDIA_HOST_PATH:-plex_media}:/plex-media:ro logging: *logging @@ -107,6 +107,6 @@ services: volumes: pgdata: - downloads: + files: # Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off). plex_media: diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 53680f95..f2becf4d 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -661,116 +661,123 @@ of a full-width panel and nothing on the right. ## Decision log -| Date | Decision | Chosen | Over | Why | -| ---------- | ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 2026-09-24 | Floating button | Back to top: a round 48 glass FAB; "round" = floats over something (media or the page) | A Material 3 rounded square; a labelled pill; the old 64 circle | The common web convention for back to top; round tells it apart from every toolbar button; 64 was oversized | -| 2026-09-24 | Device contract | Tablet portrait + landscape and up; phones out | A 1024px floor with a "too small" message | Portrait tablet use is real, and the app already renders at 820 portrait | -| 2026-09-24 | Card actions on touch | Fewer visible, the rest in a "…" menu | 44px buttons that no longer fit the card | Six 44px targets need more width than a card has | -| 2026-09-24 | Action placement | One menu with several triggers, never gesture-only, a desktop background menu that mirrors the toolbar | Visible buttons for everything | Fewer non-critical buttons on screen, without hiding any function | -| 2026-09-24 | Text selection | Text stays selectable; a click that ends a selection does nothing | Click-to-play on titles | The user must be able to copy any text the app shows | -| 2026-09-24 | A1 Button shape | r8 + a rung per context; glass is a Button variant | r12; a pill | Nested corners (r8 in r12 in r16); the pill already means "a filter or a choice" | -| 2026-09-24 | A2 Icon buttons | Two sizes, r8; a round button only on media | One size; all round | A lone toolbar icon at the row size looks lost; round rows read as tags | -| 2026-09-24 | A3 Touch size | 44 for every control primitive | 36 + gaps; Button only | One rule (HIG 44) instead of two sizes nobody remembers | -| 2026-09-24 | A4 Input | Bordered, a 55% fg-into-bg edge (≥ 3:1) | Filled; today's look as a primitive | The only option that clears WCAG 1.4.11 without a heavy fill on glass | -| 2026-09-24 | A5 Dropdown trigger | Pill in a toolbar, r8 in a form | Pill everywhere; r8 everywhere | Matches the neighbours in each context | -| 2026-09-24 | A6 Checkbox, switch, slider | Native, accent-tinted, the row is the target | Drawn primitives | Free keyboard / screen-reader / forced-colours support; only 3 sites | -| 2026-09-24 | B1 Density | Compact 32 / 28 / 24 by explicit height | Standard 38 / 32 / 26 | The owner's read of the lab; touch is covered separately (A3), and one toolbar row lines up at 28 | -| 2026-09-24 | B2 Scheme set | Midnight, Forest→cyan, Slate→fuchsia, Matrix→lime | Today's six | The accent collided with danger (YouTube), warn (Slate) and success (Forest, Matrix); Midnight and Starship were one scheme twice; 4 of 6 light accents failed 4.5:1 | -| 2026-09-24 | B3 State layers | Text colour at 10% / 16% over anything | Material's 8 / 12%; today's opacity + bg-card | Visible in every scheme and mode (≥ 1.2:1); today's row hover was 1.00:1 on a card | -| 2026-09-24 | B4 Page chrome | One sticky band + fade; filters fold when short | A floating glass bar; everything scrolls | Tabs and filters always in reach, in one place; a keyboard-up tablet keeps its content | -| 2026-09-24 | B5 Surfaces | One class + radius per role | Today's per-site mix | Nested corners and no per-site alpha drift | -| 2026-09-24 | B6 /watch | The app's surfaces, native video controls | Flat and minimal | The page strangers see should look like the product; native controls work on any device | -| 2026-09-24 | B7 Nav rail | Soft tint + edge bar, optional labels (a user setting), slim | Solid accent (recommended); a Material rail | The owner's combination: calmer than a solid fill, labels for who wants them, no width cost | -| 2026-09-24 | B8 Loading | Skeleton by shape, spinner in a control | A spinner everywhere | Nothing jumps when data lands; a long wait doesn't look stuck | -| 2026-09-24 | B9 Fade | Toolbars / headers, panel tabs, dialog edges, chip rows | Thumbnails; buttons | The north-star depth where content scrolls under chrome; not on content or small controls | -| 2026-09-24 | C1 Action classification | The table (lab C1): state + primary at rest, everyday toggles on hover, "…" on touch, destructive into the menu | Visible buttons per site, as today | One action list per object, nothing gesture-only, fewer buttons without losing a function | -| 2026-09-24 | C2 Side panels | A tap opens an overlay drawer; pin is a separate button ≥ 1024 | A tap pins and the content reflows | Below 768 the filters were unreachable; on touch a tap pinned and pushed the grid | -| 2026-09-24 | C3 Players on touch | Tap = show the controls; 44px seek hit area; 44 buttons; the YT description behind ⓘ | Today (tap also pauses; 7.5px seek bar) | Measured live: you couldn't reach the seek bar without pausing, and the hover-only description never opened on a tap | -| 2026-09-24 | C4 Dialogs | Focus by purpose; a pinned footer | Focus on Close; the footer scrolls | A form starts in its first field, and Save stays on screen with the keyboard up | -| 2026-09-24 | C5 Plex player controls | The owner's lean set: no Stop / Restart / ±10 s buttons, Back by position, Next only with a next item, Resume + From the start on the card, touch double-tap ±10 s | Every control in one row (it ran off-screen below 1024); Back by a press counter; ±10 s keyboard-only | The row fits a portrait tablet; the decision to restart is made on the card; a tablet keeps the 10 s skip | -| 2026-09-24 | Glass tuner | Retired; the schemes are the only colour choice | Keep it with ranges clamped to the rules | Free tuning is the drift the constitution exists to stop (owner) | -| 2026-09-24 | Matrix toned down | Moss #86a05a at Midnight's lightness, near-neutral greys; every white-on-accent part → `--accent-fg` | The B2 lime #a3e635 | The owner found the lime neon and tiring; it was the loudest accent (L 0.85, C 0.21), and fixed white parts sat on it at 1.5:1 | -| 2026-09-24 | Field primitives | `Input` / `Textarea` / `InnerInput` + `fieldClass`, `SelectTrigger`, the native controls; the lint rule "no raw ``" | `inputCls` and ~40 hand-styled boxes | Six edges, four radii and two focus looks for one control; the ≥ 3:1 edge held nowhere | -| 2026-09-25 | Surface roles | `SURFACE` roles in `ui/surfaces.ts`; the lint rule "no bare glass class"; menus p-1 with r8 rows | Three tiers paired ad hoc with five radii and three menu paddings | ~140 sites drifted; nested corners only hold if the radius comes with the role | -| 2026-09-25 | Static labels | Flat tonal `Badge`, no edge; a filtering tag is an edged chip | A glass pill with an edge, like the chips | Edged = pressable at a glance; a static tag that looks like a chip gets clicked and does nothing | -| 2026-09-25 | Header search | A 40 glass pill with the field edge (`lg`); what sits inside it is round | 44, r16, a glass box with a faint edge | The one rounded box among pills, and its edge under 3:1; the hero search is the YouTube/Gmail pill | -| 2026-09-25 | Switch row (F12) | The whole settings row toggles (`SwitchRow`; a hand-laid row points a label at the switch) | Only the 36×24 switch toggles | The iOS / Material settings row; the text was a dead click beside the control it names | -| 2026-09-25 | Edges and hovers (F14) | The ≥ 3:1 edge on the field family (fields, the form trigger); labelled pills keep the faint edge and the accent-edge hover | The ≥ 3:1 edge and a state-layer hover on every control | 1.4.11 asks an edge only where nothing else identifies the control; a heavy edge on every chip and tab weighs the whole UI down | -| 2026-09-25 | Rating and live (F16) | Own tokens: `--rating` (theme-scoped gold), `--live`/`--live-fg`, `--rating-media`, `--media-chrome`; palette classes lint-refused | Map them to `--warn` / `--danger` | A gold star and a red LIVE are what the eye expects; as warn/danger they would read as a status. The old amber score was 2:1 on light | -| 2026-09-25 | Fjord trial (B2 exception) | A fifth scheme, temporary: Midnight's neutrals, Playnite's steel accent (#337599 lifted to #5aa1cb) and steel rules; exit = fold into Midnight or remove | Retune Midnight straight away; borrow only the qualities | The owner wants to live with the calmer palette app-wide before it replaces the default; it breaks "own hue family" on purpose, and only until that call | -| 2026-09-25 | Whole-card row (owner) | A card row's B3 layer is the card's `::before` under its content; the open button's hit layer spans the card, the actions sit above it | The name button's own hover, ending at the first badge or button | A row that half-answers the pointer reads as broken; a badge click did nothing | -| 2026-09-25 | Badge tones (F29) | `Badge tone` neutral / accent / warn at 6%; light `--warn` #b45309 → #a14a07; edged label spans lint-refused | Keep edged status pills (a private Badge in Users, Settings "Granted", the suggestions count) | Edged = pressable at a glance; amber-700 was 4.58:1 as text and fell under 4.5 on its own tint | -| 2026-09-26 | Page band built (B4) | Controls only in the band (intros scroll); the fold shows the same row in place; one `h1` per page (`PageHeading` steps the hidden one aside); `SectionLabel` headings; Skip to content | A popover of the folded filters; the page title kept visible | A folded row is still the same controls, nothing duplicated; the rail and the tab title already name the page | -| 2026-09-26 | Fade built (B9) | `fade-below` on sticky toolbars and table headers; the scroller mask where the band sits on the backdrop | A painted band over the backdrop | The backdrop is the colour under the band, so fading the content into it is the same edge | -| 2026-09-26 | Viewport units | `dvh` everywhere (the Plex player stage with S3f) | `vh` / `h-screen` | `100vh` hides the last rows under a tablet browser's toolbar | -| 2026-09-26 | Video actions built (C1) | One list per video (menu, right-click, long-press, "…"); the card keeps watched / save / add to playlist on its picture on hover; download, details, copy, hide, reset in the menu | The six-button row under the card and the hover ⓘ | The menu is the one door every device has; the card loses a line of height and the touch card its row of 28px buttons | -| 2026-09-26 | Plex actions built (C1) | One list per Plex title (play/open, watched, add to playlist, media info, copy title, pin); the poster keeps watched / add / "…" on its picture on hover; seasons get open / watched / add / copy | The hover ⓘ, the pin-only menu, an episode's add button at rest, a collection's "Remove" X on the card | The same door as a video's; the collection grid goes to two columns on a phone so the touch "…" leaves the title its line | -| 2026-09-26 | Playlist actions built (C1) | A playlist row's "…" (play, pin, copy title / link, move, order, remove), Delete + an undo toast; one list per playlist (rename, pin, YouTube open / push / revert, delete) from its rail row and its header's "…" | The row's X column and its confirm; the header's pencil, YouTube link, push, revert and trash buttons | Remove is reversible, so it needs no confirm; the header keeps Play all; a narrow table drops "#" and Channel so the title keeps a width | -| 2026-09-26 | Channel + Plex list actions (C1) | A channel row's "…" (open, pin, YouTube, copy, full history, hide + undo, unsubscribe); a Plex playlist's title / show / season "…", Delete + undo; its header's "…" (rename, delete); an action toast lasts ≥ 8 s | The channel row's full-history, hide and unsubscribe buttons; the Plex list's red X on every row, show and season; the header's pencil and trash | Hiding and removing are reversible, so they need no confirm and the undo needs time; unsubscribe and deleting a playlist keep theirs | -| 2026-09-26 | Download / notification / Scheduler actions (C1) | A download row's "…" (play, share, save, edit, details, re-download, manage sharing, copies, cancel / delete / remove); a library row's Share stays in sight, ahead of save; a notification row's "…" and hover dismiss; the inbox band's and the Scheduler strip's "…" | The download row's five rare buttons and its cancel / delete / remove; the source link's copy button; an activity notice's Find / Undo links; the band's Clear everything; the strip's Purge discovery | The destructive rows keep their confirm, and Clear everything gains one; a row link keeps the browser's own menu; Find in feed drops the tag / search / age filters that could hide its video | -| 2026-09-26 | Channel menu: Tags + Priority (C1) | Tags A–Z as in-place checks; Priority as fixed steps 10 · 5 · 3 · 2 · 1 · 0, the current one ticked; a pick re-sorts at once and the page + focus follow the row; one drawn `MenuCheck` for every checkable row that cannot be an `` | Relative steps (+1 / +5); straight to the top; the stepper only; seven hand-drawn check marks (three box sizes, a tinted box, a × for "on", a trailing tick) | Bubbling a channel up is one pick; fixed levels keep the scale the reader already uses; the list moving where the reader can see it is the feedback | -| 2026-09-26 | Reorder without a drag (WCAG 2.5.7) | Row menus: Top / Up / Down / Bottom (YouTube + Plex playlists); saved views: four buttons with the place beside the selected view (owner: A); side panels in edit mode: an Up / Down pair per group (owner: A), the name wrapping there | A "…" on every saved-view row; a "…" on every panel group; drag only | A visible, one-click path where the list is edited; the edge moves shown off so the buttons never shift | -| 2026-09-26 | Titles in click surfaces | The surface stays a button, flagged `selectable`: its text selects, and the click that ends a selection does nothing; every plain `Pressable` pinned with its reason | Rebuilding each card and row so the title sits outside the button; exceptions for master-detail lists | Chrome selects inside a button once it is `user-select: text` (measured), so one flag on the primitive keeps the keyboard, focus and list walking every surface already has | -| 2026-09-26 | Right-click on a selection | The browser's own menu opens on selected words; beside them the object's menu | A "Copy selection" row in the app menu | The reader selected the words to do something with them; the native menu copies, searches and translates, and changes nothing else | -| 2026-09-26 | Keyboard ring on a flush row | Inside the row (a table row by its tag, a log line, a show header in its clipping card, with the card's corners) | The ring outside, as on a card | Outside it ran under the table head and past the table, and a clipping card cut it away entirely (invisible focus) | -| 2026-09-26 | Fade at rest | The sticky header's fade appears with the first 1rem of scroll (a scroll-driven animation; no fade where unsupported) | A fade that is always drawn | At rest it dimmed the first row and hid its ring's top edge; nothing is under the header yet | -| 2026-09-26 | Narrow playlist table | "#" and Channel hide by the table's own width (under 42rem), not the window's | Hiding below the `sm` window width | Beside the playlist rail a desktop window left the table 432 px and the title 0 px | -| 2026-09-26 | Background menu built | Five list pages (feed, Plex grid, channels, playlist, notifications); the toolbar's sort / view / filters and the band's actions; no destructive row | Every page with a toolbar; the "…" menus copied whole, delete included | The tab-only pages have nothing a menu would add; a blank-area right-click must not be one click from delete | -| 2026-09-26 | Media card anatomy (C) | One frame; the meta line pinned to its bottom; the length a badge on the picture; the Plex card on the same primitive as the video card | A: the year as an overline over the title; B: the year inline after it; a fixed two-line title slot | A one-line title left a hole the eye reads as a missing line; C puts nothing under a variable-height title | -| 2026-09-26 | One Play | `MediaPlayGlyph` (the media button variant) and only where the click plays | The Plex card's hand-painted accent circle; a playable dot on every card | One Play for every card; a dot that only a colour and a tooltip explain says nothing | -| 2026-09-26 | Filtering values on the info page | A value that filters (year, certificate, studio, director) is an edged chip; a fact is flat text | Everything flat, the clickable ones only hover-tinted | The S3d rule: edged = pressable, so the reader sees which values do something | -| 2026-09-26 | Shelf density | On any strip: one title line over one meta line | The grid's two lines and full meta on a strip | The tallest card set the strip's height and left holes in the cards with less to say | -| 2026-09-26 | Compact list row (B) | The title leads; a faint "…" at the end; the toggles float over the date on hover | A leading 128px action column | That column sat empty at rest, a hole at the start of every row | -| 2026-09-26 | Media-info layout (B) | A main column (collections first, crew, cast) beside a facts sidebar; an empty column is not drawn | Every section full width, stacked | The collection is the most used; the keywords' column left a void beside a short details box | -| 2026-09-26 | Mixed Home shelf (A) | Plex titles as wide art beside videos | Posters, with the videos not stretched | Stretching a video card to a poster's height left an ~80px gap | -| 2026-09-27 | Home hero | A card (hover row + Play on the picture, the whole menu, a touch "…"; owner: A); three quarters wide and centred on a wide window; one fact line; the summary at the measure | The hero's pin row alone; full width; an 18rem picture column | The one answer stands in focus, and it offers what the same card offers anywhere | -| 2026-09-27 | Home shelf tiles | A channel / playlist tile: Play, ↗ open, "…" with Open; a show acts on its next episode, ⓘ opens it; the shelf slot 232px | The tile's pin row alone; 200px | A tile without its object's actions read as broken; at 200 the Play touched the hover row | -| 2026-09-27 | Synopsis preview (C) | A ~0.9 s mouse rest opens a passive panel beside the card | The synopsis as the picture's `title` tooltip; an ⓘ to hover; none | One long tooltip line fired anywhere on the art and clashed with the cut title's own tooltip | -| 2026-09-27 | Reading measure | Prose capped at ~65 characters a line (`measure`) | The container's full width | At 110–130 characters a line the eye loses the next line's start | -| 2026-09-24 | Chip family | One `Chip` (filter / token / tag / suggest / ghost) + `DismissibleChip`; nothing under 24; the lint rule "no raw ` - - - -
- {tab === "queue" && ( - <> - {/* The field takes the whole line when the buttons would squeeze it (at 375 it had 57px). */} -
- setAddUrl(e.target.value)} - onFocus={() => pasteFromClipboard()} - onKeyDown={(e) => e.key === "Enter" && addUrl.trim() && setAddSource(addUrl.trim())} - placeholder={t("downloads.page.addUrl")} - /> - pasteFromClipboard(true)} - > - - - - {/* Local upload is a peer to "add a URL": a general file → the same library. */} - -
- - {queue.map((job, i) => ( - - {/* The queue's primary action stays in sight; cancel moved into the "…". */} - {job.status === "queued" || job.status === "running" ? ( - actions.run(job, "pause")} - title={t("downloads.actions.pause")} - > - - - ) : ( - actions.run(job, "resume")} - title={t( - job.status === "error" - ? "downloads.actions.retry" - : "downloads.actions.resume", - )} - > - {/* Retry is "do it again", not playback: the Play glyph promised a video. */} - {job.status === "error" ? ( - - ) : ( - - )} - - )} - - ))} - {queue.length === 0 && - (jobsQ.isError && !jobsQ.data ? ( - jobsQ.refetch()}> - {t("common.loadError")} - - ) : jobsQ.isLoading ? ( - - ) : ( -
- {t("downloads.empty.queue")} -
- ))} - - - )} - - {tab === "done" && ( - <> - - - {library.map((job, i) => ( - - {libraryRowActions(job).play && ( - setPlayJob(job)} - title={ - job.position_seconds ? t("card.continueTitle") : t("downloads.actions.play") - } - > - - - )} - {/* Sharing the link is the library's main errand: in sight, ahead of save. */} - actions.share(job)} title={t("downloads.actions.share")}> - - - {saveBtn(job)} - - ))} - {library.length === 0 && - (jobsQ.isError && !jobsQ.data ? ( - jobsQ.refetch()}> - {t("common.loadError")} - - ) : jobsQ.isLoading ? ( - - ) : ( -
- {t("downloads.empty.done")} -
- ))} - - - )} - - {tab === "shared" && ( - - {(sharedQ.data ?? []).map((job, i) => { - // Same media_kind + can_download gate as the library, via the shared helper. - const a = sharedRowActions(job); - return ( - - {a.play && ( - setPlayJob(job)} - title={ - job.position_seconds ? t("card.continueTitle") : t("downloads.actions.play") - } - > - - - )} - {a.save && saveBtn(job)} - - ); - })} - {(sharedQ.data ?? []).length === 0 && - (sharedQ.isError && !sharedQ.data ? ( - sharedQ.refetch()}> - {t("common.loadError")} - - ) : sharedQ.isLoading ? ( - - ) : ( -
- {t("downloads.empty.shared")} -
- ))} - - )} - - {tab === "system" && me.role === "admin" && } - - - - - {addSource && ( - { - setAddSource(null); - setAddUrl(""); - }} - /> - )} - {upload && setUpload(false)} />} - {manage && setManage(false)} />} - {metaJob && setMetaJob(null)} />} - {shareJob && setShareJob(null)} />} - {playing && ( - setPlaying(null)} - /> - )} - {editJob && setEditJob(null)} />} - {errorJob && ( - actions.run(errorJob, "resume")} - onClose={() => setErrorJob(null)} - /> - )} - -
- - ); -} diff --git a/frontend/src/components/DownloadDialog.dom.test.tsx b/frontend/src/components/DownloadDialog.dom.test.tsx new file mode 100644 index 00000000..119e2100 --- /dev/null +++ b/frontend/src/components/DownloadDialog.dom.test.tsx @@ -0,0 +1,101 @@ +// @vitest-environment jsdom +// +// The download dialog opens on the format used last (per account), so a pasted link is one Enter +// away; a format deleted since falls back to the first. Its "added" notice's View opens the tray on +// the new job. The prefs are a map that outlives a mount — a reload, as far as the dialog can tell. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { useState } from "react"; +import { api, type DownloadProfile } from "../lib/api"; + +const h = vi.hoisted(() => ({ + prefs: new Map(), + notify: vi.fn(), + showInTray: vi.fn(), +})); + +vi.mock("react-i18next", async (importOriginal) => ({ + ...(await importOriginal()), + useTranslation: () => ({ i18n: { language: "en" }, t: (key: string) => key }), +})); +vi.mock("../lib/usePref", () => ({ + usePref: (key: string, initial: T) => { + const [value, setValue] = useState(() => + h.prefs.has(key) ? (h.prefs.get(key) as T) : initial, + ); + return [ + value, + (next: T) => { + h.prefs.set(key, next); + setValue(next); + }, + ]; + }, +})); +vi.mock("../lib/notifications", async (importOriginal) => ({ + ...(await importOriginal()), + notify: h.notify, +})); +vi.mock("../lib/downloadQueue", () => ({ showInTray: h.showInTray })); + +import DownloadDialog from "./DownloadDialog"; + +const profile = (id: number, name: string) => + ({ id, name, is_builtin: false, spec: { mode: "av" } }) as unknown as DownloadProfile; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + h.prefs.clear(); + h.notify.mockClear(); + h.showInTray.mockClear(); +}); + +function open() { + vi.spyOn(api, "downloadProfiles").mockResolvedValue([profile(1, "Best"), profile(2, "Phone")]); + const enqueue = vi.spyOn(api, "enqueueDownload").mockResolvedValue({ id: 77 } as never); + render( + + {}} /> + , + ); + return enqueue; +} + +const picked = () => + screen.getByRole("button", { name: /^downloads\.dialog\.profile: / }).getAttribute("aria-label"); + +describe("DownloadDialog", () => { + it("opens on the format used last; a deleted one falls back to the first", async () => { + const enqueue = open(); + await waitFor(() => expect(picked()).toBe("downloads.dialog.profile: Best")); + fireEvent.click(screen.getByRole("button", { name: /^downloads\.dialog\.profile: / })); + const phone = (await screen.findAllByRole("menuitemradio", { hidden: true })).find( + (o) => o.textContent === "Phone", + )!; + fireEvent.click(phone); + fireEvent.click(screen.getByText("downloads.dialog.submit")); + await waitFor(() => + expect(enqueue).toHaveBeenCalledWith(expect.objectContaining({ profile_id: 2 })), + ); + cleanup(); + + open(); + await waitFor(() => expect(picked()).toBe("downloads.dialog.profile: Phone")); + cleanup(); + + h.prefs.set("downloadProfile", 99); + open(); + await waitFor(() => expect(picked()).toBe("downloads.dialog.profile: Best")); + }); + + it("its notice's View opens the tray on the new job", async () => { + open(); + await waitFor(() => expect(picked()).toBe("downloads.dialog.profile: Best")); + fireEvent.click(screen.getByText("downloads.dialog.submit")); + await waitFor(() => expect(h.notify).toHaveBeenCalledTimes(1)); + (h.notify.mock.calls[0]![0] as { action: { onClick: () => void } }).action.onClick(); + expect(h.showInTray).toHaveBeenCalledWith(77); + }); +}); diff --git a/frontend/src/components/DownloadDialog.tsx b/frontend/src/components/DownloadDialog.tsx index bf09fd3e..4b7e7d2b 100644 --- a/frontend/src/components/DownloadDialog.tsx +++ b/frontend/src/components/DownloadDialog.tsx @@ -11,13 +11,18 @@ import { invalidateDownloads } from "../lib/downloads"; const ProfileEditor = lazy(() => import("./ProfileEditor")); import { notify } from "../lib/notifications"; import { notifyActionError } from "../lib/actionErrors"; -import { navigateTo } from "../lib/nav"; +import { showInTray } from "../lib/downloadQueue"; import { profileName } from "../lib/vocab"; +import { LS } from "../lib/storage"; +import { usePref } from "../lib/usePref"; import FilterSelect from "./ui/FilterSelect"; import { Input } from "./ui/Input"; -// Preset picker shown when the user hits Download on a card / in the player, or adds a URL from -// the Download Center. Enqueues via the API and points the user at the Download Center. +// Preset picker shown when the user hits Download on a card / in the player, or brings a link (the +// Files link box, a paste anywhere). Enqueues via the API; its notice's "View" opens the tray on it. +// It opens on the format used last (per account): a pasted link is then one Enter away. +const parseProfile = (v: unknown): number | null => (typeof v === "number" ? v : null); + export default function DownloadDialog({ source, title, @@ -36,23 +41,30 @@ export default function DownloadDialog({ const [manage, setManage] = useState(false); const profiles = profilesQ.data ?? []; - const chosen = profileId ?? profiles[0]?.id ?? null; + const [lastProfile, setLastProfile] = usePref("downloadProfile", null, { + localKey: LS.downloadProfile, + parse: parseProfile, + }); + // The one used last, while it still exists (a deleted format falls back to the first). + const remembered = profiles.some((p) => p.id === lastProfile) ? lastProfile : null; + const chosen = profileId ?? remembered ?? profiles[0]?.id ?? null; const submit = async () => { if (!chosen || busy) return; setBusy(true); const displayName = name.trim() || undefined; try { - await api.enqueueDownload({ + const job = await api.enqueueDownload({ source, profile_id: chosen, display_name: displayName, }); invalidateDownloads(qc); + setLastProfile(chosen); notify({ level: "success", message: t("downloads.dialog.added"), - action: { label: t("downloads.dialog.view"), onClick: () => navigateTo("downloads") }, + action: { label: t("downloads.dialog.view"), onClick: () => showInTray(job.id) }, }); onClose(); } catch (e) { diff --git a/frontend/src/components/DownloadIntake.dom.test.tsx b/frontend/src/components/DownloadIntake.dom.test.tsx new file mode 100644 index 00000000..1ec1c64f --- /dev/null +++ b/frontend/src/components/DownloadIntake.dom.test.tsx @@ -0,0 +1,164 @@ +// @vitest-environment jsdom +// +// Downloading is an action you start anywhere: a lone http(s) link pasted onto a page (not into a +// field, not at a dialog) opens the app's one download dialog; the Files link box hands its link to +// the same dialog; the palette's "Download from link…" focuses that box on the entry it navigated to. +// The dialog itself is stubbed (it names the link it was opened on). +import { afterEach, describe, expect, it, vi } from "vitest"; +import { act, cleanup, fireEvent, render, screen } from "@testing-library/react"; + +vi.mock("react-i18next", async (importOriginal) => ({ + ...(await importOriginal()), + useTranslation: () => ({ i18n: { language: "en" }, t: (key: string) => key }), +})); +vi.mock("./DownloadDialog", () => ({ + default: ({ source, onClose }: { source: string; onClose: () => void }) => ( +
+ {source} + +
+ ), +})); + +import DownloadIntake from "./DownloadIntake"; +import { LinkBox } from "./LinkBox"; +import { clearPendingDownload, requestLinkBox } from "../lib/downloadIntake"; +import { ensureNavId } from "../lib/history"; +import { pushLayer, removeLayer } from "../lib/layerStack"; + +afterEach(() => { + cleanup(); + act(clearPendingDownload); +}); + +function paste(target: EventTarget, text: string) { + const event = new Event("paste", { bubbles: true, cancelable: true }) as ClipboardEvent; + Object.defineProperty(event, "clipboardData", { + value: { getData: (type: string) => (type === "text/plain" ? text : "") }, + }); + act(() => { + target.dispatchEvent(event); + }); + return event; +} + +const dialog = () => screen.queryByRole("dialog", { name: "download" }); + +describe("DownloadIntake · a pasted link", () => { + it("a link pasted onto the page opens the download dialog on it", async () => { + render(); + const event = paste(document.body, " https://www.tiktok.com/@a/video/1 "); + expect(await screen.findByText("https://www.tiktok.com/@a/video/1")).toBeTruthy(); + // Taken: the browser does nothing else with it. + expect(event.defaultPrevented).toBe(true); + }); + + it("text that is not one link, or a paste into a field, is left alone", async () => { + render( + <> + + + , + ); + paste(document.body, "just some words"); + paste(document.body, "https://a.example/1 https://a.example/2"); + const field = screen.getByRole("textbox", { name: "search" }); + expect(paste(field, "https://a.example/1").defaultPrevented).toBe(false); + await act(async () => {}); + expect(dialog()).toBeNull(); + }); + + it("with a dialog on top, a paste is the dialog's, not a download", async () => { + render( + <> + +
+ , + ); + paste(document.body, "https://a.example/1"); + await act(async () => {}); + expect(dialog()).toBeNull(); + }); + + it("with a player or a menu on top (a layer, no aria-modal), a paste is not a download", async () => { + render(); + const layer = pushLayer(); + try { + paste(document.body, "https://a.example/1"); + await act(async () => {}); + expect(dialog()).toBeNull(); + } finally { + removeLayer(layer); + } + }); + + it("closing the dialog closes it", async () => { + render(); + paste(document.body, "https://a.example/1"); + fireEvent.click(await screen.findByText("close")); + expect(dialog()).toBeNull(); + }); +}); + +describe("LinkBox", () => { + it("Enter hands the typed link to the dialog and empties the box", async () => { + render( + <> + + + , + ); + const field = screen.getByRole("textbox", { name: "files.link.label" }); + fireEvent.change(field, { target: { value: " https://vimeo.com/123 " } }); + fireEvent.keyDown(field, { key: "Enter" }); + expect(await screen.findByText("https://vimeo.com/123")).toBeTruthy(); + expect((field as HTMLInputElement).value).toBe(""); + }); + + it("a clipboard link read on focus never replaces what was typed while it was read", async () => { + let answer: (text: string) => void = () => {}; + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { readText: () => new Promise((resolve) => (answer = resolve)) }, + }); + render(); + const box = screen.getByRole("textbox"); + fireEvent.focus(box); + fireEvent.change(box, { target: { value: "https://typed.example/1" } }); + await act(async () => answer("https://clipboard.example/2")); + expect((box as HTMLInputElement).value).toBe("https://typed.example/1"); + }); + + it("an empty box downloads nothing", async () => { + render( + <> + + + , + ); + fireEvent.click(screen.getByText("files.link.download")); + await act(async () => {}); + expect(dialog()).toBeNull(); + }); + + it("the palette's request focuses the box on its entry, once", () => { + ensureNavId(); + act(requestLinkBox); + const { unmount } = render(); + expect(document.activeElement).toBe(screen.getByRole("textbox", { name: "files.link.label" })); + unmount(); + // Spent: the next mount on the same entry is not focused again. + render(); + expect(document.activeElement).toBe(document.body); + }); + + it("a request made for another entry (the navigation never got here) focuses nothing", () => { + ensureNavId(); + act(requestLinkBox); + // A later entry — say Back, then elsewhere — mounts the box. + window.history.pushState({}, ""); + ensureNavId(); + render(); + expect(document.activeElement).toBe(document.body); + }); +}); diff --git a/frontend/src/components/DownloadIntake.tsx b/frontend/src/components/DownloadIntake.tsx new file mode 100644 index 00000000..4297c9ae --- /dev/null +++ b/frontend/src/components/DownloadIntake.tsx @@ -0,0 +1,42 @@ +import { lazy, Suspense, useEffect } from "react"; +import { + asDownloadLink, + clearPendingDownload, + pasteIsForThePage, + requestDownload, + usePendingDownload, +} from "../lib/downloadIntake"; + +const DownloadDialog = lazy(() => import("./DownloadDialog")); + +// The app's one download dialog (lib/downloadIntake), and the paste that opens it: a lone http(s) +// link pasted onto any page — not into a field — is downloaded, the profile picked in the dialog. +export default function DownloadIntake() { + const source = usePendingDownload(); + + useEffect(() => { + const onPaste = (e: ClipboardEvent) => { + if (e.defaultPrevented || !pasteIsForThePage(e.target)) return; + const link = asDownloadLink(e.clipboardData?.getData("text/plain")); + if (!link) return; + e.preventDefault(); + requestDownload(link); + }; + document.addEventListener("paste", onPaste); + return () => document.removeEventListener("paste", onPaste); + }, []); + + return ( + + {/* The link itself under the title: a paste can be an accident, and this says what it was. */} + {source && ( + + )} + + ); +} diff --git a/frontend/src/components/DownloadTray.dom.test.tsx b/frontend/src/components/DownloadTray.dom.test.tsx new file mode 100644 index 00000000..162ffc54 --- /dev/null +++ b/frontend/src/components/DownloadTray.dom.test.tsx @@ -0,0 +1,364 @@ +// @vitest-environment jsdom +// +// The download tray: up only while something waits, runs or failed — and what finished in this tab, +// until it is closed; a row's right click and "…" are the queue's menu; a failure says why and offers +// a retry; a finish is announced once with "Copy link" and "Open"; it folds to a pill, remembered. +// The queue is fed straight in (the poll is stubbed); a poll's look is `noteQueue`, as fetchQueue does. +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { useState } from "react"; +import { api, type QueueJob } from "../lib/api"; + +const h = vi.hoisted(() => ({ + jobs: [] as unknown[], + updatedAt: 0, + openFile: vi.fn(), + share: vi.fn(async () => "https://x/watch/t"), + notify: vi.fn(), + prefs: new Map(), + intervalMs: ((_d: unknown) => 0) as (d: unknown) => number, +})); + +vi.mock("react-i18next", async (importOriginal) => ({ + ...(await importOriginal()), + useTranslation: () => ({ + i18n: { language: "en" }, + t: (key: string | string[], o?: { count?: number; title?: string }) => + `${Array.isArray(key) ? key[0] : key}${o?.count !== undefined ? `:${o.count}` : ""}${o?.title ? `:${o.title}` : ""}`, + }), +})); +vi.mock("../lib/useLiveQuery", () => ({ + useLiveQuery: (_key: unknown, _fn: unknown, opts: { intervalMs: (d: unknown) => number }) => ( + (h.intervalMs = opts.intervalMs), + { + data: h.jobs, + dataUpdatedAt: h.updatedAt, + isError: false, + isLoading: false, + refetch: vi.fn(), + } + ), +})); +// The prefs as state over a map that outlives a mount — a reload, as far as the tray can tell. How +// usePref stores them is its own business (usePref.dom.test). +vi.mock("../lib/usePref", () => ({ + usePref: (key: string, initial: T) => { + const [value, setValue] = useState(() => + h.prefs.has(key) ? (h.prefs.get(key) as T) : initial, + ); + return [ + value, + (next: T) => { + h.prefs.set(key, next); + setValue(next); + }, + ]; + }, +})); +vi.mock("./filesContext", () => ({ useFilesActions: () => ({ openFile: h.openFile }) })); +vi.mock("./confirmContext", () => ({ useConfirm: () => async () => true })); +vi.mock("../lib/downloads", async (importOriginal) => ({ + ...(await importOriginal()), + quickShareLink: h.share, +})); +vi.mock("../lib/notifications", async (importOriginal) => ({ + ...(await importOriginal()), + notify: h.notify, +})); + +import DownloadTray from "./DownloadTray"; +import { noteQueue, resetQueueWatch, showInTray } from "../lib/downloadQueue"; +import { pushLayer, removeLayer } from "../lib/layerStack"; + +beforeEach(() => { + resetQueueWatch(); + h.prefs.clear(); + h.updatedAt = 0; + globalThis.ResizeObserver ??= class { + observe() {} + unobserve() {} + disconnect() {} + } as unknown as typeof ResizeObserver; +}); +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + for (const f of [h.openFile, h.share, h.notify]) f.mockClear(); +}); + +const job = (id: number, status: string, extra: Partial = {}) => + ({ + id, + status, + progress: 40, + phase: "video", + error_code: null, + error_detail: null, + job_kind: "download", + display_name: `Clip ${id}`, + source_key: "site:tiktok.com", + folder_id: 9, + asset: { media_kind: "video" }, + ...extra, + }) as unknown as QueueJob; + +/** A poll: the queue the tray now sees. */ +function poll(jobs: QueueJob[], view?: { rerender: () => void }) { + h.jobs = jobs; + act(() => noteQueue(jobs)); + view?.rerender(); +} + +function mount() { + const qc = new QueryClient(); + const tree = () => ( + + + + ); + const { rerender } = render(tree()); + return { rerender: () => act(() => rerender(tree())) }; +} + +const tray = () => screen.queryByRole("region", { name: "tray.label" }); +const rowOf = (title: string) => screen.getByText(title).closest("[data-job-id]")!; + +describe("DownloadTray", () => { + it("is not there while nothing waits, runs or failed", () => { + poll([job(1, "done")]); + mount(); + expect(tray()).toBeNull(); + }); + + it("lists what waits, runs and failed, its sites, and sums them up", () => { + poll([job(1, "running"), job(2, "queued"), job(3, "error", { error_code: "login_required" })]); + mount(); + expect(tray()).not.toBeNull(); + expect(screen.getByText(/tray\.count\.running:2, tray\.count\.error:1/)).toBeTruthy(); + expect(within(rowOf("Clip 1")).getByText("tiktok.com")).toBeTruthy(); + expect(within(rowOf("Clip 1")).getByText("40%")).toBeTruthy(); + expect(within(rowOf("Clip 2")).getByText("downloads.status.queued")).toBeTruthy(); + }); + + it("a running job names its step: the second stream's bar does not read as a restart", () => { + poll([job(1, "running", { phase: "audio" }), job(2, "running", { phase: "merging" })]); + mount(); + const [audio, merging] = screen.getAllByRole("progressbar"); + expect(within(rowOf("Clip 1")).getByText("downloads.phase.audio")).toBeTruthy(); + expect(audio!.getAttribute("aria-valuenow")).toBe("40"); + // A step that reports no bytes has no % to claim. + expect(within(rowOf("Clip 2")).getByText("downloads.phase.merging")).toBeTruthy(); + expect(merging!.getAttribute("aria-valuenow")).toBeNull(); + expect(within(rowOf("Clip 2")).queryByText("40%")).toBeNull(); + }); + + it("a paused download is waited on slowly, and the tray neither spins nor says done", () => { + poll([job(1, "paused")]); + mount(); + // Nothing moves until its user resumes it: no fast polling for that. + expect(h.intervalMs([job(1, "paused")])).toBe(60000); + expect(h.intervalMs([job(1, "paused"), job(2, "queued")])).toBe(2000); + const marks = [...tray()!.querySelectorAll("svg")].map((i) => i.getAttribute("class") ?? ""); + expect(marks.some((c) => /lucide-(circle-check|check-circle)/.test(c))).toBe(false); + expect( + marks.some((c) => /lucide-(circle-pause|pause-circle)/.test(c) && /text-warn/.test(c)), + ).toBe(true); + }); + + it("a failure says why, and its Retry resumes it", () => { + const resume = vi.spyOn(api, "resumeDownload").mockResolvedValue({} as never); + poll([job(3, "error", { error_code: "login_required" })]); + mount(); + expect( + within(rowOf("Clip 3")).getByText("downloads.errors.fail.cat.login_required.short"), + ).toBeTruthy(); + fireEvent.click(within(rowOf("Clip 3")).getByText("downloads.actions.retry")); + return waitFor(() => expect(resume).toHaveBeenCalledWith(3)); + }); + + it("a row's right click opens the queue's menu", async () => { + poll([job(1, "running")]); + mount(); + act(() => { + fireEvent.contextMenu(rowOf("Clip 1")); + }); + await waitFor(() => + expect(screen.getAllByRole("menuitem", { hidden: true }).map((b) => b.textContent)).toContain( + "downloads.actions.pause", + ), + ); + }); + + it("a finish is announced once — Copy link, Open — and stays as Done · Open", async () => { + poll([job(1, "running")]); + const view = mount(); + poll([job(1, "done")], view); + await waitFor(() => expect(h.notify).toHaveBeenCalledTimes(1)); + const notice = h.notify.mock.calls[0]![0] as { + title: string; + message: string; + action: { label: string; onClick: () => void }; + secondaryAction: { label: string; onClick: () => void }; + }; + expect(notice.title).toBe("tray.done.title:Clip 1"); + expect(notice.message).toBe("tray.done.phone"); + expect(notice.action.label).toBe("tray.done.copyLink"); + notice.action.onClick(); + expect(h.share).toHaveBeenCalled(); + notice.secondaryAction.onClick(); + expect(h.openFile).toHaveBeenCalledWith(1, 9); + // Still on the tray, done; its Open opens it. + fireEvent.click(within(rowOf("Clip 1")).getByText("tray.done.open")); + expect(h.openFile).toHaveBeenCalledTimes(2); + view.rerender(); + expect(h.notify).toHaveBeenCalledTimes(1); + }); + + it("each finish is announced once: a later one does not repeat an earlier", async () => { + poll([job(1, "running"), job(2, "running")]); + const view = mount(); + poll([job(1, "done"), job(2, "running")], view); + await waitFor(() => expect(h.notify).toHaveBeenCalledTimes(1)); + poll([job(1, "done"), job(2, "done")], view); + await waitFor(() => expect(h.notify).toHaveBeenCalledTimes(2)); + expect(h.notify.mock.calls.map((c) => (c[0] as { title: string }).title)).toEqual([ + "tray.done.title:Clip 1", + "tray.done.title:Clip 2", + ]); + }); + + it("a plain file's notice does not promise a phone", async () => { + poll([job(1, "running", { asset: { media_kind: "file" } as never })]); + const view = mount(); + poll([job(1, "done", { asset: { media_kind: "file" } as never })], view); + await waitFor(() => + expect(h.notify).toHaveBeenCalledWith(expect.objectContaining({ message: "tray.done.file" })), + ); + }); + + it("closed, it forgets what finished and stays away until the work changes", () => { + poll([job(1, "running"), job(2, "running")]); + const view = mount(); + poll([job(1, "running"), job(2, "done")], view); + fireEvent.click(screen.getByRole("button", { name: "tray.close" })); + expect(tray()).toBeNull(); + // The same work moving on is not news… + poll([job(1, "running", { progress: 80 })], view); + expect(tray()).toBeNull(); + // …a new job is. + poll([job(1, "running"), job(4, "queued")], view); + expect(tray()).not.toBeNull(); + expect(screen.queryByText("Clip 2")).toBeNull(); + }); + + it("closed on an old failure, it stays closed after a reload", () => { + poll([job(3, "error")]); + const first = render( + + + , + ); + fireEvent.click(screen.getByRole("button", { name: "tray.close" })); + first.unmount(); + resetQueueWatch(); + poll([job(3, "error")]); + mount(); + expect(tray()).toBeNull(); + }); + + it("closed on an old failure, a new download's finish still shows", () => { + poll([job(3, "error")]); + const view = mount(); + fireEvent.click(screen.getByRole("button", { name: "tray.close" })); + expect(tray()).toBeNull(); + poll([job(4, "running"), job(3, "error")], view); + expect(tray()).not.toBeNull(); + // Done, the queue is the old failure alone again — the very queue it was closed on. + poll([job(4, "done"), job(3, "error")], view); + expect(tray()).not.toBeNull(); + expect(within(rowOf("Clip 4")).getByText("tray.done.label")).toBeTruthy(); + }); + + it("folds to a pill that sums it up, and opens again — the keyboard following", () => { + poll([job(1, "running")]); + mount(); + fireEvent.click(screen.getByRole("button", { name: "tray.fold" })); + expect(screen.queryByText("Clip 1")).toBeNull(); + const pill = screen.getByRole("button", { name: /tray\.count\.running:1/ }); + expect(document.activeElement).toBe(pill); + fireEvent.click(pill); + expect(screen.getByText("Clip 1")).toBeTruthy(); + expect(document.activeElement).toBe(screen.getByRole("button", { name: "tray.fold" })); + }); + + it("closed, it leaves the keyboard on the page", () => { + poll([job(1, "running")]); + render( + +
+ + , + ); + fireEvent.click(screen.getByRole("button", { name: "tray.close" })); + expect(document.activeElement?.id).toBe("content"); + }); + + describe("View on a job just started", () => { + it("brings a closed tray back, on the job's row", async () => { + poll([job(3, "error")]); + const view = mount(); + fireEvent.click(screen.getByRole("button", { name: "tray.close" })); + expect(tray()).toBeNull(); + act(() => showInTray(3)); + view.rerender(); + expect(tray()).not.toBeNull(); + await waitFor(() => expect(document.activeElement).toBe(rowOf("Clip 3"))); + }); + + it("unfolds a folded tray onto the row", async () => { + poll([job(1, "running"), job(2, "queued")]); + mount(); + fireEvent.click(screen.getByRole("button", { name: "tray.fold" })); + act(() => showInTray(2)); + await waitFor(() => expect(document.activeElement).toBe(rowOf("Clip 2"))); + }); + + it("asked from inside a player, it opens under it but leaves the keyboard there", async () => { + poll([job(1, "running"), job(2, "queued")]); + mount(); + fireEvent.click(screen.getByRole("button", { name: "tray.fold" })); + const player = document.createElement("button"); + document.body.append(player); + player.focus(); + const layer = pushLayer(); + act(() => showInTray(2)); + await waitFor(() => expect(screen.getByText("Clip 2")).toBeTruthy()); + expect(document.activeElement).toBe(player); + removeLayer(layer); + player.remove(); + }); + + it("a job done before the tray saw it run opens as a file", () => { + // The queue's first look only learns what is there: this one never ran in this tab. + poll([job(5, "done", { folder_id: 7 })]); + mount(); + act(() => showInTray(5)); + expect(h.openFile).toHaveBeenCalledWith(5, 7); + expect(tray()).toBeNull(); + }); + + it("a job a later poll does not have is let go — nothing opens, nothing waits for it", () => { + poll([job(1, "running")]); + const view = mount(); + fireEvent.click(screen.getByRole("button", { name: "tray.fold" })); + act(() => showInTray(9)); + h.updatedAt = Date.now() + 1000; + poll([job(1, "running")], view); + // Had the ask stayed, the job turning up later would pull the tray open on it. + poll([job(1, "running"), job(9, "running")], view); + expect(screen.queryByText("Clip 9")).toBeNull(); + expect(h.openFile).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/frontend/src/components/DownloadTray.menu.dom.test.tsx b/frontend/src/components/DownloadTray.menu.dom.test.tsx new file mode 100644 index 00000000..a28411e3 --- /dev/null +++ b/frontend/src/components/DownloadTray.menu.dom.test.tsx @@ -0,0 +1,269 @@ +// @vitest-environment jsdom +// +// The tray's row menu: the copies and cancel of a waiting, running or failed job are its ONE menu, +// opened by the row's "…" (it shows on hover or focus) or a right-click on the row; a failed job's +// first row is Retry. The cancel confirm names its ACTION on its button, and speaks of what is +// cancelled (a download, an upload, a clip). The tray's own showing and done rows are +// DownloadTray.dom.test.tsx's; a finished file's menu is the Files page's (Files.menu.dom.test.tsx). +// i18n is key-passthrough, so labels read as keys. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { useState } from "react"; +import { api, type DownloadJob } from "../lib/api"; +import { menuGroups } from "./ui/menuGroups"; + +const mocks = vi.hoisted(() => ({ + jobs: [] as unknown[], + confirm: vi.fn( + async (_o?: { title: string; message?: string; confirmLabel?: string; danger?: boolean }) => + true, + ), +})); + +vi.mock("react-i18next", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useTranslation: () => ({ i18n: { language: "en" }, t: (key: string) => key }), + }; +}); +vi.mock("./confirmContext", () => ({ useConfirm: () => mocks.confirm })); +vi.mock("../lib/useLiveQuery", () => ({ + useLiveQuery: (key: readonly unknown[]) => ({ + data: key[0] === "downloads" ? mocks.jobs : undefined, + isError: false, + isLoading: false, + refetch: vi.fn(), + }), +})); +vi.mock("../lib/usePref", () => ({ + usePref: (_key: string, initial: T) => useState(initial), +})); +vi.mock("./filesContext", () => ({ useFilesActions: () => ({ openFile: vi.fn() }) })); + +globalThis.ResizeObserver ??= class { + observe() {} + unobserve() {} + disconnect() {} +} as unknown as typeof ResizeObserver; + +import DownloadTray from "./DownloadTray"; + +const job = ( + id: number, + status: DownloadJob["status"], + extra: Partial = {}, +): DownloadJob => + ({ + id, + status, + progress: 0, + speed_bps: null, + eta_s: null, + phase: null, + error: null, + error_code: null, + error_detail: null, + queue_pos: 0, + created_at: null, + source_kind: "youtube", + source_ref: `v${id}`, + source_url: null, + poster_url: null, + profile_id: null, + spec: { mode: "av" }, + display_name: `Clip ${id}`, + display_uploader: null, + display_uploader_url: null, + extra_links: [], + can_download: status === "done", + job_kind: "download", + source_key: "youtube", + asset: { media_kind: "video", duration_s: 60 }, + ...extra, + }) as unknown as DownloadJob; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + mocks.confirm.mockClear(); +}); + +function mount() { + const qc = new QueryClient(); + const tree = () => ( + + + + ); + const { rerender } = render(tree()); + mounted.rerender = () => rerender(tree()); +} + +// The last mount's re-render: what a poll's new data does to the tray. +const mounted = { rerender: () => {} }; + +const rowOf = (text: string) => screen.getByText(text).closest("[data-nav-index]")!; +const moreOf = (text: string) => + rowOf(text).querySelector('button[aria-label="tray.rowMenu"]')!; +const menuRows = () => + screen.getAllByRole("menuitem", { hidden: true }).map((b) => b.textContent ?? ""); +async function openMenu(open: () => void) { + act(open); + await waitFor(() => + expect(screen.getAllByRole("menuitem", { hidden: true }).length).toBeGreaterThan(0), + ); +} +describe("DownloadTray · the row menu", () => { + it("the '…' hides until hover or focus", () => { + mocks.jobs = [job(1, "running")]; + mount(); + expect(moreOf("Clip 1").classList.contains("reveal")).toBe(true); + expect(rowOf("Clip 1").classList.contains("group")).toBe(true); + }); + + it("holds pause, the copies and cancel; a failed job's first row is Retry", async () => { + mocks.jobs = [job(1, "running", { source_url: "https://youtu.be/x" }), job(2, "error")]; + mount(); + await openMenu(() => fireEvent.click(moreOf("Clip 1"))); + expect(menuRows()).toEqual([ + "downloads.actions.pause", + "common.copy.title", + "common.copy.link", + "downloads.actions.cancel", + ]); + expect(moreOf("Clip 1").getAttribute("aria-expanded")).toBe("true"); + fireEvent.keyDown(document.activeElement!, { key: "Escape" }); + await openMenu(() => fireEvent.contextMenu(rowOf("Clip 2"))); + expect(menuRows()).toEqual([ + "downloads.actions.retry", + "common.copy.title", + "downloads.actions.cancel", + ]); + }); + + it("draws Retry as 'do it again', never the Play glyph", async () => { + mocks.jobs = [job(1, "error")]; + mount(); + const glyph = (el: Element) => el.querySelector("svg")?.getAttribute("class") ?? ""; + await openMenu(() => fireEvent.contextMenu(rowOf("Clip 1"))); + const row = screen + .getAllByRole("menuitem", { hidden: true }) + .find((b) => b.textContent === "downloads.actions.retry")!; + expect(glyph(row)).toContain("lucide-refresh-cw"); + }); + + it("closes on a second press of the same '…'", async () => { + mocks.jobs = [job(1, "running")]; + mount(); + await openMenu(() => fireEvent.click(moreOf("Clip 1"))); + // The press lands outside the panel, which floating-ui's dismiss closes on; the click after + // it must not open it again. + act(() => { + fireEvent.pointerDown(moreOf("Clip 1"), { pointerType: "mouse" }); + fireEvent.mouseDown(moreOf("Clip 1")); + fireEvent.click(moreOf("Clip 1"), { detail: 1 }); + }); + await waitFor(() => + expect(screen.queryAllByRole("menuitem", { hidden: true })).toHaveLength(0), + ); + expect(moreOf("Clip 1").getAttribute("aria-expanded")).toBe("false"); + }); + + it("still closes when a poll replaced the job under the open menu", async () => { + mocks.jobs = [job(1, "running")]; + mount(); + await openMenu(() => fireEvent.click(moreOf("Clip 1"))); + // The queue polls while a job runs: the next render lists a NEW object for the same job. + mocks.jobs = [job(1, "running", { progress: 40 })]; + mounted.rerender(); + act(() => { + fireEvent.pointerDown(moreOf("Clip 1"), { pointerType: "mouse" }); + fireEvent.mouseDown(moreOf("Clip 1")); + fireEvent.click(moreOf("Clip 1"), { detail: 1 }); + }); + await waitFor(() => + expect(screen.queryAllByRole("menuitem", { hidden: true })).toHaveLength(0), + ); + }); + + it("cancels only through the confirm", async () => { + mocks.jobs = [job(1, "queued")]; + const cancel = vi.spyOn(api, "cancelDownload").mockResolvedValue({} as never); + mount(); + const pickCancel = async () => { + await openMenu(() => fireEvent.click(moreOf("Clip 1"))); + fireEvent.click( + screen + .getAllByRole("menuitem", { hidden: true }) + .find((b) => b.textContent === "downloads.actions.cancel")!, + ); + }; + mocks.confirm.mockResolvedValueOnce(false); + await pickCancel(); + await waitFor(() => expect(mocks.confirm).toHaveBeenCalledTimes(1)); + expect(mocks.confirm).toHaveBeenCalledWith(expect.objectContaining({ danger: true })); + expect(cancel).not.toHaveBeenCalled(); + await pickCancel(); + await waitFor(() => expect(cancel).toHaveBeenCalledWith(1)); + }); +}); + +describe("DownloadTray · the cancel confirm", () => { + const cancelOf = async (title: string) => { + mount(); + await openMenu(() => fireEvent.click(moreOf(title))); + fireEvent.click( + screen + .getAllByRole("menuitem", { hidden: true }) + .find((b) => b.textContent === "downloads.actions.cancel")!, + ); + await waitFor(() => expect(mocks.confirm).toHaveBeenCalledTimes(1)); + return mocks.confirm.mock.calls[0]![0]!; + }; + + it("labels its button with the action, not the dialog's question", async () => { + mocks.confirm.mockResolvedValueOnce(false); + mocks.jobs = [job(1, "running")]; + const o = await cancelOf("Clip 1"); + expect([o.title, o.confirmLabel]).toEqual([ + "downloads.confirm.cancelTitle", + "downloads.confirm.cancelAction", + ]); + }); + + it("cancelling an upload says the uploaded file goes with it", async () => { + // Nothing to fetch again: the cancel deletes the original from the server. + mocks.confirm.mockResolvedValueOnce(false); + mocks.jobs = [job(3, "queued", { job_kind: "upload" })]; + const o = await cancelOf("Clip 3"); + expect([o.title, o.confirmLabel]).toEqual([ + "downloads.confirm.cancelUploadTitle", + "downloads.confirm.cancelUploadAction", + ]); + }); + + it("cancelling a clip speaks of the clip", async () => { + mocks.confirm.mockResolvedValueOnce(false); + mocks.jobs = [job(4, "running", { job_kind: "edit" })]; + const o = await cancelOf("Clip 4"); + expect([o.title, o.message, o.confirmLabel]).toEqual([ + "downloads.confirm.cancelEditTitle", + "downloads.confirm.cancelEditBody", + "downloads.confirm.cancelEditAction", + ]); + }); +}); + +describe("menuGroups", () => { + const row = (id: string) => ({ id, label: id, onSelect: () => {} }); + it("rules between non-empty groups only", () => { + const out = menuGroups([], [row("a"), row("b")], [], [row("c")]); + expect(out.map((r) => [r.id, !!r.separatorBefore])).toEqual([ + ["a", false], + ["b", false], + ["c", true], + ]); + }); +}); diff --git a/frontend/src/components/DownloadTray.tsx b/frontend/src/components/DownloadTray.tsx new file mode 100644 index 00000000..83e46e6b --- /dev/null +++ b/frontend/src/components/DownloadTray.tsx @@ -0,0 +1,458 @@ +import { useEffect, useEffectEvent, useLayoutEffect, useRef, useState } from "react"; +import { useQueryClient } from "@tanstack/react-query"; +import { useTranslation } from "react-i18next"; +import clsx from "clsx"; +import { + AlertTriangle, + CheckCircle2, + ChevronDown, + FolderOpen, + Link, + Loader2, + PauseCircle, + RotateCw, + X, +} from "lucide-react"; +import type { QueueJob } from "../lib/api"; +import { quickShareLink } from "../lib/downloads"; +import { failCopyKeys, failTitleKey } from "../lib/downloadFailure"; +import { + IN_FLIGHT, + DOWNLOAD_PHASES, + MOVING, + clearFinished, + doneRevealing, + fetchQueue, + takeUnannounced, + useFinished, + useRevealing, + useUnannounced, +} from "../lib/downloadQueue"; +import { anyBlockingLayer } from "../lib/layerStack"; +import { focusNavItem, navItemAttrs } from "../lib/listNav"; +import { notify } from "../lib/notifications"; +import { qk } from "../lib/queryKeys"; +import { LS } from "../lib/storage"; +import { useLiveQuery } from "../lib/useLiveQuery"; +import { usePref } from "../lib/usePref"; +import { ErrorModal } from "./downloadModals"; +import { jobTitle } from "./downloadTitle"; +import { sourceLabel } from "./fileFolders"; +import { useFilesActions } from "./filesContext"; +import { useDownloadActions } from "./useDownloadActions"; +import { type ActionMenuItem } from "./ui/ActionMenu"; +import { setCornerLift } from "./ui/backToTopState"; +import { Button } from "./ui/Button"; +import { ContextMenu } from "./ui/ContextMenu"; +import { IconButton } from "./ui/IconButton"; +import { menuGroups } from "./ui/menuGroups"; +import { MoreButton } from "./ui/MoreButton"; +import NavGrid from "./ui/NavGrid"; +import { Pressable } from "./ui/Pressable"; +import { CONTENT_ID } from "./ui/landmarks"; +import { SURFACE } from "./ui/surfaces"; +import { useContextMenu } from "./ui/useContextMenu"; + +// The running downloads, app-wide (DESIGN.md "Running downloads"): a tray in the bottom-right corner, +// up only while something waits, runs or failed — and what finished in this tab, "Done · Open", +// until it is closed. Folds to a pill (remembered per account). A row's right click and "…" are the +// queue's one menu; a finish is announced once with "Copy link" and "Open" (DESIGN.md "Download +// done"). It sits under every dialog and player (z-chrome), so it never covers a player's controls. + +type Fold = "open" | "folded"; +const parseFold = (v: unknown): Fold => (v === "folded" ? "folded" : "open"); +const parseClosed = (v: unknown): string | null => (typeof v === "string" ? v : null); + +// Byte progress (a bar with a %) vs a post-step that reports none (a pulse): "editing" and +// "converting" report a real % from ffmpeg's out_time. +const BYTE_PHASES = ["video", "audio", "editing", "converting"]; + +const keyOf = (job: QueueJob) => job.id; + +// The step a running job is on is named beside its bar: a download fetches its video and its audio +// as two streams, so the bar fills twice — without the name, the second start reads as a restart. +function Progress({ job }: { job: QueueJob }) { + const { t } = useTranslation(); + const bytes = !job.phase || BYTE_PHASES.includes(job.phase); + const step = + job.phase && DOWNLOAD_PHASES.includes(job.phase) ? t(`downloads.phase.${job.phase}`) : null; + return ( + + {/* One column for the step, so every row's bar starts on the same line. */} + {step && ( + + {step} + + )} + + {bytes ? ( + + ) : ( + + )} + + {bytes && {job.progress}%} + + ); +} + +export default function DownloadTray() { + const { t } = useTranslation(); + const qc = useQueryClient(); + const { openFile } = useFilesActions(); + const actions = useDownloadActions(); + + // Every page polls it now, so the idle cadence is slow: a job started here refetches at once + // (invalidateDownloads), and one started on another device shows within a minute. + const queueQ = useLiveQuery(qk.downloads(), fetchQueue, { + intervalMs: (data) => (data?.some((j) => MOVING.has(j.status)) ? 2000 : 60000), + }); + const finished = useFinished(); + const active = (queueQ.data ?? []).filter((j) => IN_FLIGHT.has(j.status) || j.status === "error"); + const activeIds = new Set(active.map((j) => j.id)); + const rows = [...active, ...finished.filter((j) => !activeIds.has(j.id))]; + + // Closed with work still on it, it stays away until that work changes: a new job, or one failing. + // Remembered (per account) past a reload: an old failure the reader has seen must not reopen it + // on every page they load. + const signature = active.map((j) => `${j.id}${j.status === "error" ? "!" : ""}`).join(","); + const [closedAt, setClosedAt] = usePref("downloadTrayClosed", null, { + localKey: LS.downloadTrayClosed, + parse: parseClosed, + }); + // A finish since it was closed is news of its own, whatever else is still on it. + const shown = rows.length > 0 && (signature !== closedAt || finished.length > 0); + const [fold, setFold] = usePref("downloadTray", "open", { + localKey: LS.downloadTray, + parse: parseFold, + }); + + const [errorJob, setErrorJob] = useState(null); + const menu = useContextMenu({ keyOf }); + + // Folding swaps the control under the keyboard for its counterpart: the focus follows it there + // (the pill, or the fold button), never left on where the pressed one was. + const pill = useRef(null); + const foldButton = useRef(null); + const focusNext = useRef(null); + const toggleFold = (next: Fold) => { + focusNext.current = next; + setFold(next); + }; + useEffect(() => { + const target = focusNext.current; + focusNext.current = null; + if (target === "folded") pill.current?.focus(); + else if (target === "open") foldButton.current?.focus(); + }, [fold]); + + // --- "View" on a job just started: the tray opens on its row ---------------------------------- + // Answered over as many renders as it takes: a closed or folded tray is brought back first, and the + // row is focused once it is on the page. A job the tray does not list is done already (it opens as + // a file), or — once a poll made after the ask has landed without it — not in the queue at all. + // Asked from inside a player (a clip's notice, the player's own download), the tray opens under it + // but leaves the keyboard there: the row is behind the player, out of the reader's sight. + const box = useRef(null); + const revealing = useRevealing(); + const answerReveal = useEffectEvent(() => { + if (!revealing) return; + const index = rows.findIndex((j) => j.id === revealing.id); + if (index >= 0) { + if (!shown) setClosedAt(null); + else if (fold === "folded") setFold("open"); + else { + if (box.current && !anyBlockingLayer()) focusNavItem(box.current, index); + doneRevealing(); + } + return; + } + const job = queueQ.data?.find((j) => j.id === revealing.id); + if (job?.status === "done") { + doneRevealing(); + openFile(job.id, job.folder_id); + } else if (queueQ.dataUpdatedAt > revealing.at) doneRevealing(); + else void qc.invalidateQueries({ queryKey: qk.downloads() }); + }); + useEffect(() => answerReveal(), [revealing, queueQ.data, shown, fold]); + + // --- the done notice ------------------------------------------------------------------------ + const announce = useEffectEvent((job: QueueJob) => { + const title = jobTitle(job); + notify({ + level: "success", + title: t("tray.done.title", { title }), + message: t(job.asset?.media_kind === "file" ? "tray.done.file" : "tray.done.phone"), + action: { label: t("tray.done.copyLink"), onClick: () => void quickShareLink(qc, job, t) }, + secondaryAction: { + label: t("tray.done.open"), + onClick: () => openFile(job.id, job.folder_id), + }, + }); + }); + const fresh = useUnannounced(); + useEffect(() => { + // The take clears them, so a second pass (StrictMode, a re-render) announces nothing again. + if (fresh.length) takeUnannounced().forEach(announce); + }, [fresh]); + + // --- Back to top steps up over the tray while it shows ----------------------------------------- + useLayoutEffect(() => { + const el = box.current; + if (!el) { + setCornerLift(0); + return; + } + const measure = () => setCornerLift(el.getBoundingClientRect().height); + measure(); + const ro = new ResizeObserver(measure); + ro.observe(el); + return () => { + ro.disconnect(); + setCornerLift(0); + }; + }, [shown, fold]); + + if (!shown) + return errorJob ? ( + actions.run(errorJob, "resume")} + onClose={() => setErrorJob(null)} + /> + ) : null; + + const counts = { + running: rows.filter((j) => j.status === "queued" || j.status === "running").length, + paused: rows.filter((j) => j.status === "paused").length, + error: rows.filter((j) => j.status === "error").length, + done: rows.filter((j) => j.status === "done").length, + }; + const summary = (["running", "paused", "error", "done"] as const) + .filter((k) => counts[k] > 0) + .map((k) => t(`tray.count.${k}`, { count: counts[k] })) + .join(", "); + const running = rows.filter((j) => j.status === "running"); + const meanProgress = running.length + ? Math.round(running.reduce((sum, j) => sum + j.progress, 0) / running.length) + : null; + // The tray's state, worst news first; a paused job is neither working nor done. + const StateIcon = counts.running + ? Loader2 + : counts.error + ? AlertTriangle + : counts.paused + ? PauseCircle + : CheckCircle2; + const stateIcon = ( + + ); + + const close = () => { + clearFinished(); + setClosedAt(signature); + // The tray goes with the button pressed: the keyboard rests on the page (its
). + document.getElementById(CONTENT_ID)?.focus({ preventScroll: true }); + }; + + const doneItems = (job: QueueJob): ActionMenuItem[] => + menuGroups([ + { + id: "open", + label: t("tray.done.open"), + icon: , + onSelect: () => openFile(job.id, job.folder_id), + }, + { + id: "copyLink", + label: t("tray.done.copyLink"), + icon: , + onSelect: () => void quickShareLink(qc, job, t), + }, + ]); + const menuItems = menu.target + ? menu.target.status === "done" + ? doneItems(menu.target) + : actions.items(menu.target) + : []; + + return ( +
+ {fold === "folded" ? ( + toggleFold("open")} + aria-expanded={false} + title={t("tray.unfold")} + className={clsx( + SURFACE.floating, + "inline-flex items-center gap-2 px-3 py-1.5 text-sm glass-hover", + )} + > + {stateIcon} + {summary} + {meanProgress !== null && ( + + + + )} + + ) : ( +
+
+ {stateIcon} + + {/* A phone's narrow tray keeps the news and drops the name (the region names it). */} + {t("tray.title")} · + {summary} + + toggleFold("folded")} + > + + + + + +
+ + {rows.map((job, i) => { + const title = jobTitle(job); + const mine = menu.open && menu.target?.id === job.id; + return ( +
+
+ + + {title} + + + {sourceLabel(job.source_key, t)} + + + setErrorJob(job)} + onRetry={() => actions.run(job, "resume")} + onOpen={() => openFile(job.id, job.folder_id)} + /> +
+ +
+ ); + })} + +
+ )} + + {errorJob && ( + actions.run(errorJob, "resume")} + onClose={() => setErrorJob(null)} + /> + )} +
+ ); +} + +/** What a tray row says under its title: how far it is, that it waits, why it failed, or that it is + * done — with the one thing to do about it in reach. */ +function RowState({ + job, + onShowError, + onRetry, + onOpen, +}: { + job: QueueJob; + onShowError: () => void; + onRetry: () => void; + onOpen: () => void; +}) { + const { t } = useTranslation(); + if (job.status === "running") return ; + if (job.status === "error") + return ( + + + + {t(failCopyKeys(job, "short"))} + + + + ); + if (job.status === "done") + return ( + + + {t("tray.done.label")} + + + ); + return {t(`downloads.status.${job.status}`)}; +} diff --git a/frontend/src/components/FileCard.tsx b/frontend/src/components/FileCard.tsx new file mode 100644 index 00000000..aaceae90 --- /dev/null +++ b/frontend/src/components/FileCard.tsx @@ -0,0 +1,117 @@ +import clsx from "clsx"; +import { useTranslation } from "react-i18next"; +import { Scissors } from "lucide-react"; +import type { LibraryFile } from "../lib/api"; +import { navItemAttrs } from "../lib/listNav"; +import { JobThumb } from "./downloadRow"; +import { jobTitle } from "./downloadTitle"; +import { PhoneBadge, RetentionBadge, SourceBadge } from "./fileBadges"; +import { MediaCard, MediaPlayGlyph } from "./ui/MediaCard"; +import { + MEDIA_BADGE, + MEDIA_BAR, + MEDIA_HOVER_LAYER, + MEDIA_PICTURE, + mediaPictureBox, + mediaTitleClass, +} from "./ui/mediaCardStyles"; +import { MoreButton } from "./ui/MoreButton"; +import { Pressable } from "./ui/Pressable"; + +// A library file in the Files grid: the one media card (ui/MediaCard) — its picture plays it (a file +// that is not media: opens it in the viewer), the title under it, then where it came from and what +// needs saying about it (kept, leaving soon). The +// right click, the long press and the "…" open the same menu as its table row (DESIGN.md "Files page": +// the grid is the list's other paint, a toggle remembered per folder). + +export function FileCard({ + file, + index, + onOpen, + menuProps, + moreProps, + menuOpen, + arrived, +}: { + file: LibraryFile; + /** Its place on the page, for the keyboard walk. */ + index: number; + /** Plays it, or opens it in the viewer. Null: a shared file its owner keeps from me. */ + onOpen: (() => void) | null; + menuProps: Record; + moreProps: Record; + menuOpen: boolean; + /** The file a link or a notice opened the page on. */ + arrived: boolean; +}) { + const { t } = useTranslation(); + const title = jobTitle(file); + const channel = file.display_uploader || file.asset?.uploader || null; + const clip = file.job_kind === "edit" || file.job_kind === "clip"; + const media = file.asset?.media_kind !== "file"; + const thumb = ; + return ( + + {onOpen ? ( + + {thumb} + {media && ( +
+ +
+ )} + + ) : ( +
{thumb}
+ )} + {clip && ( + + + {t("downloads.clipBadge")} + + )} + + {/* `media`: legible on any picture, the Plex cards' hover row's look. */} + + +
+ } + title={ + + {title} + + } + meta={ + + + + {channel && {channel}} + + + + + + + } + more={{ props: moreProps, open: menuOpen }} + moreLabel={t("files.rowMenu")} + /> + ); +} diff --git a/frontend/src/components/FilePreview.dom.test.tsx b/frontend/src/components/FilePreview.dom.test.tsx new file mode 100644 index 00000000..1f87fed4 --- /dev/null +++ b/frontend/src/components/FilePreview.dom.test.tsx @@ -0,0 +1,223 @@ +// @vitest-environment jsdom +// +// The Files viewer: what each kind of file shows (an image, a PDF in a frame, text as text — HTML too, +// never a page — or the "no preview" card), ← / → through the files it was given in their order, +// and closing when the file it shows leaves the list. i18n is key-passthrough. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { api, HttpError, type LibraryFile } from "../lib/api"; + +vi.mock("react-i18next", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useTranslation: () => ({ + i18n: { language: "en" }, + t: (key: string, o?: Record) => + o && "n" in o ? `${key}:${String(o.n)}/${String(o.total)}` : key, + }), + }; +}); + +import FilePreview from "./FilePreview"; +import { pushLayer, removeLayer } from "../lib/layerStack"; + +const file = (id: number, preview: string | null, extra: Record = {}) => + ({ + id, + status: "done", + display_name: `File ${id}`, + can_download: true, + job_kind: "upload", + shared: false, + source_key: "upload", + asset: { media_kind: "file", preview, container: "x", size_bytes: 4096, ...extra }, + }) as unknown as LibraryFile; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +function mount(files: LibraryFile[], fileId: number, onClose = vi.fn()) { + const onFileChange = vi.fn(); + const view = (list: LibraryFile[], id: number) => ( + + []} + /> + + ); + const r = render(view(files, fileId)); + return { + onFileChange, + onClose, + rerender: (l: LibraryFile[], id: number) => r.rerender(view(l, id)), + }; +} + +describe("FilePreview · what it shows", () => { + it("an image as an image, from the sandboxed preview", () => { + mount([file(1, "image")], 1); + const img = screen.getByRole("img", { name: "File 1" }); + expect(img.getAttribute("src")).toBe(api.previewUrl(1)); + }); + + it("a PDF in a frame of the preview, the browser's own viewer", () => { + mount([file(2, "pdf")], 2); + const frame = document.querySelector("iframe")!; + expect(frame.getAttribute("src")).toBe(api.previewUrl(2)); + expect(frame.title).toBe("File 2"); + }); + + it("a page's source as text, line by line — never as a page", async () => { + vi.spyOn(api, "previewText").mockResolvedValue({ + text: "

hi

\n", + truncated: false, + }); + mount([file(3, "text")], 3); + await screen.findByText("

hi

"); + expect(screen.getByText("")).toBeTruthy(); + expect(document.getElementById("x")).toBeNull(); + expect(document.querySelector("script")).toBeNull(); + expect(screen.getByText("files.preview.asText")).toBeTruthy(); + }); + + it("numbers a file's lines, the newline that ends it making no line of its own", async () => { + vi.spyOn(api, "previewText").mockResolvedValue({ text: "one\r\ntwo\n", truncated: false }); + mount([file(8, "text")], 8); + await screen.findByText("one"); + expect([...document.querySelectorAll("li")].map((li) => li.textContent)).toEqual([ + "1one", + "2two", + ]); + }); + + it("lays out a long text's first lines only, and says where the rest is", async () => { + // Every line is a row of the page: a log of tens of thousands would stall the viewer. + const text = Array.from({ length: 6000 }, (_, i) => `line ${i + 1}`).join( + String.fromCharCode(10), + ); + vi.spyOn(api, "previewText").mockResolvedValue({ text, truncated: false }); + mount([file(9, "text")], 9); + await screen.findByText("line 1"); + expect(document.querySelectorAll("li")).toHaveLength(5000); + expect(screen.getByText(/files\.preview\.firstLines/)).toBeTruthy(); + expect(screen.getByText("files.preview.downloadWhole").closest("a")!.getAttribute("href")).toBe( + api.fileUrl(9), + ); + }); + + it("says when it shows only the start of a long text", async () => { + vi.spyOn(api, "previewText").mockResolvedValue({ text: "a", truncated: true }); + mount([file(4, "text")], 4); + expect(await screen.findByText("files.preview.truncated")).toBeTruthy(); + }); + + it("a text the server refuses (binary after all) gets the no-preview card, not an error", async () => { + vi.spyOn(api, "previewText").mockRejectedValue(new HttpError(415, "x", {})); + mount([file(5, "text")], 5); + expect(await screen.findByText("files.preview.none")).toBeTruthy(); + expect(screen.queryByRole("alert")).toBeNull(); + // A new tab would only show the refusal. + expect(screen.queryByRole("link", { name: "files.preview.openInTab" })).toBeNull(); + }); + + it("a text that fails to load says so, with a retry", async () => { + vi.spyOn(api, "previewText").mockRejectedValue(new HttpError(500, "x", {})); + mount([file(6, "text")], 6); + expect(await screen.findByRole("alert", {}, { timeout: 3000 })).toBeTruthy(); + expect(screen.getByRole("button", { name: "common.retry" })).toBeTruthy(); + }); + + it("anything else: the card that offers the file itself", () => { + mount([file(7, null)], 7); + expect(screen.getByText("files.preview.none")).toBeTruthy(); + const save = screen.getAllByRole("link", { name: "downloads.actions.download" }); + expect(save.every((a) => a.getAttribute("href") === api.fileUrl(7))).toBe(true); + // Nothing to open in a tab: the preview would refuse it. + expect(screen.queryByRole("link", { name: "files.preview.openInTab" })).toBeNull(); + }); +}); + +describe("FilePreview · stepping", () => { + const list = [file(1, "image"), file(2, "pdf"), file(3, null)]; + + it("← / → go to the files beside it, in the order given", () => { + const { onFileChange } = mount(list, 2); + expect(screen.getByText("files.preview.position:2/3")).toBeTruthy(); + fireEvent.keyDown(window, { key: "ArrowRight" }); + expect(onFileChange).toHaveBeenLastCalledWith(3); + fireEvent.keyDown(window, { key: "ArrowLeft" }); + expect(onFileChange).toHaveBeenLastCalledWith(1); + }); + + it("a layer opened from it (a menu, a confirm) keeps its arrows — the viewer stays on its file", () => { + const { onFileChange } = mount(list, 2); + const confirm = pushLayer(); + try { + fireEvent.keyDown(window, { key: "ArrowRight" }); + expect(onFileChange).not.toHaveBeenCalled(); + } finally { + removeLayer(confirm); + } + fireEvent.keyDown(window, { key: "ArrowRight" }); + expect(onFileChange).toHaveBeenLastCalledWith(3); + }); + + it("a text line wider than the stage scrolls sideways under ← / →, it does not page", async () => { + vi.spyOn(api, "previewText").mockResolvedValue({ text: "a very long line", truncated: false }); + const { onFileChange } = mount([file(1, "image"), file(8, "text"), file(9, "text")], 8); + await screen.findByText("a very long line"); + const region = screen.getByRole("region"); + Object.defineProperty(region, "scrollWidth", { value: 900 }); + Object.defineProperty(region, "clientWidth", { value: 300 }); + fireEvent.keyDown(region, { key: "ArrowRight" }); + expect(onFileChange).not.toHaveBeenCalled(); + }); + + it("a text whose lines fit pages under ← / →, even with the keyboard in it", async () => { + vi.spyOn(api, "previewText").mockResolvedValue({ text: "short", truncated: false }); + const { onFileChange } = mount([file(1, "image"), file(8, "text"), file(9, "text")], 8); + await screen.findByText("short"); + const region = screen.getByRole("region"); + Object.defineProperty(region, "scrollWidth", { value: 300 }); + Object.defineProperty(region, "clientWidth", { value: 300 }); + fireEvent.keyDown(region, { key: "ArrowRight" }); + expect(onFileChange).toHaveBeenLastCalledWith(9); + }); + + it("Escape closes it — after a layer opened above it has taken its own", () => { + const { onClose } = mount(list, 2); + const menu = pushLayer(); + try { + fireEvent.keyDown(document, { key: "Escape" }); + expect(onClose).not.toHaveBeenCalled(); + } finally { + removeLayer(menu); + } + fireEvent.keyDown(document, { key: "Escape" }); + expect(onClose).toHaveBeenCalledTimes(1); + }); + + it("stops at either end", () => { + const { onFileChange } = mount(list, 1); + fireEvent.keyDown(window, { key: "ArrowLeft" }); + expect(onFileChange).not.toHaveBeenCalled(); + expect( + screen.getByRole("button", { name: "files.preview.prev" }).hasAttribute("disabled"), + ).toBe(true); + }); + + it("closes when the file it shows leaves the list", async () => { + const { onClose, rerender } = mount(list, 2); + expect(onClose).not.toHaveBeenCalled(); + act(() => rerender([list[0]!, list[2]!], 2)); + await waitFor(() => expect(onClose).toHaveBeenCalled()); + }); +}); diff --git a/frontend/src/components/FilePreview.tsx b/frontend/src/components/FilePreview.tsx new file mode 100644 index 00000000..a37f6a3f --- /dev/null +++ b/frontend/src/components/FilePreview.tsx @@ -0,0 +1,313 @@ +import { useEffect, useEffectEvent, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useTranslation } from "react-i18next"; +import { + ChevronLeft, + ChevronRight, + Download, + ExternalLink, + FileQuestion, + WrapText, +} from "lucide-react"; +import { api, HttpError, type FilePreviewKind, type LibraryFile } from "../lib/api"; +import { formatBytes } from "../lib/format"; +import { dispatch, useKeyScope } from "../lib/keymap"; +import { useLayer } from "../lib/layerStack"; +import { qk } from "../lib/queryKeys"; +import { jobTitle } from "./downloadTitle"; +import { type ActionMenuItem } from "./ui/ActionMenu"; +import { ContextMenu } from "./ui/ContextMenu"; +import { IconButton } from "./ui/IconButton"; +import { IconLink, LinkButton } from "./ui/LinkButton"; +import Modal from "./ui/Modal"; +import { MoreButton } from "./ui/MoreButton"; +import { LoadingState, StateMessage } from "./ui/QueryState"; +import { useContextMenu } from "./ui/useContextMenu"; + +// The Files viewer (DESIGN.md "Files preview"): the library's files that are not media, one at a time, +// in the shared dialog shell. What a file shows is the server's call (`asset.preview`, +// backend downloads/preview.py): an image as an image, a PDF in the browser's own viewer, a text +// format as plain text — HTML and SVG too, never rendered: on this origin a page would run with the +// reader's session. The rest gets a card that offers the file itself. ← / → step through `files`, the +// list's own order, so the viewer pages the way the list reads. +// +// Every kind fills the same stage, so a step never resizes the dialog: its ‹ › stay under the pointer. +const STAGE = "h-[min(75dvh,56rem)] w-full"; +// How many lines of a text file the viewer lays out (see TextStage). +const TEXT_LINES = 5000; + +/** A text file's head (one read, shared by the viewer's header and its stage). */ +function useTextPreview(id: number, enabled: boolean) { + return useQuery({ + queryKey: qk.filePreview(id), + queryFn: () => api.previewText(id), + enabled, + // A refusal (the "text" turned out binary) is an answer, not a blip to retry. + retry: (n, e) => !(e instanceof HttpError && e.status < 500) && n < 1, + }); +} +/** The server's "no preview" for a file it listed as text: it was binary after all. */ +const refused = (e: unknown) => e instanceof HttpError && e.status === 415; + +export default function FilePreview({ + files, + fileId, + onFileChange, + onClose, + menuItems, +}: { + /** The files the viewer steps through, in the list's order. */ + files: readonly LibraryFile[]; + fileId: number; + onFileChange: (id: number) => void; + onClose: () => void; + /** The file's row menu, behind the viewer's "…". */ + menuItems: (file: LibraryFile) => ActionMenuItem[]; +}) { + const { t } = useTranslation(); + const at = files.findIndex((f) => f.id === fileId); + const file = at >= 0 ? files[at] : undefined; + // Gone from the list under the viewer (deleted from its menu, or by another tab): nothing to show. + useEffect(() => { + if (!file) onClose(); + }, [file, onClose]); + + const step = (by: number) => { + const next = files[at + by]; + if (next) onFileChange(next.id); + }; + useKeyScope("fileViewer"); + // The viewer's own place on the layer stack: ← / → are its only while nothing is on top of it — its + // "…" menu, a confirm, the Share or Details dialog opened from that menu. There an arrow moves + // through the layer's own buttons; paging the viewer under it would show one file while the dialog + // acts on another. It mounts with its dialog (both only while there is a file), and effects run + // child first, so it sits just ABOVE the dialog's layer: the dialog is never topmost, and Escape is + // this layer's to take — passive, so the dialog keeps its Tab trap. + const layer = useLayer(!!file, onClose, { passive: true }); + const onKey = useEffectEvent((e: KeyboardEvent) => { + if (layer.blockingAbove()) return; + // The text, scrolled sideways by its arrows: a line wider than the stage is read that way. + const scroller = + e.target instanceof Element ? e.target.closest("[data-scrolls-sideways]") : null; + if (scroller && scroller.scrollWidth > scroller.clientWidth) return; + dispatch(e, "fileViewer", { + prevFile: (ev) => { + ev.preventDefault(); + step(-1); + }, + nextFile: (ev) => { + ev.preventDefault(); + step(1); + }, + }); + }); + useEffect(() => { + const handler = (e: KeyboardEvent) => onKey(e); + window.addEventListener("keydown", handler); + return () => window.removeEventListener("keydown", handler); + }, []); + + const menu = useContextMenu({ keyOf: (f) => String(f.id) }); + const text = useTextPreview(fileId, file?.asset?.preview === "text"); + if (!file) return null; + const kind = file.asset?.preview ?? null; + // A new tab would only show the refusal. + const openable = kind !== null && !refused(text.error); + const title = jobTitle(file); + + return ( + {title}} + onClose={onClose} + maxWidth="max-w-6xl" + headerActions={ + <> + step(-1)} + > + + + + {t("files.preview.position", { n: at + 1, total: files.length })} + + = files.length - 1} + onClick={() => step(1)} + > + + + {openable ? ( + + + + ) : ( + // Its place kept, so the icons after it do not shift from one file to the next. + + )} + + + + + + + } + > + {/* Keyed by file: a step starts the next one's own load, never a frame of the last one. */} + + + ); +} + +function Stage({ + file, + kind, + title, +}: { + file: LibraryFile; + kind: FilePreviewKind | null; + title: string; +}) { + const { t } = useTranslation(); + const [failed, setFailed] = useState(false); + if (kind === "image") + return failed ? ( + + {t("common.loadError")} + + ) : ( +
+ {title} setFailed(true)} + className="max-h-full max-w-full rounded-md object-contain" + /> +
+ ); + if (kind === "pdf") + return ( +