From d66174ed16071cff5c630be2cf9b1040a1106716 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 23:01:12 +0200 Subject: [PATCH 01/25] test(arch): a unit's HTTP layer does no database work beyond its pin The fence is switched on unit by unit: each route file's database work (SQL built, session data calls, model attributes set, unit tables imported) is pinned at today's count and may only shrink. A core route may not reach into a unit's tables. Every model has an owner. --- backend/tests/test_unit_fences.py | 306 ++++++++++++++++++++++++++++++ docs/gates.md | 1 + 2 files changed, 307 insertions(+) create mode 100644 backend/tests/test_unit_fences.py diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py new file mode 100644 index 00000000..f933560d --- /dev/null +++ b/backend/tests/test_unit_fences.py @@ -0,0 +1,306 @@ +"""The HTTP layer of a unit does no database work of its own, held in code rather than prose. + +A route validates, calls its unit's `queries` (reads) or `service` (writes), serializes and commits. +What it may not do is build SQL, call the session's data methods, change a model's attributes or +import a unit's tables: when a route did those, ownership, the commit and the conditional `WHERE` +lived in a different place in each handler, and a fix in one never reached its siblings. + +The rule is switched on unit by unit. `ALLOWED` pins what each HTTP file still does today, and the +pin is exact: a count that rises fails (new database work in a route), and one that falls fails +until its entry is lowered (the fence only tightens, as `baseline.json` does for basedpyright). An +empty `ALLOWED` is the end state — a file listed there is a debt, not an exemption. + +A core route (identity, admin, ops) is not a unit's HTTP layer and may use the session; it may +still not reach into a unit's tables. +""" + +import ast +from pathlib import Path + +from app.models import Base + +APP = Path(__file__).parent.parent / "app" +ROUTES = APP / "routes" + +# Each unit's tables. A new model is placed here (or in CORE_MODELS) by the test below, so a table +# never joins the schema without an owner. +UNIT_MODELS = { + "capture": { + "MediaAsset", + "DownloadProfile", + "FileFolder", + "DownloadJob", + "DownloadShare", + "DownloadQuota", + "DownloadLink", + "DownloadCredential", + "LibraryState", + }, + "reader": { + "VideoState", + "Playlist", + "PlaylistItem", + "SavedView", + "ComfortPin", + "Tag", + "ChannelTag", + }, + "catalog": { + "Channel", + "Subscription", + "Video", + "SearchFind", + "ExploredChannel", + "BlockedChannel", + }, + "plex": { + "PlexLibrary", + "PlexShow", + "PlexSeason", + "PlexItem", + "PlexPerson", + "PlexCollection", + "PlexState", + "PlexLink", + "PlexPlaylist", + "PlexPlaylistItem", + }, +} +CORE_MODELS = { + "User", + "OAuthToken", + "AuthToken", + "Invite", + "ApiQuotaUsage", + "QuotaEvent", + "AuditLog", + "LogEvent", + "DebugSession", + "DebugScreenshot", + "AppState", + "JobRun", + "SystemConfig", + "Notification", + "ApiToken", + "ApiTokenGrant", +} + +# Which unit each HTTP file belongs to (path relative to app/routes; a folder covers its files). +# Anything not listed is a core route. +UNIT_ROUTES = { + "downloads.py": "capture", + "files.py": "capture", + "public.py": "capture", + "feed.py": "reader", + "playlists.py": "reader", + "saved_views.py": "reader", + "discover.py": "reader", + "tags.py": "reader", + "channels.py": "catalog", + "search.py": "catalog", + "sync.py": "catalog", + "plex/": "plex", +} + +# What each HTTP file still does, by file: the debt a unit's sprint pays off. Exact counts. A core +# route's count is only the unit tables it imports; it is paid by the sprint of the unit it reaches. +ALLOWED = { + "channels.py": 83, + "discover.py": 7, + "downloads.py": 88, + "feed.py": 36, + "files.py": 58, + "playlists.py": 72, + "plex/_shared.py": 1, + "plex/admin.py": 12, + "plex/browse.py": 93, + "plex/collections.py": 33, + "plex/detail.py": 29, + "plex/discover.py": 44, + "plex/people.py": 20, + "plex/playback.py": 42, + "plex/playlists.py": 80, + "public.py": 12, + "runs.py": 1, + "saved_views.py": 17, + "scheduler.py": 3, + "search.py": 46, + "sync.py": 38, + "tags.py": 10, + "tokens.py": 1, +} + +# The session methods that read or write data. `commit` and `rollback` stay with the handler. +SESSION_DATA = { + "add", + "add_all", + "delete", + "execute", + "get", + "merge", + "flush", + "refresh", + "query", + "scalar", + "scalars", + "expunge", +} +# What a route may name from SQLAlchemy: the session's type, for its `db: Session` parameter, and +# the errors its commit raises (the commit is the handler's, so is catching what it throws). +SQLALCHEMY_ALLOWED = {"Session"} +SQLALCHEMY_ALLOWED_MODULES = {"sqlalchemy.exc"} +# Objects a route owns and may set attributes on; anything else it writes to is a model. +OWN_OBJECTS = {"self", "cls", "request", "response"} +UNIT_MODEL_NAMES = {name: unit for unit, names in UNIT_MODELS.items() for name in names} + + +def _unit_of(rel: str) -> str | None: + for key, unit in UNIT_ROUTES.items(): + if rel == key or (key.endswith("/") and rel.startswith(key)): + return unit + return None + + +def _session_names(tree: ast.AST) -> set[str]: + """The names a module binds to a database session: a parameter typed `Session`, or a name + assigned (or `with`-bound) from `SessionLocal()`.""" + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.arg) and isinstance(node.annotation, ast.Name): + if node.annotation.id == "Session": + names.add(node.arg) + made = None + if isinstance(node, ast.Assign) and len(node.targets) == 1: + made, target = node.value, node.targets[0] + elif isinstance(node, ast.withitem): + made, target = node.context_expr, node.optional_vars + if ( + isinstance(made, ast.Call) + and isinstance(made.func, ast.Name) + and made.func.id == "SessionLocal" + and isinstance(target, ast.Name) + ): + names.add(target.id) + return names + + +def _root(node: ast.expr) -> ast.expr: + while isinstance(node, (ast.Attribute, ast.Subscript)): + node = node.value + return node + + +def violations(source: str, unit: str | None) -> list[str]: + """Each piece of database work in an HTTP file, as `line: what`. A core route (`unit` None) is + held only to the last rule: no unit's tables.""" + tree = ast.parse(source) + found = [] + sql_names = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + module = node.module or "" + for alias in node.names: + bound = alias.asname or alias.name + if ( + module.split(".")[0] == "sqlalchemy" + and module not in SQLALCHEMY_ALLOWED_MODULES + and alias.name not in SQLALCHEMY_ALLOWED + ): + sql_names.add(bound) + if ( + alias.name in UNIT_MODEL_NAMES + and module.startswith("app.") + and "models" in module + ): + found.append( + f"{node.lineno}: imports {UNIT_MODEL_NAMES[alias.name]}'s {alias.name}" + ) + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name.split(".")[0] == "sqlalchemy": + sql_names.add((alias.asname or alias.name).split(".")[0]) + if unit is None: + return found + sessions = _session_names(tree) + for node in ast.walk(tree): + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) and node.id in sql_names: + found.append(f"{node.lineno}: uses sqlalchemy's {node.id}") + if isinstance(node, ast.Call): + f = node.func + if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name): + if f.value.id in sessions and f.attr in SESSION_DATA: + found.append(f"{node.lineno}: {f.value.id}.{f.attr}()") + targets = [] + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): + targets = [node.target] + for target in targets: + if isinstance(target, ast.Attribute): + base = _root(target) + if not (isinstance(base, ast.Name) and base.id in OWN_OBJECTS): + found.append(f"{node.lineno}: sets .{target.attr}") + return sorted(found, key=lambda v: int(v.split(":")[0])) + + +def _http_files(): + for path in sorted(ROUTES.rglob("*.py")): + rel = path.relative_to(ROUTES).as_posix() + yield rel, path.read_text(encoding="utf-8") + + +def test_every_table_has_an_owner(): + tables = {mapper.class_.__name__ for mapper in Base.registry.mappers} + owned = set(UNIT_MODEL_NAMES) | CORE_MODELS + assert tables - owned == set(), "a model with no unit: place it in UNIT_MODELS or CORE_MODELS" + assert owned - tables == set(), "UNIT_MODELS/CORE_MODELS name a model that no longer exists" + assert not set(UNIT_MODEL_NAMES) & CORE_MODELS + + +def test_every_listed_route_exists(): + rels = [rel for rel, _ in _http_files()] + for key in UNIT_ROUTES: + assert any(rel == key or (key.endswith("/") and rel.startswith(key)) for rel in rels), key + assert set(ALLOWED) <= set(rels), set(ALLOWED) - set(rels) + + +def test_no_http_file_does_database_work_beyond_its_pin(): + drift = {} + for rel, src in _http_files(): + found = violations(src, _unit_of(rel)) + pinned = ALLOWED.get(rel, 0) + if len(found) != pinned: + drift[rel] = (pinned, len(found), found if len(found) > pinned else "lower its pin") + assert drift == {}, ( + "an HTTP file's database work moved off its pin (pinned, now, what): move new work into" + " the unit's queries/service; when a count fell, lower ALLOWED to it" + ) + + +# The detector itself: each kind of database work it must see, and what it must leave alone. +DETECTOR_CASES = { + "from sqlalchemy import select\nq = select(1)": 1, + "import sqlalchemy as sa\nsa.select(1).where(sa.true())": 2, + "from sqlalchemy import func\nn = func.count()": 1, + "from sqlalchemy.orm import Session\ndef f(db: Session): db.commit(); db.rollback()": 0, + "from sqlalchemy.orm import Session\ndef f(db: Session): db.add(x)": 1, + "def f(conn: Session): conn.execute(q)": 1, + "from app.db import SessionLocal\nwith SessionLocal() as s:\n s.get(X, 1)": 1, + "from app.db import SessionLocal\ns = SessionLocal()\ns.scalars(q)": 1, + "def f(job): job.status = 'x'": 1, + "def f(job): job.meta['k'] = 1": 0, + "def f(job): job.parts.count += 1": 1, + "def f(self, request, response): self.x = 1; request.state.y = 2; response.status_code = 3": 0, + "from app.models import DownloadJob, User": 1, + "from app.capture.models import DownloadJob as J": 1, + "def f(cache): cache.get('k')": 0, + "from sqlalchemy.exc import IntegrityError\ntry:\n pass\nexcept IntegrityError:\n pass": 0, +} + + +def test_the_detector_sees_each_kind_of_database_work(): + for source, expected in DETECTOR_CASES.items(): + assert len(violations(source, "capture")) == expected, source + assert violations("from app.models import Video\nfrom sqlalchemy import select", None) == [ + "1: imports catalog's Video" + ] diff --git a/docs/gates.md b/docs/gates.md index 75537e98..14276a61 100644 --- a/docs/gates.md +++ b/docs/gates.md @@ -113,6 +113,7 @@ project checks needs a row. | CVE gate | HIGH/CRITICAL with a fix available only | `--ignore-unfixed` + `--severity HIGH,CRITICAL`: a finding with no fix cannot be acted on by an upgrade; it shows in the scan output. A fixable one fails the publish, and the only way past it is `.trivyignore.yaml`. | 0 open (E32 S1) | | npm audit | `--omit=dev --audit-level=high` | What ships to the browser, at high or above. The dev toolchain's advisories were burned down by hand in E32 S1 (18 → 0). | 0 | | knip | `entry` — one file added | Not a narrowing: knip's Vite plugin reads entries from index.html only, so the shared link's page (`watch.html` → `src/watchMain.tsx`) is named here, or the lane reports that whole entry unused. It adds files to the lane, never takes any out. | 1 entry | +| unit fence | the database work the HTTP layer still does, pinned | `backend/tests/test_unit_fences.py`: a unit's route file may not build SQL, call the session's data methods, set a model's attributes or import a unit's tables; a core route may not import a unit's tables. Switched on unit by unit: `ALLOWED` pins each file's count exactly, so a rise fails and a fall fails until the pin is lowered — the list may only shrink. End state: empty, and the pin mechanism goes. | 826 in 23 files (E42 S0) | | React Compiler | the functions it skips, pinned | `components/compilerCoverage.contract.test.ts` — the list may only shrink. End state: only VirtualGrid (TanStack Virtual, an IncompatibleLibrary) remains. | 93 (E32 S1) → 29 (E32 S3) → 1, VirtualGrid (E32 S4) | ## Accepted findings From 3a688fa0f5fb761e112894b8fc4f96c0da1c0dad Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 23:45:53 +0200 Subject: [PATCH 02/25] refactor(capture): the unit's tables move into downloads/models The nine Capture tables (files, jobs, folders, shares, links, quota, cookies, profiles, library resume points) leave app/models.py for the unit's own module; app.tables registers every unit's tables for the migrations, the test database and the ownership check. No schema change. --- backend/alembic/env.py | 2 +- backend/app/downloads/backfill.py | 2 +- backend/app/downloads/credentials.py | 2 +- backend/app/downloads/folders.py | 2 +- backend/app/downloads/gc.py | 2 +- backend/app/downloads/jobdb.py | 2 +- backend/app/downloads/links.py | 2 +- backend/app/downloads/models.py | 385 ++++++++++++++++++ backend/app/downloads/og.py | 2 +- backend/app/downloads/pipeline.py | 3 +- backend/app/downloads/quota.py | 2 +- backend/app/downloads/runner.py | 2 +- backend/app/downloads/service.py | 11 +- backend/app/downloads/upload.py | 3 +- backend/app/models.py | 361 +--------------- backend/app/routes/downloads.py | 5 +- backend/app/routes/files.py | 6 +- backend/app/routes/public.py | 3 +- backend/app/routes/tokens.py | 3 +- backend/app/tables.py | 9 + backend/app/worker.py | 3 +- backend/tests/conftest.py | 2 +- backend/tests/test_api_tokens.py | 12 +- backend/tests/test_clip.py | 3 +- backend/tests/test_download_credentials.py | 3 +- .../tests/test_download_failure_catalog.py | 3 +- backend/tests/test_download_job_locks.py | 3 +- backend/tests/test_download_limits.py | 3 +- backend/tests/test_download_ref_count.py | 3 +- backend/tests/test_download_share_notice.py | 3 +- backend/tests/test_download_transitions.py | 3 +- backend/tests/test_downloads_finalize.py | 3 +- backend/tests/test_edit_cap.py | 2 +- backend/tests/test_failure_lines.py | 2 +- backend/tests/test_file_folders.py | 3 +- backend/tests/test_files_api.py | 3 +- backend/tests/test_files_preview.py | 2 +- backend/tests/test_folder_links.py | 5 +- backend/tests/test_library_progress.py | 2 +- backend/tests/test_link_token_at_rest.py | 3 +- backend/tests/test_not_found_detail.py | 2 +- backend/tests/test_odd_frame.py | 2 +- backend/tests/test_phone_safe_contract.py | 2 +- backend/tests/test_unit_fences.py | 3 +- backend/tests/test_upload.py | 3 +- backend/tests/test_upload_conversion.py | 3 +- backend/tests/test_worker_dispatch.py | 2 +- .../src/lib/api/downloads.contract.test.ts | 6 +- 48 files changed, 469 insertions(+), 429 deletions(-) create mode 100644 backend/app/downloads/models.py create mode 100644 backend/app/tables.py diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 2870589f..7bd6efbe 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config, pool from app.config import settings from app.db import Base from app.logs import configure_logging -import app.models # noqa: F401 (import so models register on Base.metadata) +import app.tables # noqa: F401 (import so every table registers on Base.metadata) config = context.config config.set_main_option("sqlalchemy.url", settings.database_url) diff --git a/backend/app/downloads/backfill.py b/backend/app/downloads/backfill.py index 13d035ab..ae400902 100644 --- a/backend/app/downloads/backfill.py +++ b/backend/app/downloads/backfill.py @@ -22,7 +22,7 @@ from app.db import SessionLocal from app.downloads import adapters from app.downloads import edit as editmod from app.downloads import formats, service -from app.models import MediaAsset +from app.downloads.models import MediaAsset log = logs.get_logger(__name__) diff --git a/backend/app/downloads/credentials.py b/backend/app/downloads/credentials.py index 648c441d..026634e2 100644 --- a/backend/app/downloads/credentials.py +++ b/backend/app/downloads/credentials.py @@ -33,7 +33,7 @@ from app import logs from app.downloads import clip as clipmod from app.downloads import formats from app.downloads.adapters import _host_is -from app.models import DownloadCredential, MediaAsset +from app.downloads.models import DownloadCredential, MediaAsset from app.security import decrypt_optional, encrypt log = logs.get_logger(__name__) diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py index 50f2d98e..a431a6bc 100644 --- a/backend/app/downloads/folders.py +++ b/backend/app/downloads/folders.py @@ -32,7 +32,7 @@ from sqlalchemy.orm import Session from app import sysconfig from app.downloads.status import JobStatus -from app.models import DownloadJob, FileFolder, MediaAsset +from app.downloads.models import DownloadJob, FileFolder, MediaAsset class FolderKind(StrEnum): diff --git a/backend/app/downloads/gc.py b/backend/app/downloads/gc.py index 50f6bb42..9cfd052f 100644 --- a/backend/app/downloads/gc.py +++ b/backend/app/downloads/gc.py @@ -31,7 +31,7 @@ from app import apitokens, logs, sysconfig from app.config import settings from app.downloads import quota, service, storage from app.downloads.status import JobStatus -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset from app.notifications import create_notification log = logs.get_logger(__name__) diff --git a/backend/app/downloads/jobdb.py b/backend/app/downloads/jobdb.py index a8e95b29..14b34b4a 100644 --- a/backend/app/downloads/jobdb.py +++ b/backend/app/downloads/jobdb.py @@ -6,7 +6,7 @@ from sqlalchemy import text from app.config import settings from app.db import SessionLocal -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset # --- small DB helpers (short-lived sessions; the download itself holds no transaction) ------- diff --git a/backend/app/downloads/links.py b/backend/app/downloads/links.py index 80a0ac77..5d46335a 100644 --- a/backend/app/downloads/links.py +++ b/backend/app/downloads/links.py @@ -24,7 +24,7 @@ from app import security from app.config import settings from app.downloads import storage from app.downloads.status import JobStatus -from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset +from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset _GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks # The most files a folder link lists: as many as the owner's own Files list shows (`routes.files. diff --git a/backend/app/downloads/models.py b/backend/app/downloads/models.py new file mode 100644 index 00000000..79661895 --- /dev/null +++ b/backend/app/downloads/models.py @@ -0,0 +1,385 @@ +"""Capture's tables: the downloaded files (`MediaAsset`), each user's jobs over them and the folders +they sit in, the shares and public links that hand them on, a user's quota, cookies and format +profiles, and where they stopped playing one. Only this unit writes them; another unit reads them +along a declared edge, or asks this unit's service.""" + +from datetime import date, datetime, timezone + +from sqlalchemy import ( + BigInteger, + Boolean, + CheckConstraint, + Date, + DateTime, + ForeignKey, + Index, + Integer, + JSON, + String, + Text, + UniqueConstraint, + func, + text, +) +from sqlalchemy.orm import Mapped, mapped_column, validates + +from app.db import Base +from app.models import DISPLAY_TEXT_MAX, TimestampMixin, UpdatedAtMixin + + +class LibraryState(Base): + """Where a user stopped in a library item (their own download, or one shared with them) — the + library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry, + a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive + both. A revoked share leaves its recipient's row behind, unread — only a job the user can still + reach is ever read or written — and it is back if the item is shared again. Rows exist only for a + position worth keeping (see `routes._common.kept_position`); no row means "from the start".""" + + __tablename__ = "library_states" + __table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + job_id: Mapped[int] = mapped_column( + ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True + ) + # int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500). + position_seconds: Mapped[int] = mapped_column(Integer, nullable=False) + progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class MediaAsset(Base, TimestampMixin): + """A physical downloaded media file — the SHARED cache layer of the download center. + + Mirrors the app's "shared catalog, private per-user state" model: one row per unique + ``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests + the same video with a format that hashes to the same `format_sig`, their `download_job` + links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is + shared; per-user edited derivatives (phase 2) are never cached here. + + `rel_path` is relative to `settings.files_root` (the mount), so the physical filename is + system-decided and bound to the source id — never renamed (the user's custom name is display + metadata on `download_job`, applied only when downloading to their own machine). The naming + fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built + uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees + the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file + stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's).""" + + __tablename__ = "media_assets" + __table_args__ = ( + UniqueConstraint( + "source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key" + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" + source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc + format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec + status: Mapped[str] = mapped_column( + String(16), default="pending", server_default="pending", index=True + ) # "pending" | "ready" | "error" + rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root + storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic + # A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the + # card / watch page / link preview can still show an image. Relative to files_root. + poster_path: Mapped[str | None] = mapped_column(Text) + size_bytes: Mapped[int | None] = mapped_column(BigInteger) + duration_s: Mapped[int | None] = mapped_column(Integer) + width: Mapped[int | None] = mapped_column(Integer) + height: Mapped[int | None] = mapped_column(Integer) + container: Mapped[str | None] = mapped_column(String(16)) + vcodec: Mapped[str | None] = mapped_column(String(32)) + acodec: Mapped[str | None] = mapped_column(String(32)) + # "video" | "audio" | "file" — the media-kind discriminator that drives which library actions + # apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is + # still pending (kind is known once the file is probed at finalize). It is the single stored home + # of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an + # earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here). + media_kind: Mapped[str | None] = mapped_column(String(16)) + nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + # Denormalized naming/metadata for the Plex tree + ad-hoc display. + title: Mapped[str | None] = mapped_column(Text) + uploader: Mapped[str | None] = mapped_column(String(255)) + # yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel + # render as a clickable link. Null for sources without one (e.g. reddit) or older rows. + uploader_url: Mapped[str | None] = mapped_column(Text) + upload_date: Mapped[date | None] = mapped_column(Date) + thumbnail_url: Mapped[str | None] = mapped_column(Text) + # yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from" + # reference shown in the Files library. Null until the worker extracts it (older/queued rows + # fall back to a URL derived from source_kind+source_ref). + source_webpage_url: Mapped[str | None] = mapped_column(Text) + error: Mapped[str | None] = mapped_column(Text) + ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + # The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload + # streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet. + sha256: Mapped[str | None] = mapped_column(String(64)) + + @validates("rel_path") + def _forget_hash(self, _key: str, rel_path: str | None) -> str | None: + # A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them. + self.sha256 = None + return rel_path + + +class DownloadProfile(Base, TimestampMixin): + """A named download preset — a reusable format profile. + + `user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …); + a user's saved custom profiles carry their `user_id`. `spec` is the format definition + (mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/ + thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache + `format_sig`.""" + + __tablename__ = "download_profiles" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) # NULL for a builtin preset + name: Mapped[str] = mapped_column(String(80)) + spec: Mapped[dict] = mapped_column(JSON) + is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class FileFolder(Base, TimestampMixin): + """A folder of the Files library (`app.downloads.folders`): a row, never a directory. + + A SOURCE folder (`source_key` set: youtube / plex / upload / site:) is made the first time a + user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder + (neither set, one per user) is where they swap files with a machine of theirs. Every file — + a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a + file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it.""" + + __tablename__ = "file_folders" + __table_args__ = ( + UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"), + Index( + "uq_file_folder_own_name", + "user_id", + func.lower(text("name")), + unique=True, + postgresql_where=text("name IS NOT NULL"), + ), + # One Hub folder per user. + Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")), + CheckConstraint( + "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)" + " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)" + " OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)", + name="ck_file_folder_kind", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind + # "site:" + a host name (DNS caps one at 253 characters). + source_key: Mapped[str | None] = mapped_column(String(260)) + name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) + retention_days: Mapped[int | None] = mapped_column(Integer) + + +def _ready_on_insert(context) -> datetime | None: + """A job inserted already done (an upload kept as it came) was ready the moment it was made; one + that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`).""" + done = context.get_current_parameters().get("status") == "done" + return datetime.now(timezone.utc) if done else None + + +class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): + """A user's request to download one source — the per-user layer over `MediaAsset`. + + The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live + `progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is + no server push channel, and the worker is a separate process anyway). `profile_snapshot` + freezes the spec at enqueue so later edits/deletion of the profile don't change this job. + `display_name` is the user's own name for the file (display + the Content-Disposition on + local download); the on-disk asset is never renamed. `asset_id` links to the shared file + (SET NULL if the asset is GC'd — the job then shows as expired). + + Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded + ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key) + so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the + asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id` + point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe.""" + + __tablename__ = "download_jobs" + __table_args__ = ( + Index("ix_download_jobs_claim", "status", "queue_pos"), + # Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`). + CheckConstraint( + "status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at" + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + asset_id: Mapped[int | None] = mapped_column( + ForeignKey("media_assets.id", ondelete="SET NULL"), index=True + ) + source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" + source_ref: Mapped[str] = mapped_column(String(512)) + # "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a + # user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or + # "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`). + job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download") + # For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted). + source_job_id: Mapped[int | None] = mapped_column( + ForeignKey("download_jobs.id", ondelete="SET NULL") + ) + source_asset_id: Mapped[int | None] = mapped_column( + ForeignKey("media_assets.id", ondelete="SET NULL") + ) + # For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip + # jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop + # in fractions of the box it was drawn in (`downloads.clip.normalize`). + edit_spec: Mapped[dict | None] = mapped_column(JSON) + profile_id: Mapped[int | None] = mapped_column( + ForeignKey("download_profiles.id", ondelete="SET NULL") + ) + profile_snapshot: Mapped[dict] = mapped_column(JSON) + display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) + # Per-user display overrides + extras (like display_name — metadata only, never the file). + # `display_uploader`/`display_uploader_url` override the asset's auto channel name/link; + # `extra_links` is an optional list of extra reference URLs shown on the card + watch page. + display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) + display_uploader_url: Mapped[str | None] = mapped_column(Text) + extra_links: Mapped[list | None] = mapped_column(JSON) + status: Mapped[str] = mapped_column( + String(16), default="queued", server_default="queued", index=True + ) # queued | running | paused | done | error | canceled + progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + speed_bps: Mapped[int | None] = mapped_column(BigInteger) + eta_s: Mapped[int | None] = mapped_column(Integer) + phase: Mapped[str | None] = mapped_column(String(32)) + error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail) + error_code: Mapped[str | None] = mapped_column( + String(24) + ) # category, see downloads.errors.classify + # The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL. + error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose + queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + # The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own + # folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether + # the user keeps it past its folder's retention, and when it goes (stamped once done; NULL = + # never). `expiry_warned` marks the one warning the GC sends before it deletes the file. + folder_id: Mapped[int | None] = mapped_column( + ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True + ) + # The origin the file came from (the source folder's key), recorded where it is placed: an edit + # traces its origin through its parent, and that parent may be deleted or expire later — the + # file must still know which source folder it may go back to. NULL only for a file placed before + # the column existed outside a source folder; `folders.source_key_for` works those out. + source_key: Mapped[str | None] = mapped_column(String(260)) + kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) + expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + # When the file became done — what the library's "Added" means: a download that failed for days + # and then went through is new to its user, so it sorts as new. `created_at` stays when it was + # asked for (the queue's order). Set again when a file is made anew (a retry, a re-download). + ready_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), default=_ready_on_insert + ) + + +class DownloadShare(Base, TimestampMixin): + """A user→user grant: `to_user` may access `from_user`'s downloaded file (private by + default). Since assets are already deduped/shared on disk, a share is just an ACL row.""" + + __tablename__ = "download_shares" + __table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),) + + id: Mapped[int] = mapped_column(primary_key=True) + job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE")) + from_user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + + +class DownloadQuota(Base, UpdatedAtMixin): + """Per-user download limits (admin-managed). A row exists only when an admin customizes a + user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the + byte cap entirely (e.g. for the admin's own account).""" + + __tablename__ = "download_quotas" + + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + max_bytes: Mapped[int] = mapped_column(BigInteger) + max_concurrent: Mapped[int] = mapped_column(Integer) + max_jobs: Mapped[int] = mapped_column(Integer) + unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + + +class DownloadLink(Base, TimestampMixin): + """A public capability link to one download, or to one of its owner's own folders — the "share + by link" surface. + + Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}` + player page (Google-Drive style); a folder link opens the folder's list there, with what is in + the folder at the time it is opened. The token itself is never stored as written: a link is + found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner + copies the link from again — absent without a key (`downloads/links.py`). Optional per-link + controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash` + (argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which + grants a *registered* user access via the in-app "Shared with me" list.""" + + __tablename__ = "download_links" + __table_args__ = ( + # A link opens exactly one thing: a file or a folder. + CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + job_id: Mapped[int | None] = mapped_column( + ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True + ) + folder_id: Mapped[int | None] = mapped_column( + ForeignKey("file_folders.id", ondelete="CASCADE"), index=True + ) + token_hash: Mapped[str] = mapped_column(String(64), unique=True) + token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged + created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + password_hash: Mapped[str | None] = mapped_column(Text) + # Bumped whenever the password changes; folded into the signed watch grant so rotating the + # password immediately invalidates any grants issued under the old one (instead of them living + # out the 6h grant TTL). + password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin): + """A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …). + + Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host + where they're logged in (e.g. a closed Facebook group they belong to), and a download of a + yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The + blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit — + the routes are write-only (host + updated_at only). See `downloads.credentials`. + + Two load-bearing security controls live around this row, NOT on it: + * **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the + user id, so the authed asset is private to that user and can never be cache-served to + another user (`downloads.credentials.cookie_salt`). + * **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches + this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=` + can't exfiltrate the user's session to a look-alike host. + + `host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the + raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch).""" + + __tablename__ = "download_credentials" + __table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + host_key: Mapped[str] = mapped_column(String(64)) + cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged diff --git a/backend/app/downloads/og.py b/backend/app/downloads/og.py index 10a0e37e..f2d4d29c 100644 --- a/backend/app/downloads/og.py +++ b/backend/app/downloads/og.py @@ -16,7 +16,7 @@ from sqlalchemy.orm import Session from app.config import settings from app.downloads import links as linksmod from app.downloads import storage -from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset +from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset _OG_BLOCK = re.compile(r".*?", re.DOTALL) diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index 307d3886..c6ae8622 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -28,7 +28,8 @@ from app.downloads.jobdb import ( ) from app.downloads.stated import StatedFailure from app.downloads.status import JobStatus -from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem +from app.models import DISPLAY_TEXT_MAX, PlexItem +from app.downloads.models import DownloadJob, MediaAsset from app.plex import lookups as plex_lookups from app.plex import paths as plex_paths from app.notifications import create_notification diff --git a/backend/app/downloads/quota.py b/backend/app/downloads/quota.py index d5832910..ef58665a 100644 --- a/backend/app/downloads/quota.py +++ b/backend/app/downloads/quota.py @@ -26,7 +26,7 @@ from sqlalchemy.orm import Session from app import sysconfig from app.downloads.status import JobKind, JobStatus -from app.models import DownloadJob, DownloadQuota, MediaAsset +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset # Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures # (error, canceled) don't count. diff --git a/backend/app/downloads/runner.py b/backend/app/downloads/runner.py index 9af835ef..bbcf3db3 100644 --- a/backend/app/downloads/runner.py +++ b/backend/app/downloads/runner.py @@ -23,7 +23,7 @@ from app.db import SessionLocal from app.downloads import edit as editmod from app.downloads import formats from app.downloads.status import JobKind -from app.models import DownloadJob +from app.downloads.models import DownloadJob log = logs.get_logger(__name__) diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index bbe5d82c..54169d71 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -21,15 +21,8 @@ from app.downloads import credentials from app.downloads import edit as editmod from app.downloads import folders, formats, mediakind, quota, storage from app.downloads.status import HOLDING, JobKind, JobStatus -from app.models import ( - DISPLAY_TEXT_MAX, - Channel, - DownloadJob, - MediaAsset, - PlexItem, - PlexShow, - Video, -) +from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, Video +from app.downloads.models import DownloadJob, MediaAsset from app.plex import lookups as plex_lookups from app.plex import paths as plex_paths diff --git a/backend/app/downloads/upload.py b/backend/app/downloads/upload.py index 49291c39..6ef5e159 100644 --- a/backend/app/downloads/upload.py +++ b/backend/app/downloads/upload.py @@ -37,7 +37,8 @@ from app.config import settings from app.downloads import edit as editmod from app.downloads import folders, mediakind, playable, probe, quota, service, storage from app.downloads.status import JobKind, JobStatus -from app.models import DownloadJob, MediaAsset, display_text +from app.models import display_text +from app.downloads.models import DownloadJob, MediaAsset def _title_from_filename(filename: str) -> str: diff --git a/backend/app/models.py b/backend/app/models.py index 9ebfbe74..000baa44 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,4 +1,4 @@ -from datetime import date, datetime, timezone +from datetime import date, datetime from sqlalchemy import ( BigInteger, @@ -18,7 +18,6 @@ from sqlalchemy import ( Text, UniqueConstraint, func, - text, ) from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR from sqlalchemy.orm import Mapped, mapped_column, relationship, validates @@ -465,27 +464,6 @@ class VideoState(Base, UpdatedAtMixin): progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) -class LibraryState(Base): - """Where a user stopped in a library item (their own download, or one shared with them) — the - library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry, - a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive - both. A revoked share leaves its recipient's row behind, unread — only a job the user can still - reach is ever read or written — and it is back if the item is shared again. Rows exist only for a - position worth keeping (see `routes._common.kept_position`); no row means "from the start".""" - - __tablename__ = "library_states" - __table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) - job_id: Mapped[int] = mapped_column( - ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True - ) - # int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500). - position_seconds: Mapped[int] = mapped_column(Integer, nullable=False) - progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - - class SearchFind(Base, TimestampMixin): """Per-user record of a video the user surfaced via their own live YouTube search. @@ -861,141 +839,6 @@ class Notification(Base): ) -class MediaAsset(Base, TimestampMixin): - """A physical downloaded media file — the SHARED cache layer of the download center. - - Mirrors the app's "shared catalog, private per-user state" model: one row per unique - ``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests - the same video with a format that hashes to the same `format_sig`, their `download_job` - links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is - shared; per-user edited derivatives (phase 2) are never cached here. - - `rel_path` is relative to `settings.files_root` (the mount), so the physical filename is - system-decided and bound to the source id — never renamed (the user's custom name is display - metadata on `download_job`, applied only when downloading to their own machine). The naming - fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built - uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees - the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file - stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's).""" - - __tablename__ = "media_assets" - __table_args__ = ( - UniqueConstraint( - "source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key" - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" - source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc - format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec - status: Mapped[str] = mapped_column( - String(16), default="pending", server_default="pending", index=True - ) # "pending" | "ready" | "error" - rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root - storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic - # A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the - # card / watch page / link preview can still show an image. Relative to files_root. - poster_path: Mapped[str | None] = mapped_column(Text) - size_bytes: Mapped[int | None] = mapped_column(BigInteger) - duration_s: Mapped[int | None] = mapped_column(Integer) - width: Mapped[int | None] = mapped_column(Integer) - height: Mapped[int | None] = mapped_column(Integer) - container: Mapped[str | None] = mapped_column(String(16)) - vcodec: Mapped[str | None] = mapped_column(String(32)) - acodec: Mapped[str | None] = mapped_column(String(32)) - # "video" | "audio" | "file" — the media-kind discriminator that drives which library actions - # apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is - # still pending (kind is known once the file is probed at finalize). It is the single stored home - # of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an - # earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here). - media_kind: Mapped[str | None] = mapped_column(String(16)) - nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - # Denormalized naming/metadata for the Plex tree + ad-hoc display. - title: Mapped[str | None] = mapped_column(Text) - uploader: Mapped[str | None] = mapped_column(String(255)) - # yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel - # render as a clickable link. Null for sources without one (e.g. reddit) or older rows. - uploader_url: Mapped[str | None] = mapped_column(Text) - upload_date: Mapped[date | None] = mapped_column(Date) - thumbnail_url: Mapped[str | None] = mapped_column(Text) - # yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from" - # reference shown in the Files library. Null until the worker extracts it (older/queued rows - # fall back to a URL derived from source_kind+source_ref). - source_webpage_url: Mapped[str | None] = mapped_column(Text) - error: Mapped[str | None] = mapped_column(Text) - ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - # The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload - # streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet. - sha256: Mapped[str | None] = mapped_column(String(64)) - - @validates("rel_path") - def _forget_hash(self, _key: str, rel_path: str | None) -> str | None: - # A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them. - self.sha256 = None - return rel_path - - -class DownloadProfile(Base, TimestampMixin): - """A named download preset — a reusable format profile. - - `user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …); - a user's saved custom profiles carry their `user_id`. `spec` is the format definition - (mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/ - thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache - `format_sig`.""" - - __tablename__ = "download_profiles" - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int | None] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), index=True - ) # NULL for a builtin preset - name: Mapped[str] = mapped_column(String(80)) - spec: Mapped[dict] = mapped_column(JSON) - is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - - -class FileFolder(Base, TimestampMixin): - """A folder of the Files library (`app.downloads.folders`): a row, never a directory. - - A SOURCE folder (`source_key` set: youtube / plex / upload / site:) is made the first time a - user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder - (neither set, one per user) is where they swap files with a machine of theirs. Every file — - a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a - file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it.""" - - __tablename__ = "file_folders" - __table_args__ = ( - UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"), - Index( - "uq_file_folder_own_name", - "user_id", - func.lower(text("name")), - unique=True, - postgresql_where=text("name IS NOT NULL"), - ), - # One Hub folder per user. - Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")), - CheckConstraint( - "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)" - " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)" - " OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)", - name="ck_file_folder_kind", - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) - kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind - # "site:" + a host name (DNS caps one at 253 characters). - source_key: Mapped[str | None] = mapped_column(String(260)) - name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) - retention_days: Mapped[int | None] = mapped_column(Integer) - - class ApiToken(Base, TimestampMixin): """A user's token for a machine to reach their Files library with (`app.apitokens`): sent as `Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its @@ -1037,208 +880,6 @@ class ApiTokenGrant(Base): can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") -def _ready_on_insert(context) -> datetime | None: - """A job inserted already done (an upload kept as it came) was ready the moment it was made; one - that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`).""" - done = context.get_current_parameters().get("status") == "done" - return datetime.now(timezone.utc) if done else None - - -class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): - """A user's request to download one source — the per-user layer over `MediaAsset`. - - The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live - `progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is - no server push channel, and the worker is a separate process anyway). `profile_snapshot` - freezes the spec at enqueue so later edits/deletion of the profile don't change this job. - `display_name` is the user's own name for the file (display + the Content-Disposition on - local download); the on-disk asset is never renamed. `asset_id` links to the shared file - (SET NULL if the asset is GC'd — the job then shows as expired). - - Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded - ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key) - so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the - asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id` - point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe.""" - - __tablename__ = "download_jobs" - __table_args__ = ( - Index("ix_download_jobs_claim", "status", "queue_pos"), - # Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`). - CheckConstraint( - "status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at" - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) - asset_id: Mapped[int | None] = mapped_column( - ForeignKey("media_assets.id", ondelete="SET NULL"), index=True - ) - source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" - source_ref: Mapped[str] = mapped_column(String(512)) - # "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a - # user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or - # "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`). - job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download") - # For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted). - source_job_id: Mapped[int | None] = mapped_column( - ForeignKey("download_jobs.id", ondelete="SET NULL") - ) - source_asset_id: Mapped[int | None] = mapped_column( - ForeignKey("media_assets.id", ondelete="SET NULL") - ) - # For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip - # jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop - # in fractions of the box it was drawn in (`downloads.clip.normalize`). - edit_spec: Mapped[dict | None] = mapped_column(JSON) - profile_id: Mapped[int | None] = mapped_column( - ForeignKey("download_profiles.id", ondelete="SET NULL") - ) - profile_snapshot: Mapped[dict] = mapped_column(JSON) - display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) - # Per-user display overrides + extras (like display_name — metadata only, never the file). - # `display_uploader`/`display_uploader_url` override the asset's auto channel name/link; - # `extra_links` is an optional list of extra reference URLs shown on the card + watch page. - display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) - display_uploader_url: Mapped[str | None] = mapped_column(Text) - extra_links: Mapped[list | None] = mapped_column(JSON) - status: Mapped[str] = mapped_column( - String(16), default="queued", server_default="queued", index=True - ) # queued | running | paused | done | error | canceled - progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - speed_bps: Mapped[int | None] = mapped_column(BigInteger) - eta_s: Mapped[int | None] = mapped_column(Integer) - phase: Mapped[str | None] = mapped_column(String(32)) - error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail) - error_code: Mapped[str | None] = mapped_column( - String(24) - ) # category, see downloads.errors.classify - # The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL. - error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose - queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - # The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own - # folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether - # the user keeps it past its folder's retention, and when it goes (stamped once done; NULL = - # never). `expiry_warned` marks the one warning the GC sends before it deletes the file. - folder_id: Mapped[int | None] = mapped_column( - ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True - ) - # The origin the file came from (the source folder's key), recorded where it is placed: an edit - # traces its origin through its parent, and that parent may be deleted or expire later — the - # file must still know which source folder it may go back to. NULL only for a file placed before - # the column existed outside a source folder; `folders.source_key_for` works those out. - source_key: Mapped[str | None] = mapped_column(String(260)) - kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) - expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - # When the file became done — what the library's "Added" means: a download that failed for days - # and then went through is new to its user, so it sorts as new. `created_at` stays when it was - # asked for (the queue's order). Set again when a file is made anew (a retry, a re-download). - ready_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), default=_ready_on_insert - ) - - -class DownloadShare(Base, TimestampMixin): - """A user→user grant: `to_user` may access `from_user`'s downloaded file (private by - default). Since assets are already deduped/shared on disk, a share is just an ACL row.""" - - __tablename__ = "download_shares" - __table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),) - - id: Mapped[int] = mapped_column(primary_key=True) - job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE")) - from_user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), index=True - ) - to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) - - -class DownloadQuota(Base, UpdatedAtMixin): - """Per-user download limits (admin-managed). A row exists only when an admin customizes a - user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the - byte cap entirely (e.g. for the admin's own account).""" - - __tablename__ = "download_quotas" - - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), primary_key=True - ) - max_bytes: Mapped[int] = mapped_column(BigInteger) - max_concurrent: Mapped[int] = mapped_column(Integer) - max_jobs: Mapped[int] = mapped_column(Integer) - unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - - -class DownloadLink(Base, TimestampMixin): - """A public capability link to one download, or to one of its owner's own folders — the "share - by link" surface. - - Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}` - player page (Google-Drive style); a folder link opens the folder's list there, with what is in - the folder at the time it is opened. The token itself is never stored as written: a link is - found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner - copies the link from again — absent without a key (`downloads/links.py`). Optional per-link - controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash` - (argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which - grants a *registered* user access via the in-app "Shared with me" list.""" - - __tablename__ = "download_links" - __table_args__ = ( - # A link opens exactly one thing: a file or a folder. - CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - job_id: Mapped[int | None] = mapped_column( - ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True - ) - folder_id: Mapped[int | None] = mapped_column( - ForeignKey("file_folders.id", ondelete="CASCADE"), index=True - ) - token_hash: Mapped[str] = mapped_column(String(64), unique=True) - token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged - created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) - allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - password_hash: Mapped[str | None] = mapped_column(Text) - # Bumped whenever the password changes; folded into the signed watch grant so rotating the - # password immediately invalidates any grants issued under the old one (instead of them living - # out the 6h grant TTL). - password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - - -class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin): - """A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …). - - Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host - where they're logged in (e.g. a closed Facebook group they belong to), and a download of a - yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The - blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit — - the routes are write-only (host + updated_at only). See `downloads.credentials`. - - Two load-bearing security controls live around this row, NOT on it: - * **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the - user id, so the authed asset is private to that user and can never be cache-served to - another user (`downloads.credentials.cookie_salt`). - * **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches - this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=` - can't exfiltrate the user's session to a look-alike host. - - `host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the - raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch).""" - - __tablename__ = "download_credentials" - __table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) - host_key: Mapped[str] = mapped_column(String(64)) - cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged - - # --- Plex integration (optional) ------------------------------------------------------------- # A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is # from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py index 2c897fe1..31d68988 100644 --- a/backend/app/routes/downloads.py +++ b/backend/app/routes/downloads.py @@ -33,7 +33,8 @@ from app.downloads import quota, service, storage from app.downloads import urlguard from app.downloads.folders import FolderKind from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus -from app.models import ( +from app.models import User, display_text +from app.downloads.models import ( DownloadJob, DownloadLink, FileFolder, @@ -42,8 +43,6 @@ from app.models import ( DownloadShare, LibraryState, MediaAsset, - User, - display_text, ) from app.routes._common import ( ProgressIn, diff --git a/backend/app/routes/files.py b/backend/app/routes/files.py index d777b79b..fcf385d8 100644 --- a/backend/app/routes/files.py +++ b/backend/app/routes/files.py @@ -44,16 +44,14 @@ from app.downloads import folders, preview, quota, service, storage from app.downloads import upload as uploadmod from app.downloads.status import JobStatus from app.mailtext import lang_for_user -from app.models import ( - DISPLAY_TEXT_MAX, +from app.models import DISPLAY_TEXT_MAX, User, display_text +from app.downloads.models import ( DownloadJob, DownloadLink, DownloadShare, FileFolder, LibraryState, MediaAsset, - User, - display_text, ) from app.refusal import refused from app.routes._common import capped_stream_to, not_found, owned_or_404 diff --git a/backend/app/routes/public.py b/backend/app/routes/public.py index d0d69ff8..1fc6949c 100644 --- a/backend/app/routes/public.py +++ b/backend/app/routes/public.py @@ -17,7 +17,8 @@ from app.db import get_db from app.downloads import links as linksmod from app.downloads import storage from app.downloads.status import JobStatus -from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset from app.ratelimit import RateLimiter, client_key from app.security import verify_password from app.refusal import refused diff --git a/backend/app/routes/tokens.py b/backend/app/routes/tokens.py index 40e3edc2..34426c2c 100644 --- a/backend/app/routes/tokens.py +++ b/backend/app/routes/tokens.py @@ -22,7 +22,8 @@ from app.audit import AuditAction from app.auth import current_user from app.db import get_db from app.downloads import folders -from app.models import ApiToken, ApiTokenGrant, FileFolder, User +from app.models import ApiToken, ApiTokenGrant, User +from app.downloads.models import FileFolder from app.refusal import refused from app.routes._common import owned_or_404 from app.security import hash_token diff --git a/backend/app/tables.py b/backend/app/tables.py new file mode 100644 index 00000000..5727fe9b --- /dev/null +++ b/backend/app/tables.py @@ -0,0 +1,9 @@ +"""Every table, registered on `Base.metadata`. + +Each unit declares its tables in its own `models` module, so importing `app.models` alone no longer +names the whole schema. Whatever needs all of it — the migrations, the test database, a check over +every mapper — imports this module. +""" + +import app.downloads.models # Capture's tables +import app.models # noqa: F401 — the core tables diff --git a/backend/app/worker.py b/backend/app/worker.py index b8180456..11db8327 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -46,7 +46,8 @@ from app.downloads.jobdb import ( _staging_dir, ) from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus -from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset +from app.models import DISPLAY_TEXT_MAX +from app.downloads.models import DownloadJob, MediaAsset log = logs.get_logger(__name__) _stop = threading.Event() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 32e614ae..c00ea44f 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -25,7 +25,7 @@ from fastapi.testclient import TestClient from sqlalchemy import create_engine, event, text from starlette.requests import HTTPConnection -import app.models # register every model on Base.metadata +import app.tables # every table on Base.metadata import app.state as app_state from app import apitokens, security from app.auth import current_user diff --git a/backend/tests/test_api_tokens.py b/backend/tests/test_api_tokens.py index aa7e64c9..ca70144d 100644 --- a/backend/tests/test_api_tokens.py +++ b/backend/tests/test_api_tokens.py @@ -25,16 +25,8 @@ from app.config import settings from app.db import SessionLocal, get_db from app.main import app from app.notifications import create_notification -from app.models import ( - ApiToken, - ApiTokenGrant, - AuditLog, - DownloadJob, - FileFolder, - MediaAsset, - Notification, - User, -) +from app.models import ApiToken, ApiTokenGrant, AuditLog, Notification, User +from app.downloads.models import DownloadJob, FileFolder, MediaAsset from app.ratelimit import RateLimiter from app.routes import tokens as tokens_routes from app.routes.auth import bump_session_epoch diff --git a/backend/tests/test_clip.py b/backend/tests/test_clip.py index a0af274c..6b492959 100644 --- a/backend/tests/test_clip.py +++ b/backend/tests/test_clip.py @@ -18,7 +18,8 @@ from app.downloads import errors as dlerrors from app.downloads import formats, pipeline, quota, runner, service from app.downloads import probe as probemod from app.downloads.edit import INPUT_GUARD -from app.models import Channel, DownloadJob, MediaAsset, PlexItem, PlexLibrary, PlexShow, Video +from app.models import Channel, PlexItem, PlexLibrary, PlexShow, Video +from app.downloads.models import DownloadJob, MediaAsset from app.plex import paths as plex_paths # --- the spec ----------------------------------------------------------------------------------- diff --git a/backend/tests/test_download_credentials.py b/backend/tests/test_download_credentials.py index 33314592..f5f4c2ed 100644 --- a/backend/tests/test_download_credentials.py +++ b/backend/tests/test_download_credentials.py @@ -18,7 +18,8 @@ from cryptography.fernet import Fernet from app import security from app.downloads import credentials, formats, service -from app.models import DownloadCredential, DownloadJob, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadCredential, DownloadJob, MediaAsset @pytest.fixture(autouse=True) diff --git a/backend/tests/test_download_failure_catalog.py b/backend/tests/test_download_failure_catalog.py index 10ea902d..9f221473 100644 --- a/backend/tests/test_download_failure_catalog.py +++ b/backend/tests/test_download_failure_catalog.py @@ -24,7 +24,8 @@ from app import security from app.config import settings from app.db import engine from app.downloads import credentials, formats, pipeline, service -from app.models import DownloadJob, DownloadQuota, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset from app.worker import _process_job _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e" diff --git a/backend/tests/test_download_job_locks.py b/backend/tests/test_download_job_locks.py index 0d69faf5..a053dda0 100644 --- a/backend/tests/test_download_job_locks.py +++ b/backend/tests/test_download_job_locks.py @@ -18,7 +18,8 @@ from sqlalchemy import select from app.apitokens import FilesCaller from app.db import SessionLocal from app.downloads import service -from app.models import DownloadJob, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, MediaAsset from app.routes.downloads import cancel_download, pause_download from app.routes.files import delete_file, own_job_for_update diff --git a/backend/tests/test_download_limits.py b/backend/tests/test_download_limits.py index e2d97fe9..40422dc1 100644 --- a/backend/tests/test_download_limits.py +++ b/backend/tests/test_download_limits.py @@ -13,7 +13,8 @@ its OWN session, so every test COMMITS its setup before calling it. from app.config import settings from app.downloads import errors as dlerrors from app.downloads import formats, pipeline, quota -from app.models import DownloadJob, DownloadQuota, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset from app.worker import _claim_job diff --git a/backend/tests/test_download_ref_count.py b/backend/tests/test_download_ref_count.py index 3b8a52ae..534b613b 100644 --- a/backend/tests/test_download_ref_count.py +++ b/backend/tests/test_download_ref_count.py @@ -13,7 +13,8 @@ DB lane (`db` fixture) — skipped without Postgres. from sqlalchemy import select, text from app.downloads import service -from app.models import DownloadJob, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, MediaAsset def _user(db, email="ref@example.com"): diff --git a/backend/tests/test_download_share_notice.py b/backend/tests/test_download_share_notice.py index 85c12376..3833a75d 100644 --- a/backend/tests/test_download_share_notice.py +++ b/backend/tests/test_download_share_notice.py @@ -4,7 +4,8 @@ you") — the only place the title lived, so a Hungarian reader got the English. from sqlalchemy import select -from app.models import DownloadJob, MediaAsset, Notification +from app.models import Notification +from app.downloads.models import DownloadJob, MediaAsset def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user): diff --git a/backend/tests/test_download_transitions.py b/backend/tests/test_download_transitions.py index 212e22d6..057a93a2 100644 --- a/backend/tests/test_download_transitions.py +++ b/backend/tests/test_download_transitions.py @@ -8,7 +8,8 @@ behaviour-equivalently. DB lane (`db` fixture) — skipped without Postgres. from sqlalchemy import text -from app.models import DownloadJob, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, MediaAsset def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None): diff --git a/backend/tests/test_downloads_finalize.py b/backend/tests/test_downloads_finalize.py index 3b793e2e..b93f5ef1 100644 --- a/backend/tests/test_downloads_finalize.py +++ b/backend/tests/test_downloads_finalize.py @@ -11,7 +11,8 @@ from sqlalchemy import text from app.downloads import service from app.downloads.status import JobStatus -from app.models import DownloadJob, User +from app.models import User +from app.downloads.models import DownloadJob def _running_job(db) -> DownloadJob: diff --git a/backend/tests/test_edit_cap.py b/backend/tests/test_edit_cap.py index fbc52efa..f88fc339 100644 --- a/backend/tests/test_edit_cap.py +++ b/backend/tests/test_edit_cap.py @@ -13,7 +13,7 @@ from app.config import settings from app.downloads import errors as dlerrors from app.downloads import pipeline, quota, service from app.downloads import upload as uploadmod -from app.models import DownloadJob, DownloadQuota, MediaAsset +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") diff --git a/backend/tests/test_failure_lines.py b/backend/tests/test_failure_lines.py index 3ade4eee..4e43e501 100644 --- a/backend/tests/test_failure_lines.py +++ b/backend/tests/test_failure_lines.py @@ -20,7 +20,7 @@ import yt_dlp from app import logs, sysconfig, worker from app.downloads import pipeline from app.downloads.stated import StatedFailure -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset from app.plex import stream as plex_stream diff --git a/backend/tests/test_file_folders.py b/backend/tests/test_file_folders.py index a3b71fc1..b13b6f79 100644 --- a/backend/tests/test_file_folders.py +++ b/backend/tests/test_file_folders.py @@ -17,7 +17,8 @@ from app.downloads import folders, formats, service from app.downloads import gc as gcmod from app.db import SessionLocal from app.downloads.status import JobStatus -from app.models import DownloadJob, FileFolder, MediaAsset, Notification +from app.models import Notification +from app.downloads.models import DownloadJob, FileFolder, MediaAsset NOW = datetime.now(timezone.utc) diff --git a/backend/tests/test_files_api.py b/backend/tests/test_files_api.py index 1f4f4395..543b8eb3 100644 --- a/backend/tests/test_files_api.py +++ b/backend/tests/test_files_api.py @@ -18,7 +18,8 @@ from app.config import settings from app.downloads import folders, service from app.downloads import upload as uploadmod from app.downloads.status import JobStatus -from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset +from app.models import AuditLog +from app.downloads.models import DownloadJob, DownloadShare, FileFolder, MediaAsset NOW = datetime.now(timezone.utc) diff --git a/backend/tests/test_files_preview.py b/backend/tests/test_files_preview.py index 904ede98..0002314a 100644 --- a/backend/tests/test_files_preview.py +++ b/backend/tests/test_files_preview.py @@ -11,7 +11,7 @@ from sqlalchemy import select from app.config import settings from app.downloads import folders, preview, service from app.downloads.status import JobStatus -from app.models import DownloadJob, DownloadShare, MediaAsset +from app.downloads.models import DownloadJob, DownloadShare, MediaAsset from app.routes.files import TRUNCATED_HEADER diff --git a/backend/tests/test_folder_links.py b/backend/tests/test_folder_links.py index 2bbcf4a5..b8fb98a8 100644 --- a/backend/tests/test_folder_links.py +++ b/backend/tests/test_folder_links.py @@ -18,7 +18,8 @@ from app.config import settings from app.db import engine from app.downloads import folders, og from app.downloads import probe as probemod -from app.models import AuditLog, DownloadJob, DownloadLink, FileFolder, MediaAsset +from app.models import AuditLog +from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset _VIDEO = { "streams": [ @@ -317,7 +318,7 @@ def test_a_password_file_link_says_it_opens_a_file(shelf, client): def test_a_file_whose_bytes_are_gone_is_not_offered(shelf, client, db): from pathlib import Path - from app.models import MediaAsset + from app.downloads.models import MediaAsset token = _link(shelf)["token"] gone = shelf["inside"][0] diff --git a/backend/tests/test_library_progress.py b/backend/tests/test_library_progress.py index 3c033f05..a2ee9b1e 100644 --- a/backend/tests/test_library_progress.py +++ b/backend/tests/test_library_progress.py @@ -9,7 +9,7 @@ cannot reach, is ever read or written. DB-backed (real Postgres). import pytest from sqlalchemy import select -from app.models import DownloadJob, DownloadShare, LibraryState, MediaAsset +from app.downloads.models import DownloadJob, DownloadShare, LibraryState, MediaAsset from app.routes._common import ProgressIn, kept_position diff --git a/backend/tests/test_link_token_at_rest.py b/backend/tests/test_link_token_at_rest.py index d57bfb0b..ceab6dd6 100644 --- a/backend/tests/test_link_token_at_rest.py +++ b/backend/tests/test_link_token_at_rest.py @@ -18,7 +18,8 @@ from app import security from app.config import settings from app.db import engine from app.downloads import probe as probemod -from app.models import AuditLog, DownloadLink +from app.models import AuditLog +from app.downloads.models import DownloadLink _VIDEO = { "streams": [ diff --git a/backend/tests/test_not_found_detail.py b/backend/tests/test_not_found_detail.py index 0351d71e..7fbb5e50 100644 --- a/backend/tests/test_not_found_detail.py +++ b/backend/tests/test_not_found_detail.py @@ -7,7 +7,7 @@ the shared one, so every other owner check kept the English. Needs the DB. """ from app.auth import require_write_scope -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user): diff --git a/backend/tests/test_odd_frame.py b/backend/tests/test_odd_frame.py index 78b1cffe..f1f9b8e5 100644 --- a/backend/tests/test_odd_frame.py +++ b/backend/tests/test_odd_frame.py @@ -17,7 +17,7 @@ import pytest from app.config import settings from app.downloads import pipeline, playable, probe, service, storage from app.downloads import upload as uploadmod -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") diff --git a/backend/tests/test_phone_safe_contract.py b/backend/tests/test_phone_safe_contract.py index 1a0f659c..3ee4b469 100644 --- a/backend/tests/test_phone_safe_contract.py +++ b/backend/tests/test_phone_safe_contract.py @@ -17,7 +17,7 @@ import pytest from app.config import settings from app.downloads import pipeline, playable, probe -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset APP = Path(__file__).resolve().parents[1] / "app" diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py index f933560d..096c46c9 100644 --- a/backend/tests/test_unit_fences.py +++ b/backend/tests/test_unit_fences.py @@ -17,7 +17,8 @@ still not reach into a unit's tables. import ast from pathlib import Path -from app.models import Base +import app.tables # noqa: F401 — every unit's tables, for the ownership check +from app.db import Base APP = Path(__file__).parent.parent / "app" ROUTES = APP / "routes" diff --git a/backend/tests/test_upload.py b/backend/tests/test_upload.py index 90f11046..0b385594 100644 --- a/backend/tests/test_upload.py +++ b/backend/tests/test_upload.py @@ -33,7 +33,8 @@ from app.downloads import mediakind, pipeline, playable, quota, service, storage from app.downloads import probe as probemod from app.downloads import upload as uploadmod from app.downloads.stated import StatedFailure -from app.models import DISPLAY_TEXT_MAX, DownloadJob, DownloadQuota, MediaAsset +from app.models import DISPLAY_TEXT_MAX +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset # --- pure: the kind invariant --------------------------------------------------------------- diff --git a/backend/tests/test_upload_conversion.py b/backend/tests/test_upload_conversion.py index aeb850b9..c7ec930f 100644 --- a/backend/tests/test_upload_conversion.py +++ b/backend/tests/test_upload_conversion.py @@ -21,7 +21,8 @@ from app.downloads import pipeline, quota, storage from app.downloads import probe as probemod from app.downloads import upload as uploadmod from app.downloads.stated import StatedFailure -from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification +from app.models import Notification +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset from sqlalchemy import select, update from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session diff --git a/backend/tests/test_worker_dispatch.py b/backend/tests/test_worker_dispatch.py index 2593a306..3b726fba 100644 --- a/backend/tests/test_worker_dispatch.py +++ b/backend/tests/test_worker_dispatch.py @@ -12,7 +12,7 @@ from app.config import settings from app.downloads import pipeline, runner, service, storage from app.downloads import probe as probemod from app.downloads import upload as uploadmod -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset @pytest.mark.parametrize( diff --git a/frontend/src/lib/api/downloads.contract.test.ts b/frontend/src/lib/api/downloads.contract.test.ts index 8110bcdb..369a5e8c 100644 --- a/frontend/src/lib/api/downloads.contract.test.ts +++ b/frontend/src/lib/api/downloads.contract.test.ts @@ -29,8 +29,10 @@ describe("display-name bound", () => { it("is the width the columns are actually declared with", () => { // The mutation guard for the check above: both constants could agree on a number neither // column uses. `String(DISPLAY_TEXT_MAX)` is how the three display columns are declared, so a - // literal creeping back into one of them fails here. - const models = src("../../../../backend/app/models.py"); + // literal creeping back into one of them fails here. `User.display_name` is a core table; the + // job's two are Capture's. + const models = + src("../../../../backend/app/models.py") + src("../../../../backend/app/downloads/models.py"); const declared = [...models.matchAll(/display_(?:name|uploader): Mapped.*?String\((.*?)\)/g)]; expect(declared).toHaveLength(3); for (const m of declared) expect(m[1]).toBe("DISPLAY_TEXT_MAX"); From e9c60f72cd734446088be34f6425f55e69748e8d Mon Sep 17 00:00:00 2001 From: npeter83 Date: Mon, 19 Oct 2026 00:19:03 +0200 Subject: [PATCH 03/25] refactor(capture): the HTTP layer reads through queries and writes through the service Capture's routes (downloads, files, public watch links) and the token list no longer build SQL, call the session or set a model's fields: reads are downloads/queries, writes the unit's service (service, folders, links, quota), and each write flushes for the handler to commit. Whose a row is goes through userscope (owned_by, owned_row), which owned_or_404 now uses too. The fence holds Capture below its routes as well: it reads only Catalog's and Plex's tables, writes no other unit's, filters owners through userscope and commits only in its jobs and the upload ingest. A test per service use case, among them that the tray lists only the caller's files. --- backend/.basedpyright/baseline.json | 42 -- backend/app/downloads/credentials.py | 5 +- backend/app/downloads/folders.py | 32 +- backend/app/downloads/links.py | 128 +++++- backend/app/downloads/queries.py | 406 ++++++++++++++++++ backend/app/downloads/quota.py | 48 ++- backend/app/downloads/service.py | 220 +++++++++- backend/app/routes/_common.py | 5 +- backend/app/routes/downloads.py | 440 +++++--------------- backend/app/routes/files.py | 331 ++++----------- backend/app/routes/public.py | 60 ++- backend/app/routes/tokens.py | 19 +- backend/app/userscope.py | 78 +++- backend/tests/test_capture_service.py | 295 +++++++++++++ backend/tests/test_download_job_locks.py | 1 + backend/tests/test_download_limits.py | 3 +- backend/tests/test_folder_links.py | 2 +- backend/tests/test_unit_fences.py | 212 +++++++++- docs/gates.md | 2 +- frontend/src/lib/api/files.contract.test.ts | 9 +- 20 files changed, 1632 insertions(+), 706 deletions(-) create mode 100644 backend/app/downloads/queries.py create mode 100644 backend/tests/test_capture_service.py diff --git a/backend/.basedpyright/baseline.json b/backend/.basedpyright/baseline.json index af17ec70..6ef8921d 100644 --- a/backend/.basedpyright/baseline.json +++ b/backend/.basedpyright/baseline.json @@ -692,48 +692,6 @@ } } ], - "./app/routes/downloads.py": [ - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 17, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 82, - "endColumn": 84, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 87, - "endColumn": 89, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 30, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 49, - "endColumn": 54, - "lineCount": 1 - } - } - ], "./app/routes/playlists.py": [ { "code": "reportOptionalOperand", diff --git a/backend/app/downloads/credentials.py b/backend/app/downloads/credentials.py index 026634e2..587af04f 100644 --- a/backend/app/downloads/credentials.py +++ b/backend/app/downloads/credentials.py @@ -35,6 +35,7 @@ from app.downloads import formats from app.downloads.adapters import _host_is from app.downloads.models import DownloadCredential, MediaAsset from app.security import decrypt_optional, encrypt +from app.userscope import owned_by log = logs.get_logger(__name__) @@ -157,7 +158,7 @@ def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None: def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None: return db.execute( select(DownloadCredential).where( - DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key + owned_by(DownloadCredential.user_id, user_id), DownloadCredential.host_key == host_key ) ).scalar_one_or_none() @@ -214,7 +215,7 @@ def status_list(db: Session, user_id: int) -> list[dict]: have = { c.host_key: c for c in db.execute( - select(DownloadCredential).where(DownloadCredential.user_id == user_id) + select(DownloadCredential).where(owned_by(DownloadCredential.user_id, user_id)) ).scalars() } out = [] diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py index a431a6bc..f829375b 100644 --- a/backend/app/downloads/folders.py +++ b/backend/app/downloads/folders.py @@ -33,6 +33,7 @@ from sqlalchemy.orm import Session from app import sysconfig from app.downloads.status import JobStatus from app.downloads.models import DownloadJob, FileFolder, MediaAsset +from app.userscope import owned_by class FolderKind(StrEnum): @@ -106,7 +107,9 @@ def source_key_for(db: Session, job: DownloadJob) -> str: def source_folder(db: Session, user_id: int, key: str) -> FileFolder: """The user's source folder for `key`, made on first use. A race on the unique (user, key) is settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive.""" - stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key) + stmt = select(FileFolder).where( + owned_by(FileFolder.user_id, user_id), FileFolder.source_key == key + ) folder = db.execute(stmt).scalar_one_or_none() if folder is not None: return folder @@ -131,7 +134,7 @@ def hub_folder(db: Session, user_id: int) -> FileFolder: """The user's Hub folder, made on first use; a race on its one-per-user index is settled like `source_folder`'s.""" stmt = select(FileFolder).where( - FileFolder.user_id == user_id, FileFolder.kind == FolderKind.HUB + owned_by(FileFolder.user_id, user_id), FileFolder.kind == FolderKind.HUB ) folder = db.execute(stmt).scalar_one_or_none() if folder is not None: @@ -310,3 +313,28 @@ def delete_own(db: Session, folder: FileFolder) -> int: db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete db.delete(folder) return len(jobs) + + +def create_own(db: Session, user_id: int, name: str) -> FileFolder: + """A new folder of the user's own; FolderError("file_folder_name_taken") when they already have + one by that name (the unique (user, lower(name))).""" + folder = FileFolder(user_id=user_id, kind=FolderKind.OWN, name=name) + try: + with db.begin_nested(): + db.add(folder) + db.flush() + except IntegrityError as e: + raise FolderError("file_folder_name_taken") from e + return folder + + +def rename(db: Session, folder: FileFolder, name: str) -> None: + """Rename one of the user's own folders (a source folder's name is its origin's).""" + if folder.kind != FolderKind.OWN: + raise FolderError("file_folder_not_own") + folder.name = name + try: + with db.begin_nested(): + db.flush() + except IntegrityError as e: + raise FolderError("file_folder_name_taken") from e diff --git a/backend/app/downloads/links.py b/backend/app/downloads/links.py index 5d46335a..54546761 100644 --- a/backend/app/downloads/links.py +++ b/backend/app/downloads/links.py @@ -15,20 +15,24 @@ grant appended to the file URL, so the raw password never rides in the media URL import hashlib import hmac import secrets -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from sqlalchemy import select +from sqlalchemy import update as sql_update from sqlalchemy.orm import Session -from app import security +from app import audit, security from app.config import settings from app.downloads import storage from app.downloads.status import JobStatus from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset +from app.models import User +from app.utils import iso +from app.userscope import owned_by _GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks -# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files. -# MAX_LIST`; a test holds the two equal — this layer may not import that one). +# The most files a folder link lists: as many as the owner's own Files list shows +# (`queries.MAX_LIST`; a test holds the two equal — `queries` imports this module). FOLDER_MAX = 500 @@ -59,7 +63,7 @@ def in_folder(folder: FileFolder) -> tuple: moved out stops opening through the link at its next request; one moved in starts.""" return ( DownloadJob.folder_id == folder.id, - DownloadJob.user_id == folder.user_id, + owned_by(DownloadJob.user_id, folder.user_id), DownloadJob.status == JobStatus.DONE, ) @@ -170,3 +174,117 @@ def public_meta( "file_url": file_url, "poster_url": poster_url, } + + +# --- the owner's writes -------------------------------------------------------------------- +# Each flushes and records its audit row; the handler commits. + + +def expiry(days: int | None) -> datetime | None: + """None/0/negative → no expiry; otherwise N days out, capped at ~10 years.""" + if not days or days <= 0: + return None + return datetime.now(timezone.utc) + timedelta(days=min(days, 3650)) + + +def audit_view(link: DownloadLink) -> dict: + """A link's settings as the audit records them: whether it has a password, never which.""" + return { + "allow_download": link.allow_download, + "has_password": link.password_hash is not None, + "expires_at": iso(link.expires_at), + } + + +def create( + db: Session, + user: User, + *, + password: str, + allow_download: bool, + expires_days: int | None, + **target: int, +) -> tuple[DownloadLink, str]: + """A link to `target` (`job_id=` or `folder_id=`, already checked to be the user's), and the + token it is opened by — which is in hand only now.""" + token = new_token() + link = DownloadLink( + **target, + created_by=user.id, + allow_download=allow_download, + password_hash=security.hash_password(password) if password else None, + expires_at=expiry(expires_days), + ) + seal(link, token) + db.add(link) + db.flush() + audit.record( + db, + user.id, + audit.AuditAction.DOWNLOAD_LINK_CREATE, + target_type="download_link", + target_id=link.id, + after={**target, **audit_view(link)}, + ) + return link, token + + +def update( + db: Session, + user: User, + link: DownloadLink, + *, + allow_download: bool | None = None, + expires_days: int | None = None, + password: str | None = None, + sent: frozenset[str] = frozenset(), +) -> None: + """Change what the caller sent (`sent` names the fields, so `expires_days=None` can mean "never + expires"). Setting, clearing or changing the password bumps its version, so any grant issued + under the previous one stops working now rather than living out its TTL.""" + before = audit_view(link) + if "allow_download" in sent: + link.allow_download = bool(allow_download) + if "expires_days" in sent: + link.expires_at = expiry(expires_days) + if "password" in sent: + pw = (password or "").strip() + link.password_hash = security.hash_password(pw) if pw else None + link.password_version += 1 + after = audit_view(link) + if after != before or "password" in sent: + audit.record( + db, + user.id, + audit.AuditAction.DOWNLOAD_LINK_UPDATE, + target_type="download_link", + target_id=link.id, + before=before, + after=after, + ) + db.flush() + + +def revoke(db: Session, user: User, link: DownloadLink) -> None: + link_id = link.id + db.delete(link) + audit.record( + db, + user.id, + audit.AuditAction.DOWNLOAD_LINK_REVOKE, + target_type="download_link", + target_id=link_id, + after={"job_id": link.job_id} if link.job_id is not None else {"folder_id": link.folder_id}, + ) + db.flush() + + +def count_view(db: Session, link: DownloadLink) -> None: + """One more opening of the link. In SQL, not `+= 1` on the loaded row: two opens at once would + both write the same count.""" + db.execute( + sql_update(DownloadLink) + .where(DownloadLink.id == link.id) + .values(view_count=DownloadLink.view_count + 1) + .execution_options(synchronize_session=False) + ) diff --git a/backend/app/downloads/queries.py b/backend/app/downloads/queries.py new file mode 100644 index 00000000..edab65c0 --- /dev/null +++ b/backend/app/downloads/queries.py @@ -0,0 +1,406 @@ +"""Capture's reads: what the library, the download tray, the share dialogs and the public watch pages +show. Each returns rows or plain values and writes nothing; whose a row is goes through +`userscope.owned_by` / `owned_row`, so a row of someone else's reads as a missing one. + +The HTTP layer (`routes.downloads`, `routes.files`, `routes.public`) asks here and serializes; the +writes are the service's (`service`, `folders`, `links`, `quota`).""" + +from collections.abc import Sequence +from datetime import datetime, timezone + +from sqlalchemy import func, or_, select +from sqlalchemy.orm import Session + +from app.apitokens import FilesCaller, Right +from app.collation import collated +from app.downloads import folders, links, quota +from app.downloads.models import ( + DownloadJob, + DownloadLink, + DownloadProfile, + DownloadQuota, + DownloadShare, + FileFolder, + LibraryState, + MediaAsset, +) +from app.downloads.status import FINISHED, JobStatus +from app.models import User +from app.userscope import owned_by, owned_row +from app.utils import like_escape + +# Cap a listing so a heavy library can't return an unbounded result set (the old Download Center's +# own cap; far beyond any real library). +MAX_LIST = 500 + + +# --- jobs and their files ------------------------------------------------------------------ + + +def own_job(db: Session, user: User, job_id: int) -> DownloadJob | None: + return owned_row(db, DownloadJob, job_id, user) + + +def job(db: Session, job_id: int) -> DownloadJob | None: + """A job by id, whoever's — for a public link, which already names its job.""" + return db.get(DownloadJob, job_id) + + +def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob | None: + """A file the caller may read: their own, or one shared with them.""" + row = db.get(DownloadJob, job_id) + if row is None: + return None + if row.user_id == user.id or shared_with(db, job_id, user) is not None: + return row + return None + + +def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob | None: + """A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's + own in a folder it may read.""" + if caller.grants is None: + return accessible_job(db, caller.user, job_id) + row = own_job(db, caller.user, job_id) + if row is None or not caller.may(row.folder_id, Right.READ): + return None + return row + + +def asset_of(db: Session, job: DownloadJob | None) -> MediaAsset | None: + """The file under a job, if it has one yet.""" + return db.get(MediaAsset, job.asset_id) if job is not None and job.asset_id else None + + +def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]: + ids = {j.asset_id for j in jobs if j.asset_id} + if not ids: + return {} + rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars() + return {a.id: a for a in rows} + + +def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]: + """`user`'s resume point in each of `jobs` that has one (the library player's `library_states`).""" + ids = [j.id for j in jobs] + if not ids: + return {} + rows = db.execute( + select(LibraryState.job_id, LibraryState.position_seconds).where( + owned_by(LibraryState.user_id, user), LibraryState.job_id.in_(ids) + ) + ).all() + return {job_id: pos for job_id, pos in rows} + + +def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]: + """Where each file came from (its source folder's key), for the list's Source column, the + "back to its source folder" move and the download tray's site: a file in a source folder has its + folder's key; one moved into an own folder (or someone else's, shared) is traced back by + `folders.source_key_for`.""" + ids = {j.folder_id for j in jobs if j.folder_id is not None} + by_id = ( + {f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()} + if ids + else {} + ) + out: dict[int, str] = {} + for j in jobs: + folder = by_id.get(j.folder_id) if j.folder_id is not None else None + if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key: + out[j.id] = folder.source_key + else: + out[j.id] = folders.source_key_for(db, j) + return out + + +def queue(db: Session, user: User, recent: datetime) -> list[DownloadJob]: + """The download tray: what is waiting, running, paused or failed, and what finished (done or + canceled) since `recent`. A done job is timed from when it became done, never from its last + write: a rename, a move or the GC's expiry warning touches an old file's row, and the tray would + announce it as just finished.""" + return list( + db.execute( + select(DownloadJob) + .where( + owned_by(DownloadJob.user_id, user), + DownloadJob.status.not_in(FINISHED) + | ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent)) + | ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)), + ) + .order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()) + .limit(MAX_LIST) + ).scalars() + ) + + +def statuses_by_source(db: Session, user: User, statuses: Sequence[str]) -> list[tuple[str, str]]: + """(source_ref, status) of each of `user`'s jobs in one of `statuses`.""" + return list( + db.execute( + select(DownloadJob.source_ref, DownloadJob.status).where( + owned_by(DownloadJob.user_id, user), DownloadJob.status.in_(tuple(statuses)) + ) + ).tuples() + ) + + +def library( + db: Session, + user: User, + *, + folder_id: int | None, + listable: set[int] | None, + shared: bool, + term: str, + sort: str, + order: str, + lang: str, +) -> list[DownloadJob]: + """The caller's done files (optionally one folder's, or only in the folders a token may list), + or the ones shared with them; searched by name and channel, sorted by when they were added, by + name (in the reader's alphabet) or by size. A file is added to its owner's list when it is done, + not when it was asked for (a download that failed for days and then went through is new); one + shared with the caller, when it was shared.""" + stmt = ( + select(DownloadJob) + .join(MediaAsset, MediaAsset.id == DownloadJob.asset_id) + .where(DownloadJob.status == JobStatus.DONE) + ) + if shared: + stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where( + owned_by(DownloadShare.to_user_id, user) + ) + else: + stmt = stmt.where(owned_by(DownloadJob.user_id, user)) + if folder_id is not None: + stmt = stmt.where(DownloadJob.folder_id == folder_id) + elif listable is not None: + stmt = stmt.where(DownloadJob.folder_id.in_(listable)) + name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "") + if term: + like = f"%{like_escape(term)}%" + stmt = stmt.where( + or_( + name.ilike(like), + func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like), + ) + ) + key = { + "added": DownloadShare.created_at if shared else DownloadJob.ready_at, + "name": collated(name, lang), + "size": MediaAsset.size_bytes, + }[sort] + first = key.asc() if order == "asc" else key.desc() + tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc() + return list(db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars()) + + +def recent_jobs(db: Session) -> list[DownloadJob]: + """Everyone's newest jobs, for the admin's list.""" + return list( + db.execute( + select(DownloadJob) + .order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()) + .limit(MAX_LIST) + ).scalars() + ) + + +# --- shares -------------------------------------------------------------------------------- + + +def shared_with(db: Session, job_id: int, user: User | int) -> DownloadShare | None: + """The grant that hands this job to `user`, if there is one.""" + return db.execute( + select(DownloadShare).where( + DownloadShare.job_id == job_id, owned_by(DownloadShare.to_user_id, user) + ) + ).scalar_one_or_none() + + +def shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]: + rows = db.execute( + select(DownloadShare.job_id, DownloadShare.created_at).where( + owned_by(DownloadShare.to_user_id, user), + DownloadShare.job_id.in_([j.id for j in jobs]), + ) + ).all() + return {job_id: at for job_id, at in rows} + + +# --- folders ------------------------------------------------------------------------------- + + +def own_folder(db: Session, user: User, folder_id: int) -> FileFolder | None: + return owned_row(db, FileFolder, folder_id, user) + + +def folders_by_id(db: Session, ids: set[int]) -> dict[int, FileFolder]: + """These folders, by id — for a reader that already knows which ones (a token's grants).""" + if not ids: + return {} + return {f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()} + + +def folders_with_totals( + db: Session, user: User, listable: set[int] | None +) -> list[tuple[FileFolder, int, int]]: + """The user's folders (only `listable` ones, when given) with how many done files each holds and + their size. A source folder its files have all left stays (count 0).""" + done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE) + stmt = select( + FileFolder, + func.count(DownloadJob.id), + func.coalesce(func.sum(MediaAsset.size_bytes), 0), + ).where(owned_by(FileFolder.user_id, user)) + if listable is not None: + stmt = stmt.where(FileFolder.id.in_(listable)) + rows = db.execute( + stmt.outerjoin(DownloadJob, done) + .outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id) + .group_by(FileFolder.id) + .order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id) + ).tuples() + return [(f, count, int(size or 0)) for f, count, size in rows] + + +def folder_totals(db: Session, folder_id: int) -> tuple[int, int]: + count, size = db.execute( + select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0)) + .select_from(DownloadJob) + .outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id) + .where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE) + ).one() + return count, int(size) + + +def linked_folders(db: Session, folder_ids: list[int]) -> set[int]: + """Which of these folders a live public link opens (one not past its expiry).""" + now = datetime.now(timezone.utc) + rows = db.execute( + select(DownloadLink.folder_id).where( + DownloadLink.folder_id.in_(folder_ids), + or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now), + ) + ).scalars() + return {folder_id for folder_id in rows if folder_id is not None} + + +# --- links --------------------------------------------------------------------------------- + + +def own_link(db: Session, user: User, link_id: int) -> DownloadLink | None: + """A link to one of the user's files or folders.""" + link = db.get(DownloadLink, link_id) + if link is None: + return None + target = ( + db.get(DownloadJob, link.job_id) + if link.job_id is not None + else db.get(FileFolder, link.folder_id) + ) + if target is None or target.user_id != user.id: + return None + return link + + +def links_to( + db: Session, *, job_id: int | None = None, folder_id: int | None = None +) -> list[DownloadLink]: + """The links to one file or one folder, newest first.""" + target = ( + DownloadLink.job_id == job_id if job_id is not None else DownloadLink.folder_id == folder_id + ) + return list( + db.execute(select(DownloadLink).where(target).order_by(DownloadLink.id.desc())).scalars() + ) + + +def link_folder(db: Session, link: DownloadLink) -> FileFolder | None: + return db.get(FileFolder, link.folder_id) if link.folder_id is not None else None + + +def folder_job(db: Session, folder: FileFolder, job_id: int) -> DownloadJob | None: + """One of the files a link to `folder` reaches (`links.in_folder`), while it is in it.""" + return db.execute( + select(DownloadJob).where(DownloadJob.id == job_id, *links.in_folder(folder)) + ).scalar_one_or_none() + + +def owner_suspended(db: Session, job: DownloadJob) -> bool: + """The job's owner is suspended (or gone): the worker leaves their queued jobs where they are.""" + owner = db.get(User, job.user_id) + return owner is None or owner.is_suspended + + +# --- profiles ------------------------------------------------------------------------------ + + +def profiles_for(db: Session, user: User) -> list[DownloadProfile]: + """The built-in format profiles, then the user's own.""" + return list( + db.execute( + select(DownloadProfile) + .where(DownloadProfile.is_builtin.is_(True) | owned_by(DownloadProfile.user_id, user)) + .order_by( + DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id + ) + ).scalars() + ) + + +def own_profile(db: Session, user: User, profile_id: int) -> DownloadProfile | None: + """One of the user's own profiles (a built-in is no one's to change).""" + p = owned_row(db, DownloadProfile, profile_id, user) + return None if p is None or p.is_builtin else p + + +def usable_profile(db: Session, user: User, profile_id: int) -> DownloadProfile | None: + """A profile the user may enqueue with: a built-in, or their own.""" + p = db.get(DownloadProfile, profile_id) + if p is None or (not p.is_builtin and p.user_id != user.id): + return None + return p + + +def default_profile(db: Session) -> DownloadProfile | None: + """The first built-in, for an enqueue that named no format.""" + return db.execute( + select(DownloadProfile) + .where(DownloadProfile.is_builtin.is_(True)) + .order_by(DownloadProfile.sort_order) + .limit(1) + ).scalar_one_or_none() + + +# --- storage and quota (admin) ------------------------------------------------------------- + + +def storage_totals(db: Session) -> tuple[int, int, int]: + """(files on disk, their bytes, every asset row). On disk is `quota.ON_DISK`: ready files, and + uploads waiting for or failed in their conversion — the same disk the footprint and the GC cap + count.""" + on_disk = db.execute( + select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK) + ).one() + total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar_one() + return on_disk[0], int(on_disk[1]), total_assets + + +def done_files_per_user(db: Session) -> dict[int, int]: + """How many finished files each user's Files lists.""" + return dict( + db.execute( + select(DownloadJob.user_id, func.count()) + .where(DownloadJob.status == JobStatus.DONE) + .group_by(DownloadJob.user_id) + ) + .tuples() + .all() + ) + + +def custom_quota(db: Session, user_id: int) -> DownloadQuota | None: + """The admin's own limits for this user, if any were set.""" + return db.get(DownloadQuota, user_id) diff --git a/backend/app/downloads/quota.py b/backend/app/downloads/quota.py index ef58665a..aca55ae0 100644 --- a/backend/app/downloads/quota.py +++ b/backend/app/downloads/quota.py @@ -27,6 +27,7 @@ from sqlalchemy.orm import Session from app import sysconfig from app.downloads.status import JobKind, JobStatus from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset +from app.userscope import owned_by # Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures # (error, canceled) don't count. @@ -85,7 +86,7 @@ def footprint(db: Session, user_id: int) -> int: held_assets = ( select(DownloadJob.asset_id) .where( - DownloadJob.user_id == user_id, + owned_by(DownloadJob.user_id, user_id), or_( DownloadJob.status.in_(_HOLDING), and_(DownloadJob.job_kind == JobKind.UPLOAD, DownloadJob.status == JobStatus.ERROR), @@ -109,7 +110,7 @@ def _count(db: Session, user_id: int, states) -> int: db.execute( select(func.count()) .select_from(DownloadJob) - .where(DownloadJob.user_id == user_id, DownloadJob.status.in_(states)) + .where(owned_by(DownloadJob.user_id, user_id), DownloadJob.status.in_(states)) ).scalar() or 0 ) @@ -201,3 +202,46 @@ def usage(db: Session, user_id: int) -> dict: "max_jobs": lim.max_jobs, "unlimited": lim.unlimited, } + + +def set_custom( + db: Session, + user_id: int, + *, + max_bytes: int | None = None, + max_concurrent: int | None = None, + max_jobs: int | None = None, + unlimited: bool | None = None, +) -> None: + """The admin's own limits for a user: starts from what applies now, then each value given (a + size floored at 0, a count at 1). The handler commits.""" + row = db.get(DownloadQuota, user_id) + if row is None: + cur = resolve(db, user_id) + row = DownloadQuota( + user_id=user_id, + max_bytes=cur.max_bytes, + max_concurrent=cur.max_concurrent, + max_jobs=cur.max_jobs, + unlimited=cur.unlimited, + ) + db.add(row) + if max_bytes is not None: + row.max_bytes = max(0, max_bytes) + if max_concurrent is not None: + row.max_concurrent = max(1, max_concurrent) + if max_jobs is not None: + row.max_jobs = max(1, max_jobs) + if unlimited is not None: + row.unlimited = unlimited + db.flush() + + +def reset_custom(db: Session, user_id: int) -> bool: + """Back to the defaults; False when the user had no limits of their own.""" + row = db.get(DownloadQuota, user_id) + if row is None: + return False + db.delete(row) + db.flush() + return True diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index 54169d71..f1b85e0e 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -1,16 +1,19 @@ -"""Enqueue + cache resolution for the download center (the per-user layer over MediaAsset). +"""Capture's writes: enqueue + cache resolution, the job transitions, and what the library's +dialogs change (profiles, resume points, shares, a file's display fields). `enqueue` is the single entry point used by the API: it resolves (or creates) the shared MediaAsset for the requested source+format, links a per-user DownloadJob to it, and — on a cache hit (asset already ready) — marks the job done instantly with no re-download. The worker -handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and -the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping. +handles everything else. Every function here flushes and leaves the commit to its caller (the +handler); the folder, link and quota writes are `folders`, `links` and `quota`. """ import uuid from datetime import datetime, timezone +from sqlalchemy import delete as delete_rows from sqlalchemy import func, select, update +from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -21,10 +24,17 @@ from app.downloads import credentials from app.downloads import edit as editmod from app.downloads import folders, formats, mediakind, quota, storage from app.downloads.status import HOLDING, JobKind, JobStatus -from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, Video -from app.downloads.models import DownloadJob, MediaAsset +from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, User, Video +from app.downloads.models import ( + DownloadJob, + DownloadProfile, + DownloadShare, + LibraryState, + MediaAsset, +) from app.plex import lookups as plex_lookups from app.plex import paths as plex_paths +from app.userscope import owned_by, owned_row class EditError(Exception): @@ -223,8 +233,7 @@ def enqueue( folders.stamp_done(db, job) asset.last_access_at = datetime.now(timezone.utc) - db.commit() - db.refresh(job) + db.flush() return job @@ -303,8 +312,7 @@ def enqueue_edit( folders.stamp_done(db, job) asset.last_access_at = datetime.now(timezone.utc) - db.commit() - db.refresh(job) + db.flush() return job @@ -392,8 +400,7 @@ def enqueue_clip( folders.stamp_done(db, job) asset.last_access_at = datetime.now(timezone.utc) - db.commit() - db.refresh(job) + db.flush() return job @@ -501,8 +508,8 @@ def release_asset(db: Session, job: DownloadJob) -> None: # --- Job state transitions --------------------------------------------------------------------- # The single home for the pause/resume/cancel/delete/redownload transitions + their asset # bookkeeping (release, failed-asset reset, fresh-asset rebuild), so the routes are thin callers and -# the worker/GC could share the same definitions (B-sub-8). Each commits its own transition; the -# route keeps only auth + HTTP mapping + serialization. +# the worker/GC could share the same definitions (B-sub-8). Each flushes its transition and the caller +# commits it; the route keeps only auth + HTTP mapping + serialization. def pause(db: Session, job: DownloadJob) -> None: @@ -510,7 +517,7 @@ def pause(db: Session, job: DownloadJob) -> None: in any other state.""" if job.status in (JobStatus.QUEUED, JobStatus.RUNNING): job.status = JobStatus.PAUSED - db.commit() + db.flush() def resume(db: Session, job: DownloadJob) -> None: @@ -534,7 +541,7 @@ def resume(db: Session, job: DownloadJob) -> None: asset.status = "pending" asset.error = None job.status = JobStatus.QUEUED - db.commit() + db.flush() def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset: @@ -569,14 +576,14 @@ def cancel(db: Session, job: DownloadJob) -> None: if job.status not in (JobStatus.DONE, JobStatus.CANCELED): release_asset(db, job) # release while the job still counts as holding job.status = JobStatus.CANCELED - db.commit() + db.flush() def delete(db: Session, job: DownloadJob) -> None: """Delete a job, releasing its asset hold first (which frees the file iff this was the last holder).""" release_asset(db, job) db.delete(job) - db.commit() + db.flush() def redownload(db: Session, job: DownloadJob) -> None: @@ -625,4 +632,181 @@ def redownload(db: Session, job: DownloadJob) -> None: job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot job.expires_at = None # stamped again when the rebuild is done job.expiry_warned = False - db.commit() + db.flush() + + +# --- the library's writes ------------------------------------------------------------------ +# What the HTTP layer changes for a user: each flushes, the handler commits. + + +def lock_own_job(db: Session, user: User, job_id: int) -> DownloadJob | None: + """One of the user's jobs, re-read under a row lock, for a transition that moves its asset's ref + count (resume, cancel, delete, redownload) or writes the status those decide on (pause). Two + overlapping requests for one job run one after the other and the second decides on the first's + committed result — never both releasing the same hold, and never a pause turning a job a cancel + just released back into a holder; either can free a file another job still holds. None: not + theirs, or gone. + + The asset is locked BEFORE the job: the order a finishing worker child takes them in + (`finalize_if_running`), so the two can never deadlock.""" + while True: + owned = owned_row(db, DownloadJob, job_id, user) + if owned is None: + return None + asset_id = owned.asset_id + if asset_id is not None: + db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update()) + job = db.execute( + select(DownloadJob) + .where(DownloadJob.id == job_id) + .with_for_update() + .execution_options(populate_existing=True) + ).scalar_one_or_none() + if job is None: + return None + if job.asset_id == asset_id: + return job + db.rollback() # another request moved the job to another asset meanwhile: lock that one + + +def hold_for_share(db: Session, user: User, job_id: int) -> DownloadJob | None: + """One of the user's files, held (FOR SHARE) for the rest of the transaction: a delete of it at + the same moment (the user's, or the retention sweep's) waits and takes the new link with it, + rather than failing the link's foreign key — or, done first, leaves None.""" + if owned_row(db, DownloadJob, job_id, user) is None: + return None + return db.execute( + select(DownloadJob) + .where(DownloadJob.id == job_id) + .with_for_update(read=True) + .execution_options(populate_existing=True) + ).scalar_one_or_none() + + +def create_profile(db: Session, user: User, name: str, spec: dict) -> DownloadProfile: + """A format profile of the user's own, after their others.""" + maxpos = db.execute( + select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where( + owned_by(DownloadProfile.user_id, user) + ) + ).scalar_one() + p = DownloadProfile( + user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1 + ) + db.add(p) + db.flush() + return p + + +def update_profile(profile: DownloadProfile, *, name: str | None, spec: dict | None) -> None: + """Rename a profile and/or replace its spec; None leaves that one as it is.""" + if name is not None: + profile.name = name[:80] + if spec is not None: + profile.spec = spec + + +def delete_profile(db: Session, profile: DownloadProfile) -> None: + db.delete(profile) + db.flush() + + +def set_position(db: Session, user: User, job_id: int, position: int) -> None: + """The library player's resume point in a file the user can play; 0 clears it. + + A clear is a statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon + of one stop may both clear it, and the second then deletes nothing rather than failing. A set is + an upsert, one row per (user, job) even when two checkpoints race, not a read-then-insert that + the second would fail on the unique key.""" + if not position: + db.execute( + delete_rows(LibraryState).where( + owned_by(LibraryState.user_id, user), LibraryState.job_id == job_id + ) + ) + return + now = datetime.now(timezone.utc) + db.execute( + pg_insert(LibraryState) + .values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now) + .on_conflict_do_update( + constraint="uq_library_user_job", + set_={"position_seconds": position, "progress_updated_at": now}, + ) + ) + + +def share(db: Session, job: DownloadJob, sender: User, recipient: User) -> bool: + """Hand a file to another user, with a notice for them; False when it was already theirs.""" + from app.notifications import create_notification + + exists = db.execute( + select(DownloadShare.id).where( + DownloadShare.job_id == job.id, owned_by(DownloadShare.to_user_id, recipient) + ) + ).first() + if exists is not None: + return False + db.add(DownloadShare(job_id=job.id, from_user_id=sender.id, to_user_id=recipient.id)) + create_notification( + db, + user_id=recipient.id, + type="download_shared", + title=(job.display_name or "A download") + " was shared with you", + # The client names the notice in the reader's language from `data`; the title is the + # English fallback only. + data={ + "kind": "download_shared", + "from": sender.email, + "job_id": job.id, + "title": job.display_name, + }, + commit=False, + ) + db.flush() + return True + + +def unshare(db: Session, job_id: int, recipient: User) -> bool: + """Take a file back from one user (its owner's act, or the recipient's own dismissal); False + when it was not theirs.""" + row = db.execute( + select(DownloadShare).where( + DownloadShare.job_id == job_id, owned_by(DownloadShare.to_user_id, recipient) + ) + ).scalar_one_or_none() + if row is None: + return False + db.delete(row) + db.flush() + return True + + +# The display fields a user may set on their file; the file on disk keeps its own name. +DISPLAY_FIELDS = frozenset( + {"display_name", "display_uploader", "display_uploader_url", "extra_links"} +) + + +def set_display(job: DownloadJob, **fields: object) -> None: + """Set a file's display fields (each already cleaned by the caller); a field not given is left.""" + unknown = set(fields) - DISPLAY_FIELDS + if unknown: + raise ValueError(f"not a display field: {sorted(unknown)}") + for name, value in fields.items(): + setattr(job, name, value) + + +def touch(asset: MediaAsset) -> None: + """The file was just served: the cache's idle clock starts again.""" + asset.last_access_at = datetime.now(timezone.utc) + + +def store_sha256(db: Session, asset_id: int, digest: str) -> None: + """Keep a file's hash once it is known — only where none is, so a concurrent hash of the same + file never overwrites the first.""" + db.execute( + update(MediaAsset) + .where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None)) + .values(sha256=digest) + ) diff --git a/backend/app/routes/_common.py b/backend/app/routes/_common.py index 740f3f2d..567a2713 100644 --- a/backend/app/routes/_common.py +++ b/backend/app/routes/_common.py @@ -16,6 +16,7 @@ from app.db import Base from app.models import POSITION_MAX_S, User from app.issues import gitea from app.refusal import refused +from app.userscope import owned_row from app.youtube.client import YouTubeError, refusal_reason M = TypeVar("M", bound=Base) @@ -90,8 +91,8 @@ def owned_or_404( else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger. It replaced the near-identical per-route ownership helpers, and is called directly — a thin `_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out.""" - obj = get_or_404(db, model, id_, what=what, lock=lock) - if getattr(obj, owner_attr) != user.id: + obj = owned_row(db, model, id_, user, owner_attr=owner_attr, lock=lock) + if obj is None: raise not_found(what) return obj diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py index 31d68988..b872c0e7 100644 --- a/backend/app/routes/downloads.py +++ b/backend/app/routes/downloads.py @@ -3,23 +3,22 @@ All routes are behind `current_user`; admin routes add `admin_user`. Live job state is polled by the frontend (the worker is a separate process); this module never downloads — it enqueues, manages job state, and shares finished files. The library itself — listing, renaming, moving, deleting and -serving a done file, and uploading one — is `routes.files`. +serving a done file, and uploading one — is `routes.files`. Reads come from `downloads.queries`, +writes go through Capture's service (`service`, `links`, `quota`); this module validates, answers in +HTTP and commits. """ import re -from typing import Literal +from typing import TYPE_CHECKING, Literal from datetime import datetime, timedelta, timezone from urllib.parse import parse_qs, urlparse from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import FileResponse from pydantic import BaseModel, Field -from sqlalchemy import delete, func, select -from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session from app.mailtext import lang_for_user -from app.collation import collated from app import audit, sysconfig from app.auth import admin_user, current_user from app.config import settings @@ -29,44 +28,32 @@ from app.downloads import credentials as dlcreds from app.downloads import edit as editmod from app.downloads import folders from app.downloads import links as linksmod -from app.downloads import quota, service, storage +from app.downloads import queries, quota, service, storage from app.downloads import urlguard from app.downloads.folders import FolderKind -from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus +from app.downloads.status import NOT_REFETCHABLE, JobStatus from app.models import User, display_text -from app.downloads.models import ( - DownloadJob, - DownloadLink, - FileFolder, - DownloadProfile, - DownloadQuota, - DownloadShare, - LibraryState, - MediaAsset, -) from app.routes._common import ( ProgressIn, get_or_404, kept_position, not_found, - owned_or_404, ) from app.routes.files import ( - MAX_LIST, accessible_job, - assets_for, + own_folder, + own_job, own_job_for_update, - positions_for, quota_detail, serialize, serialize_job, - source_keys, ) -from app.security import hash_password -from app.userscope import is_messageable_user, messageable_clauses -from app.utils import iso +from app.userscope import emails_of, is_messageable_user, messageable_others, user_by_email from app.refusal import refused +if TYPE_CHECKING: + from app.downloads.models import DownloadJob, DownloadLink, DownloadProfile, FileFolder + router = APIRouter(prefix="/api/downloads", tags=["downloads"]) admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"]) @@ -118,7 +105,7 @@ def resolve_source(raw: str) -> tuple[str, str]: # --- profiles ------------------------------------------------------------------------------ -def _serialize_profile(p: DownloadProfile) -> dict: +def _serialize_profile(p: "DownloadProfile") -> dict: return { "id": p.id, "name": p.name, @@ -130,18 +117,7 @@ def _serialize_profile(p: DownloadProfile) -> dict: @router.get("/profiles") def list_profiles(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]: - rows = ( - db.execute( - select(DownloadProfile) - .where((DownloadProfile.is_builtin.is_(True)) | (DownloadProfile.user_id == user.id)) - .order_by( - DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id - ) - ) - .scalars() - .all() - ) - return [_serialize_profile(p) for p in rows] + return [_serialize_profile(p) for p in queries.profiles_for(db, user)] class CreateProfileIn(BaseModel): @@ -157,15 +133,7 @@ def create_profile( spec = body.spec if not name: raise refused(400, "profile_name_required") - maxpos = db.execute( - select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where( - DownloadProfile.user_id == user.id - ) - ).scalar_one() - p = DownloadProfile( - user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1 - ) - db.add(p) + p = service.create_profile(db, user, name, spec) db.commit() return _serialize_profile(p) @@ -182,31 +150,32 @@ def update_profile( user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: - p = db.get(DownloadProfile, profile_id) - if p is None or p.is_builtin or p.user_id != user.id: - raise not_found("profile") + p = _own_profile(db, user, profile_id) sent = body.model_fields_set + name = None if "name" in sent: name = (body.name or "").strip() if not name: raise refused(400, "profile_name_required") - p.name = name[:80] - if "spec" in sent: - if body.spec is None: - raise refused(400, "spec_not_object") - p.spec = body.spec + if "spec" in sent and body.spec is None: + raise refused(400, "spec_not_object") + service.update_profile(p, name=name, spec=body.spec if "spec" in sent else None) db.commit() return _serialize_profile(p) +def _own_profile(db: Session, user: User, profile_id: int) -> "DownloadProfile": + p = queries.own_profile(db, user, profile_id) + if p is None: + raise not_found("profile") + return p + + @router.delete("/profiles/{profile_id}") def delete_profile( profile_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: - p = db.get(DownloadProfile, profile_id) - if p is None or p.is_builtin or p.user_id != user.id: - raise not_found("profile") - db.delete(p) + service.delete_profile(db, _own_profile(db, user, profile_id)) db.commit() return {"deleted": profile_id} @@ -276,18 +245,13 @@ def _resolve_spec( """Pick the format spec for an enqueue: an explicit profile_id (builtin or the user's own), an inline spec, or the first builtin as a fallback.""" if profile_id is not None: - p = db.get(DownloadProfile, profile_id) - if p is None or (not p.is_builtin and p.user_id != user.id): + p = queries.usable_profile(db, user, profile_id) + if p is None: raise not_found("profile") return p.spec, p.id if spec is not None: return spec, None - fallback = db.execute( - select(DownloadProfile) - .where(DownloadProfile.is_builtin.is_(True)) - .order_by(DownloadProfile.sort_order) - .limit(1) - ).scalar_one_or_none() + fallback = queries.default_profile(db) if fallback is None: raise refused(400, "no_profile") return fallback.spec, fallback.id @@ -325,6 +289,7 @@ def enqueue_download( job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name) except quota.QuotaExceeded as e: raise HTTPException(status_code=422, detail=quota_detail(e)) from e + db.commit() return serialize_job(db, job) @@ -354,6 +319,7 @@ def enqueue_edit( raise HTTPException(status_code=422, detail=quota_detail(e)) from e except service.EditError as e: raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e + db.commit() return serialize_job(db, job) @@ -397,38 +363,22 @@ def enqueue_clip( raise HTTPException(status_code=status, detail=detail) from e except quota.QuotaExceeded as e: raise HTTPException(status_code=422, detail=quota_detail(e)) from e + db.commit() return serialize_job(db, job) # How long a finished job stays in the queue: the tray polls it from every page and must see a job # turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's. -# A done job is timed from when it became done, never from its last write: a rename, a move or the -# GC's expiry warning touches an old file's row, and the tray would announce it as just finished. QUEUE_RECENT = timedelta(minutes=10) @router.get("") def list_downloads(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]: """The queue: what is waiting, running, paused or failed, and what finished a moment ago.""" - recent = datetime.now(timezone.utc) - QUEUE_RECENT - jobs = ( - db.execute( - select(DownloadJob) - .where( - DownloadJob.user_id == user.id, - DownloadJob.status.not_in(FINISHED) - | ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent)) - | ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)), - ) - .order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()) - .limit(MAX_LIST) - ) - .scalars() - .all() - ) - assets = assets_for(db, jobs) - positions = positions_for(db, user, jobs) - sources = source_keys(db, jobs) + jobs = queries.queue(db, user, datetime.now(timezone.utc) - QUEUE_RECENT) + assets = queries.assets_for(db, jobs) + positions = queries.positions_for(db, user, jobs) + sources = queries.source_keys(db, jobs) return [ { **serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None), @@ -452,14 +402,8 @@ _INDEX_RANK: dict[str, int] = { def my_download_index(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict: """source_ref -> best active status, for the feed's per-card "downloaded / queued" badge. Small + cheap so the feed can poll it without loading the full job list.""" - rows = db.execute( - select(DownloadJob.source_ref, DownloadJob.status).where( - DownloadJob.user_id == user.id, - DownloadJob.status.in_(tuple(_INDEX_RANK)), - ) - ).all() out: dict[str, str] = {} - for ref, status in rows: + for ref, status in queries.statuses_by_source(db, user, tuple(_INDEX_RANK)): if _INDEX_RANK.get(status, 0) > _INDEX_RANK.get(out.get(ref, ""), 0): out[ref] = status return out @@ -471,6 +415,7 @@ def pause_download( ) -> dict: job = own_job_for_update(db, user, job_id) service.pause(db, job) + db.commit() return serialize_job(db, job) @@ -480,6 +425,7 @@ def resume_download( ) -> dict: job = own_job_for_update(db, user, job_id) service.resume(db, job) + db.commit() return serialize_job(db, job) @@ -507,6 +453,7 @@ def redownload_download( if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED): raise refused(409, "still_downloading") service.redownload(db, job) + db.commit() return serialize_job(db, job) @@ -516,6 +463,7 @@ def cancel_download( ) -> dict: job = own_job_for_update(db, user, job_id) service.cancel(db, job) + db.commit() return serialize_job(db, job) @@ -533,24 +481,7 @@ def set_library_progress( one shared with them). A position too early or too close to the end clears it instead.""" accessible_job(db, user, job_id) position = kept_position(body) - mine = (LibraryState.user_id == user.id) & (LibraryState.job_id == job_id) - if not position: - # A statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon of - # one stop may both clear it, and the second then deletes nothing rather than failing. - db.execute(delete(LibraryState).where(mine)) - db.commit() - return {"ok": True, "job_id": job_id, "position_seconds": 0} - now = datetime.now(timezone.utc) - # One row per (user, job) even when two checkpoints race (the same pair): an upsert, not a - # read-then-insert that the second would fail on the unique key. - db.execute( - pg_insert(LibraryState) - .values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now) - .on_conflict_do_update( - constraint="uq_library_user_job", - set_={"position_seconds": position, "progress_updated_at": now}, - ) - ) + service.set_position(db, user, job_id, position) db.commit() return {"ok": True, "job_id": job_id, "position_seconds": position} @@ -563,8 +494,7 @@ def storyboard_meta( Best-effort: for a very long/high-res source the sprite may not generate in time — then `available` is False and the editor shows a plain timeline.""" - job = accessible_job(db, user, job_id) - asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None + asset = queries.asset_of(db, accessible_job(db, user, job_id)) if asset is None or asset.status != "ready" or not asset.rel_path: raise refused(409, "download_not_ready") meta = editmod.ensure_storyboard(asset, settings.files_root) @@ -584,8 +514,7 @@ def storyboard_meta( @router.get("/{job_id}/poster.jpg") def poster_image(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)): """The locally-generated poster frame for a download whose source had no thumbnail.""" - job = accessible_job(db, user, job_id) - asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None + asset = queries.asset_of(db, accessible_job(db, user, job_id)) if asset is None or not asset.poster_path: raise not_found("poster") path = storage.safe_abs_path(settings.files_root, asset.poster_path) @@ -598,8 +527,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session = def storyboard_image( job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ): - job = accessible_job(db, user, job_id) - asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None + asset = queries.asset_of(db, accessible_job(db, user, job_id)) if asset is None or not asset.storyboard_path: raise not_found("filmstrip") path = storage.safe_abs_path(settings.files_root, asset.storyboard_path) @@ -622,45 +550,20 @@ def share_download( user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: - job = owned_or_404(db, DownloadJob, job_id, user, what="download") + job = own_job(db, user, job_id) if job.status != JobStatus.DONE or job.asset_id is None: raise refused(409, "share_unfinished") target = body.email.strip().lower() if not target: raise refused(400, "recipient_required") - recipient = db.execute( - select(User).where(func.lower(User.email) == target) - ).scalar_one_or_none() + recipient = user_by_email(db, target) # Uniform "no user" for a missing OR non-messageable (suspended/deactivated) target, so # sharing can't be used to probe which addresses exist or to reach a suspended account. - if not is_messageable_user(recipient): + if recipient is None or not is_messageable_user(recipient): raise not_found("recipient") if recipient.id == user.id: raise refused(400, "share_own") - exists = db.execute( - select(DownloadShare.id).where( - DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id - ) - ).first() - if exists is None: - db.add(DownloadShare(job_id=job_id, from_user_id=user.id, to_user_id=recipient.id)) - from app.notifications import create_notification - - create_notification( - db, - user_id=recipient.id, - type="download_shared", - title=(job.display_name or "A download") + " was shared with you", - # The client names the notice in the reader's language from `data`; the title is the - # English fallback only. - data={ - "kind": "download_shared", - "from": user.email, - "job_id": job_id, - "title": job.display_name, - }, - commit=False, - ) + if service.share(db, job, user, recipient): db.commit() return {"ok": True, "shared_with": recipient.email} @@ -672,22 +575,10 @@ def unshare_download( user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: - owned_or_404( - db, DownloadJob, job_id, user, what="download" - ) # ownership guard (404s if not the caller's job) - recipient = db.execute( - select(User).where(func.lower(User.email) == email.strip().lower()) - ).scalar_one_or_none() - if recipient is not None: - row = db.execute( - select(DownloadShare).where( - DownloadShare.job_id == job_id, - DownloadShare.to_user_id == recipient.id, - ) - ).scalar_one_or_none() - if row is not None: - db.delete(row) - db.commit() + own_job(db, user, job_id) # ownership guard (404s if not the caller's job) + recipient = user_by_email(db, email) + if recipient is not None and service.unshare(db, job_id, recipient): + db.commit() return {"ok": True} @@ -700,54 +591,26 @@ def share_recipients( ) -> list[dict]: """Registered users this user can share a download with (excludes self + any non-messageable account — suspended or deactivated — so the picker never leaks those addresses).""" - rows = ( - db.execute( - select(User) - .where(User.id != user.id, *messageable_clauses()) - .order_by(collated(User.email, lang_for_user(user))) - ) - .scalars() - .all() - ) + rows = messageable_others(db, user, lang_for_user(user)) return [{"id": u.id, "email": u.email, "display_name": u.display_name} for u in rows] # --- public watch links (share by link) ---------------------------------------------------- -def _own_link(db: Session, user: User, link_id: int) -> DownloadLink: - link = db.get(DownloadLink, link_id) +def _own_link(db: Session, user: User, link_id: int) -> "DownloadLink": + link = queries.own_link(db, user, link_id) if link is None: raise not_found("link") - target = ( - db.get(DownloadJob, link.job_id) - if link.job_id is not None - else db.get(FileFolder, link.folder_id) - ) - if target is None or target.user_id != user.id: - raise not_found("link") return link -def _link_expiry(days: int | None) -> datetime | None: - # None/0/negative → no expiry; otherwise N days out, capped at ~10 years. The Pydantic model - # already coerced/validated the type, so a non-number is a 422 before we get here. - if not days or days <= 0: - return None - return datetime.now(timezone.utc) + timedelta(days=min(days, 3650)) - - -def _links_of(db: Session, *where) -> list[dict]: - rows = db.execute(select(DownloadLink).where(*where).order_by(DownloadLink.id.desc())) - return [linksmod.owner_view(row) for row in rows.scalars()] - - @router.get("/{job_id}/links") def list_links( job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> list[dict]: - owned_or_404(db, DownloadJob, job_id, user, what="download") - return _links_of(db, DownloadLink.job_id == job_id) + own_job(db, user, job_id) + return [linksmod.owner_view(link) for link in queries.links_to(db, job_id=job_id)] class CreateLinkIn(BaseModel): @@ -763,30 +626,18 @@ class CreateLinkIn(BaseModel): def _new_link( db: Session, user: User, body: CreateLinkIn, **target: int -) -> tuple[DownloadLink, str]: - """Make a link to `target` (`job_id=` or `folder_id=`, already checked to be the user's), and - the token it is opened by — which is in hand only now.""" - pw = body.password.strip() - token = linksmod.new_token() - link = DownloadLink( - **target, - created_by=user.id, - allow_download=body.allow_download, - password_hash=hash_password(pw) if pw else None, - expires_at=_link_expiry(body.expires_days), - ) - linksmod.seal(link, token) - db.add(link) - db.flush() - audit.record( +) -> tuple["DownloadLink", str]: + """A link to `target` (`job_id=` or `folder_id=`, already checked to be the user's) and its + token (`links.create`). The Pydantic model already coerced the expiry, so a non-number is a 422 + before this runs.""" + return linksmod.create( db, - user.id, - audit.AuditAction.DOWNLOAD_LINK_CREATE, - target_type="download_link", - target_id=link.id, - after={**target, **_link_audit(link)}, + user, + password=body.password.strip(), + allow_download=body.allow_download, + expires_days=body.expires_days, + **target, ) - return link, token @router.post("/{job_id}/links") @@ -804,28 +655,20 @@ def create_link( return linksmod.owner_view(link, token=token) -def _shareable_job(db: Session, user: User, job_id: int) -> DownloadJob: - """One of the user's files, held (FOR SHARE) for the rest of the transaction: a delete of it at - the same moment (the user's, or the retention sweep's) waits and takes the new link with it, - rather than failing the link's foreign key — or, done first, leaves a 404.""" - owned_or_404(db, DownloadJob, job_id, user, what="download") - job = db.execute( - select(DownloadJob) - .where(DownloadJob.id == job_id) - .with_for_update(read=True) - .execution_options(populate_existing=True) - ).scalar_one_or_none() +def _shareable_job(db: Session, user: User, job_id: int) -> "DownloadJob": + """One of the user's files, held for the link's transaction (`service.hold_for_share`).""" + job = service.hold_for_share(db, user, job_id) if job is None: raise not_found("download") return job -def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder: +def _shareable_folder(db: Session, user: User, folder_id: int) -> "FileFolder": """One of the user's own folders. A source folder lets its files go on its own clock, and the Hub is the user's channel to a machine of theirs: neither is a shelf to hand out. Held (FOR SHARE) for the rest of the transaction, so a delete of it at the same moment waits and takes the new link with it, rather than failing the link's foreign key.""" - owned_or_404(db, FileFolder, folder_id, user, what="folder") + own_folder(db, user, folder_id) try: folder = folders.own_folder(db, user.id, folder_id) except folders.FolderError as e: @@ -839,8 +682,8 @@ def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder: def list_folder_links( folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> list[dict]: - owned_or_404(db, FileFolder, folder_id, user, what="folder") - return _links_of(db, DownloadLink.folder_id == folder_id) + own_folder(db, user, folder_id) + return [linksmod.owner_view(link) for link in queries.links_to(db, folder_id=folder_id)] @router.post("/folders/{folder_id}/links") @@ -856,15 +699,6 @@ def create_folder_link( return linksmod.owner_view(link, token=token) -def _link_audit(link: DownloadLink) -> dict: - """A link's settings as the audit records them: whether it has a password, never which.""" - return { - "allow_download": link.allow_download, - "has_password": link.password_hash is not None, - "expires_at": iso(link.expires_at), - } - - class UpdateLinkIn(BaseModel): allow_download: bool | None = None expires_days: int | None = None @@ -879,29 +713,15 @@ def update_link( db: Session = Depends(get_db), ) -> dict: link = _own_link(db, user, link_id) - before = _link_audit(link) - sent = body.model_fields_set - if "allow_download" in sent: - link.allow_download = bool(body.allow_download) - if "expires_days" in sent: - link.expires_at = _link_expiry(body.expires_days) - if "password" in sent: - pw = (body.password or "").strip() - link.password_hash = hash_password(pw) if pw else None - # Bump the version so any grant issued under the previous password stops working now, rather - # than living out its 6h TTL (setting/clearing/changing the password all count as a rotation). - link.password_version += 1 - after = _link_audit(link) - if after != before or "password" in sent: - audit.record( - db, - user.id, - audit.AuditAction.DOWNLOAD_LINK_UPDATE, - target_type="download_link", - target_id=link_id, - before=before, - after=after, - ) + linksmod.update( + db, + user, + link, + allow_download=body.allow_download, + expires_days=body.expires_days, + password=body.password, + sent=frozenset(body.model_fields_set), + ) db.commit() return linksmod.owner_view(link) @@ -910,16 +730,7 @@ def update_link( def revoke_link( link_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: - link = _own_link(db, user, link_id) - db.delete(link) - audit.record( - db, - user.id, - audit.AuditAction.DOWNLOAD_LINK_REVOKE, - target_type="download_link", - target_id=link_id, - after={"job_id": link.job_id} if link.job_id is not None else {"folder_id": link.folder_id}, - ) + linksmod.revoke(db, user, _own_link(db, user, link_id)) db.commit() return {"ok": True, "deleted": link_id} @@ -931,20 +742,9 @@ def revoke_link( def admin_list_downloads( admin: User = Depends(admin_user), db: Session = Depends(get_db) ) -> list[dict]: - jobs = ( - db.execute( - select(DownloadJob) - .order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()) - .limit(MAX_LIST) - ) - .scalars() - .all() - ) - assets = assets_for(db, jobs) - emails = { - u.id: u.email - for u in db.execute(select(User).where(User.id.in_({j.user_id for j in jobs}))).scalars() - } + jobs = queries.recent_jobs(db) + assets = queries.assets_for(db, jobs) + emails = emails_of(db, {j.user_id for j in jobs}) out = [] for j in jobs: data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None) @@ -956,27 +756,11 @@ def admin_list_downloads( @admin_router.get("/storage") def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict: - # Every file on disk (`quota.ON_DISK`): ready ones, and uploads waiting for or failed in their - # conversion — the same disk the footprint and the GC cap count. - on_disk = db.execute( - select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK) - ).one() - total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar() + files_on_disk, total_bytes, total_assets = queries.storage_totals(db) # EVERY user, not only those holding a file: a quota is set before anyone downloads. - emails: dict[int, str] = { - uid: email - for uid, email in db.execute(select(User.id, User.email).order_by(User.email)).tuples() - } + emails = emails_of(db) uids = list(emails) - # What each one's Files lists: their finished files. - files: dict[int, int] = { - uid: n - for uid, n in db.execute( - select(DownloadJob.user_id, func.count()) - .where(DownloadJob.status == JobStatus.DONE) - .group_by(DownloadJob.user_id) - ).tuples() - } + files = queries.done_files_per_user(db) def row(uid: int) -> dict: lim = quota.resolve(db, uid) @@ -990,8 +774,8 @@ def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_d } return { - "files_on_disk": on_disk[0], - "total_bytes": int(on_disk[1]), + "files_on_disk": files_on_disk, + "total_bytes": total_bytes, "total_assets": total_assets, "total_cap_bytes": sysconfig.get_int(db, "download_total_max_bytes"), "per_user": [row(uid) for uid in uids], @@ -1003,10 +787,9 @@ def admin_get_quota( user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db) ) -> dict: lim = quota.resolve(db, user_id) - row = db.get(DownloadQuota, user_id) return { "user_id": user_id, - "custom": row is not None, + "custom": queries.custom_quota(db, user_id) is not None, "max_bytes": lim.max_bytes, "max_concurrent": lim.max_concurrent, "max_jobs": lim.max_jobs, @@ -1031,25 +814,14 @@ def admin_set_quota( db: Session = Depends(get_db), ) -> dict: get_or_404(db, User, user_id, what="user") - cur = quota.resolve(db, user_id) - row = db.get(DownloadQuota, user_id) - if row is None: - row = DownloadQuota( - user_id=user_id, - max_bytes=cur.max_bytes, - max_concurrent=cur.max_concurrent, - max_jobs=cur.max_jobs, - unlimited=cur.unlimited, - ) - db.add(row) - if body.max_bytes is not None: - row.max_bytes = max(0, body.max_bytes) - if body.max_concurrent is not None: - row.max_concurrent = max(1, body.max_concurrent) - if body.max_jobs is not None: - row.max_jobs = max(1, body.max_jobs) - if body.unlimited is not None: - row.unlimited = body.unlimited + quota.set_custom( + db, + user_id, + max_bytes=body.max_bytes, + max_concurrent=body.max_concurrent, + max_jobs=body.max_jobs, + unlimited=body.unlimited, + ) db.commit() return admin_get_quota(user_id, admin, db) @@ -1058,8 +830,6 @@ def admin_set_quota( def admin_reset_quota( user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db) ) -> dict: - row = db.get(DownloadQuota, user_id) - if row is not None: - db.delete(row) + if quota.reset_custom(db, user_id): db.commit() return admin_get_quota(user_id, admin, db) diff --git a/backend/app/routes/files.py b/backend/app/routes/files.py index fcf385d8..dd16f0ea 100644 --- a/backend/app/routes/files.py +++ b/backend/app/routes/files.py @@ -5,8 +5,10 @@ the per-user layer (name, folder, kept, expiry, shares and links all hang off it bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk. In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above -the routes are shared with that module — serialization, the ownership locks, the upload spool — -because a download and a library file are one row at two moments. +the routes are shared with that module — serialization, the ownership checks, the upload spool — +because a download and a library file are one row at two moments. What is read comes from +`downloads.queries`, what is written goes through Capture's service (`service`, `folders`); this +module validates, answers in HTTP and commits. A machine with an API token (`app.apitokens`) may use the routes that take `files_caller`: list the folders and files, fetch a file and its hash, upload, delete — each only in a folder its token was @@ -19,49 +21,37 @@ import hashlib import os import tempfile import threading -from collections.abc import AsyncIterator, Sequence +from collections.abc import AsyncIterator from dataclasses import dataclass -from datetime import datetime, timezone from pathlib import Path -from typing import BinaryIO, Literal +from typing import TYPE_CHECKING, BinaryIO, Literal from urllib.parse import unquote, urlparse from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import FileResponse, Response from pydantic import BaseModel, Field -from sqlalchemy import func, or_, select, update -from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from app import audit, sysconfig from app.apitokens import FilesCaller, Right, files_caller from app.audit import AuditAction from app.auth import current_user -from app.collation import collated from app.config import settings from app.db import get_db, release_read_connection -from app.downloads import folders, preview, quota, service, storage +from app.downloads import folders, preview, queries, quota, service, storage from app.downloads import upload as uploadmod from app.downloads.status import JobStatus from app.mailtext import lang_for_user from app.models import DISPLAY_TEXT_MAX, User, display_text -from app.downloads.models import ( - DownloadJob, - DownloadLink, - DownloadShare, - FileFolder, - LibraryState, - MediaAsset, -) from app.refusal import refused -from app.routes._common import capped_stream_to, not_found, owned_or_404 -from app.utils import iso, like_escape +from app.routes._common import capped_stream_to, not_found +from app.utils import iso + +if TYPE_CHECKING: + from app.downloads.models import DownloadJob, FileFolder, MediaAsset router = APIRouter(prefix="/api/files", tags=["files"]) -# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's -# own cap; far beyond any real library). -MAX_LIST = 500 # The longest a folder may keep a file, in days — ten years is "forever" in all but name. _MAX_RETENTION_DAYS = 3650 _HASH_CHUNK = 1 << 20 @@ -73,7 +63,7 @@ _HASHING = threading.BoundedSemaphore(_HASHERS) # --- shared with routes.downloads ------------------------------------------------------- -def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict: +def serialize(job: "DownloadJob", asset: "MediaAsset | None") -> dict: ready = asset is not None and asset.status == "ready" and bool(asset.rel_path) return { "id": job.id, @@ -133,58 +123,18 @@ def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict: } -def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]: - ids = {j.asset_id for j in jobs if j.asset_id} - if not ids: - return {} - rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars() - return {a.id: a for a in rows} - - -def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]: - """`user`'s resume point in each of `jobs` that has one (the library player's `library_states`).""" - ids = [j.id for j in jobs] - if not ids: - return {} - rows = db.execute( - select(LibraryState.job_id, LibraryState.position_seconds).where( - LibraryState.user_id == user.id, LibraryState.job_id.in_(ids) - ) - ).all() - return {job_id: pos for job_id, pos in rows} - - -def serialize_job(db: Session, job: DownloadJob) -> dict: +def serialize_job(db: Session, job: "DownloadJob") -> dict: """Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step every single-job handler shares.""" - asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None - return serialize(job, asset) + return serialize(job, queries.asset_of(db, job)) -def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob: - """The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload) - or writes the status those decide on (pause). The job is re-read under a row lock, so two - overlapping requests for one job run one after the other and the second decides on the first's - committed result — never both releasing the same hold, and never a pause turning a job a cancel - just released back into a holder; either can free a file another job still holds. - - The asset is locked BEFORE the job: the order a finishing worker child takes them in - (`service.finalize_if_running`), so the two can never deadlock.""" - while True: - asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id - if asset_id is not None: - db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update()) - job = db.execute( - select(DownloadJob) - .where(DownloadJob.id == job_id) - .with_for_update() - .execution_options(populate_existing=True) - ).scalar_one_or_none() - if job is None: - raise not_found("download") - if job.asset_id == asset_id: - return job - db.rollback() # another request moved the job to another asset meanwhile: lock that one +def own_job_for_update(db: Session, user: User, job_id: int) -> "DownloadJob": + """One of the user's jobs under its row lock (`service.lock_own_job`), or a 404.""" + job = service.lock_own_job(db, user, job_id) + if job is None: + raise not_found("download") + return job def quota_detail(e: quota.QuotaExceeded) -> dict: @@ -320,30 +270,32 @@ async def spool_upload( tmp_path.unlink(missing_ok=True) -def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob: - """A file the caller may read: their own, or one shared with them.""" - job = db.get(DownloadJob, job_id) +def accessible_job(db: Session, user: User, job_id: int) -> "DownloadJob": + """A file the caller may read (`queries.accessible_job`: their own, or shared with them).""" + job = queries.accessible_job(db, user, job_id) if job is None: raise not_found("download") - if job.user_id == user.id: - return job - shared = db.execute( - select(DownloadShare.id).where( - DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id - ) - ).first() - if shared is None: - raise not_found("download") return job -def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob: - """A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's - own in a folder it may read.""" - if caller.grants is None: - return accessible_job(db, caller.user, job_id) - job = db.get(DownloadJob, job_id) - if job is None or job.user_id != caller.user.id or not caller.may(job.folder_id, Right.READ): +def own_job(db: Session, user: User, job_id: int) -> "DownloadJob": + job = queries.own_job(db, user, job_id) + if job is None: + raise not_found("download") + return job + + +def own_folder(db: Session, user: User, folder_id: int) -> "FileFolder": + folder = queries.own_folder(db, user, folder_id) + if folder is None: + raise not_found("folder") + return folder + + +def readable_job(db: Session, caller: FilesCaller, job_id: int) -> "DownloadJob": + """A file `caller` may read (`queries.readable_job`), or a 404.""" + job = queries.readable_job(db, caller, job_id) + if job is None: raise not_found("download") return job @@ -351,7 +303,7 @@ def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob: # --- folders ------------------------------------------------------------------------------- -def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool = False) -> dict: +def _serialize_folder(folder: "FileFolder", count: int, size: int, linked: bool = False) -> dict: return { "id": folder.id, "kind": folder.kind, @@ -367,18 +319,6 @@ def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool = } -def _linked(db: Session, folder_ids: list[int]) -> set[int]: - """Which of these folders a live public link opens (one not past its expiry).""" - now = datetime.now(timezone.utc) - rows = db.execute( - select(DownloadLink.folder_id).where( - DownloadLink.folder_id.in_(folder_ids), - or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now), - ) - ).scalars() - return {folder_id for folder_id in rows if folder_id is not None} - - @router.get("/folders") def list_folders( caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db) @@ -386,23 +326,9 @@ def list_folders( """The caller's folders with how many files each holds and their size (a token: the ones it may list). A source folder its files have all left stays (count 0): the client decides whether to show it.""" - done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE) - stmt = select( - FileFolder, - func.count(DownloadJob.id), - func.coalesce(func.sum(MediaAsset.size_bytes), 0), - ).where(FileFolder.user_id == caller.user.id) - listable = caller.folders_with(Right.LIST) - if listable is not None: - stmt = stmt.where(FileFolder.id.in_(listable)) - rows = db.execute( - stmt.outerjoin(DownloadJob, done) - .outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id) - .group_by(FileFolder.id) - .order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id) - ).all() - linked = _linked(db, [f.id for f, _, _ in rows]) - return [_serialize_folder(f, count, int(size), f.id in linked) for f, count, size in rows] + rows = queries.folders_with_totals(db, caller.user, caller.folders_with(Right.LIST)) + linked = queries.linked_folders(db, [f.id for f, _, _ in rows]) + return [_serialize_folder(f, count, size, f.id in linked) for f, count, size in rows] class FolderIn(BaseModel): @@ -420,13 +346,10 @@ def _folder_name(raw: str) -> str: def create_folder( body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: - folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name)) try: - with db.begin_nested(): - db.add(folder) - db.flush() - except IntegrityError as e: # the unique (user, lower(name)): one folder per name - raise refused(409, "file_folder_name_taken") from e + folder = folders.create_own(db, user.id, _folder_name(body.name)) + except folders.FolderError as e: + raise refused(409, e.reason) from e audit.record( db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id ) @@ -449,17 +372,13 @@ def update_folder( ) -> dict: """Rename one of the caller's own folders (a source folder's name is its origin's), and/or set how long any of their folders keeps a file (`folders.set_retention`).""" - folder = owned_or_404(db, FileFolder, folder_id, user, what="folder") + folder = own_folder(db, user, folder_id) sent = body.model_fields_set if "name" in sent: - if folder.kind != folders.FolderKind.OWN: - raise refused(409, "file_folder_not_own") - folder.name = _folder_name(body.name or "") try: - with db.begin_nested(): - db.flush() - except IntegrityError as e: - raise refused(409, "file_folder_name_taken") from e + folders.rename(db, folder, _folder_name(body.name or "")) + except folders.FolderError as e: + raise refused(409, e.reason) from e audit.record( db, user.id, @@ -484,18 +403,9 @@ def update_folder( after={"retention_days": body.retention_days}, ) db.commit() - count, size = _folder_totals(db, folder.id) - return _serialize_folder(folder, count, size, folder.id in _linked(db, [folder.id])) - - -def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]: - count, size = db.execute( - select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0)) - .select_from(DownloadJob) - .outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id) - .where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE) - ).one() - return count, int(size) + count, size = queries.folder_totals(db, folder.id) + linked = folder.id in queries.linked_folders(db, [folder.id]) + return _serialize_folder(folder, count, size, linked) @router.delete("/folders/{folder_id}") @@ -503,7 +413,7 @@ def delete_folder( folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: """Delete one of the caller's own folders; its files go back to their source folders.""" - folder = owned_or_404(db, FileFolder, folder_id, user, what="folder") + folder = own_folder(db, user, folder_id) try: moved = folders.delete_own(db, folder) except folders.FolderError as e: @@ -545,13 +455,7 @@ def remove_shared_with_me( """Remove a file that was shared WITH this user from their 'Shared with me' list. Deletes only the recipient's share grant — the owner's job and the physical file are untouched (this is a per-user dismissal, not a delete).""" - row = db.execute( - select(DownloadShare).where( - DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id - ) - ).scalar_one_or_none() - if row is not None: - db.delete(row) + if service.unshare(db, job_id, user): db.commit() return {"ok": True, "removed": job_id} @@ -580,46 +484,22 @@ def list_files( raise refused(403, "api_token_scope") if folder_id is not None and folder_id not in listable: raise not_found("folder") - stmt = ( - select(DownloadJob) - .join(MediaAsset, MediaAsset.id == DownloadJob.asset_id) - .where(DownloadJob.status == JobStatus.DONE) - ) shared = scope == "shared" - if shared: - stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where( - DownloadShare.to_user_id == user.id - ) - else: - stmt = stmt.where(DownloadJob.user_id == user.id) - if folder_id is not None: - stmt = stmt.where(DownloadJob.folder_id == folder_id) - elif listable is not None: - stmt = stmt.where(DownloadJob.folder_id.in_(listable)) - name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "") - term = q.strip() - if term: - like = f"%{like_escape(term)}%" - stmt = stmt.where( - or_( - name.ilike(like), - func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like), - ) - ) - # A file is added to its owner's list when it is done, not when it was asked for (a download that - # failed for days and then went through is new); one shared with the caller, when it was shared. - key = { - "added": DownloadShare.created_at if shared else DownloadJob.ready_at, - "name": collated(name, lang_for_user(user)), - "size": MediaAsset.size_bytes, - }[sort] - first = key.asc() if order == "asc" else key.desc() - tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc() - jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all() - assets = assets_for(db, jobs) - positions = positions_for(db, user, jobs) - shared_at = _shared_at(db, user, jobs) if shared else {} - sources = source_keys(db, jobs) + jobs = queries.library( + db, + user, + folder_id=folder_id, + listable=listable, + shared=shared, + term=q.strip(), + sort=sort, + order=order, + lang=lang_for_user(user), + ) + assets = queries.assets_for(db, jobs) + positions = queries.positions_for(db, user, jobs) + shared_at = queries.shared_at(db, user, jobs) if shared else {} + sources = queries.source_keys(db, jobs) return [ { **serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None), @@ -632,36 +512,6 @@ def list_files( ] -def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]: - """Where each file came from (its source folder's key), for the list's Source column, the - "back to its source folder" move and the download tray's site: a file in a source folder has its folder's key; one moved into - an own folder (or someone else's, shared) is traced back by `folders.source_key_for`.""" - ids = {j.folder_id for j in jobs if j.folder_id is not None} - by_id = ( - {f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()} - if ids - else {} - ) - out: dict[int, str] = {} - for j in jobs: - folder = by_id.get(j.folder_id) if j.folder_id is not None else None - if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key: - out[j.id] = folder.source_key - else: - out[j.id] = folders.source_key_for(db, j) - return out - - -def _shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]: - rows = db.execute( - select(DownloadShare.job_id, DownloadShare.created_at).where( - DownloadShare.to_user_id == user.id, - DownloadShare.job_id.in_([j.id for j in jobs]), - ) - ).all() - return {job_id: at for job_id, at in rows} - - _MAX_EXTRA_LINKS = 8 @@ -709,18 +559,20 @@ def update_file( """Rename a file and its channel, set its reference links (all display-only: the file on disk keeps its own name), move it to another folder (`folders.move`), or keep it until deleted / hand it back to its folder's retention.""" - job = owned_or_404(db, DownloadJob, job_id, user, what="download") + job = own_job(db, user, job_id) sent = body.model_fields_set + display: dict[str, object] = {} if "name" in sent: - job.display_name = display_text(body.name) + display["display_name"] = display_text(body.name) if "uploader" in sent: - job.display_uploader = display_text(body.uploader) + display["display_uploader"] = display_text(body.uploader) if "uploader_url" in sent: - job.display_uploader_url = _clean_url(body.uploader_url) + display["display_uploader_url"] = _clean_url(body.uploader_url) if "extra_links" in sent: raw = body.extra_links or [] cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS] - job.extra_links = cleaned or None + display["extra_links"] = cleaned or None + service.set_display(job, **display) if "folder_id" in sent: if body.folder_id is None: raise refused(400, "file_folder_gone") @@ -766,7 +618,8 @@ def delete_file( target_id=job_id, summary=caller.via, ) - service.delete(db, job) # commits, the audit row with it + service.delete(db, job) + db.commit() # the audit row with it return {"ok": True, "deleted": job_id} @@ -785,7 +638,7 @@ def upload_file( return serialize_job(db, ingest_spooled(db, upload)) -def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob: +def ingest_spooled(db: Session, upload: SpooledUpload) -> "DownloadJob": """The spooled upload into the library (`upload.ingest_upload`, which commits its own rows), its refusals answered as HTTP.""" if upload.size == 0: @@ -809,8 +662,8 @@ def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob: return job -def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]: - asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None +def _ready_file(db: Session, job: "DownloadJob") -> tuple["MediaAsset", Path]: + asset = queries.asset_of(db, job) if asset is None or asset.status != "ready" or not asset.rel_path: raise refused(409, "download_not_ready") path = storage.safe_abs_path(settings.files_root, asset.rel_path) @@ -819,7 +672,7 @@ def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]: return asset, path -def serve_file(db: Session, job: DownloadJob) -> FileResponse: +def serve_file(db: Session, job: "DownloadJob") -> FileResponse: """The file of `job` (the caller may read it), range-aware, named by the user's display name. The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list @@ -837,7 +690,7 @@ def serve_file(db: Session, job: DownloadJob) -> FileResponse: if asset.sha256: digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode() headers["Repr-Digest"] = f"sha-256=:{digest}:" - asset.last_access_at = datetime.now(timezone.utc) # the handler commits it + service.touch(asset) # the handler commits it media_type = storage.served_media_type(asset.media_kind, asset.container) if media_type is None: return FileResponse( @@ -892,13 +745,13 @@ def preview_file( text = preview.text_utf8(head) if text is None: raise refused(415, "file_no_preview") - asset.last_access_at = datetime.now(timezone.utc) + service.touch(asset) db.commit() return Response(text, media_type=preview.media_type(kind, asset.container), headers=headers) filename = storage.download_filename( job.display_name or asset.title or asset.source_ref, asset.container, path ) - asset.last_access_at = datetime.now(timezone.utc) + service.touch(asset) db.commit() return FileResponse( path, @@ -934,10 +787,6 @@ def get_sha256( finally: _HASHING.release() digest = h.hexdigest() - db.execute( - update(MediaAsset) - .where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None)) - .values(sha256=digest) - ) + service.store_sha256(db, asset_id, digest) db.commit() return {"sha256": digest} diff --git a/backend/app/routes/public.py b/backend/app/routes/public.py index 1fc6949c..f0312c15 100644 --- a/backend/app/routes/public.py +++ b/backend/app/routes/public.py @@ -6,24 +6,26 @@ the app or any account data. Password-protected links exchange the password once a short-lived signed grant that the media URL carries (a `