From 3a688fa0f5fb761e112894b8fc4f96c0da1c0dad Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 18 Oct 2026 23:45:53 +0200 Subject: [PATCH] refactor(capture): the unit's tables move into downloads/models The nine Capture tables (files, jobs, folders, shares, links, quota, cookies, profiles, library resume points) leave app/models.py for the unit's own module; app.tables registers every unit's tables for the migrations, the test database and the ownership check. No schema change. --- backend/alembic/env.py | 2 +- backend/app/downloads/backfill.py | 2 +- backend/app/downloads/credentials.py | 2 +- backend/app/downloads/folders.py | 2 +- backend/app/downloads/gc.py | 2 +- backend/app/downloads/jobdb.py | 2 +- backend/app/downloads/links.py | 2 +- backend/app/downloads/models.py | 385 ++++++++++++++++++ backend/app/downloads/og.py | 2 +- backend/app/downloads/pipeline.py | 3 +- backend/app/downloads/quota.py | 2 +- backend/app/downloads/runner.py | 2 +- backend/app/downloads/service.py | 11 +- backend/app/downloads/upload.py | 3 +- backend/app/models.py | 361 +--------------- backend/app/routes/downloads.py | 5 +- backend/app/routes/files.py | 6 +- backend/app/routes/public.py | 3 +- backend/app/routes/tokens.py | 3 +- backend/app/tables.py | 9 + backend/app/worker.py | 3 +- backend/tests/conftest.py | 2 +- backend/tests/test_api_tokens.py | 12 +- backend/tests/test_clip.py | 3 +- backend/tests/test_download_credentials.py | 3 +- .../tests/test_download_failure_catalog.py | 3 +- backend/tests/test_download_job_locks.py | 3 +- backend/tests/test_download_limits.py | 3 +- backend/tests/test_download_ref_count.py | 3 +- backend/tests/test_download_share_notice.py | 3 +- backend/tests/test_download_transitions.py | 3 +- backend/tests/test_downloads_finalize.py | 3 +- backend/tests/test_edit_cap.py | 2 +- backend/tests/test_failure_lines.py | 2 +- backend/tests/test_file_folders.py | 3 +- backend/tests/test_files_api.py | 3 +- backend/tests/test_files_preview.py | 2 +- backend/tests/test_folder_links.py | 5 +- backend/tests/test_library_progress.py | 2 +- backend/tests/test_link_token_at_rest.py | 3 +- backend/tests/test_not_found_detail.py | 2 +- backend/tests/test_odd_frame.py | 2 +- backend/tests/test_phone_safe_contract.py | 2 +- backend/tests/test_unit_fences.py | 3 +- backend/tests/test_upload.py | 3 +- backend/tests/test_upload_conversion.py | 3 +- backend/tests/test_worker_dispatch.py | 2 +- .../src/lib/api/downloads.contract.test.ts | 6 +- 48 files changed, 469 insertions(+), 429 deletions(-) create mode 100644 backend/app/downloads/models.py create mode 100644 backend/app/tables.py diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 2870589f..7bd6efbe 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config, pool from app.config import settings from app.db import Base from app.logs import configure_logging -import app.models # noqa: F401 (import so models register on Base.metadata) +import app.tables # noqa: F401 (import so every table registers on Base.metadata) config = context.config config.set_main_option("sqlalchemy.url", settings.database_url) diff --git a/backend/app/downloads/backfill.py b/backend/app/downloads/backfill.py index 13d035ab..ae400902 100644 --- a/backend/app/downloads/backfill.py +++ b/backend/app/downloads/backfill.py @@ -22,7 +22,7 @@ from app.db import SessionLocal from app.downloads import adapters from app.downloads import edit as editmod from app.downloads import formats, service -from app.models import MediaAsset +from app.downloads.models import MediaAsset log = logs.get_logger(__name__) diff --git a/backend/app/downloads/credentials.py b/backend/app/downloads/credentials.py index 648c441d..026634e2 100644 --- a/backend/app/downloads/credentials.py +++ b/backend/app/downloads/credentials.py @@ -33,7 +33,7 @@ from app import logs from app.downloads import clip as clipmod from app.downloads import formats from app.downloads.adapters import _host_is -from app.models import DownloadCredential, MediaAsset +from app.downloads.models import DownloadCredential, MediaAsset from app.security import decrypt_optional, encrypt log = logs.get_logger(__name__) diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py index 50f2d98e..a431a6bc 100644 --- a/backend/app/downloads/folders.py +++ b/backend/app/downloads/folders.py @@ -32,7 +32,7 @@ from sqlalchemy.orm import Session from app import sysconfig from app.downloads.status import JobStatus -from app.models import DownloadJob, FileFolder, MediaAsset +from app.downloads.models import DownloadJob, FileFolder, MediaAsset class FolderKind(StrEnum): diff --git a/backend/app/downloads/gc.py b/backend/app/downloads/gc.py index 50f6bb42..9cfd052f 100644 --- a/backend/app/downloads/gc.py +++ b/backend/app/downloads/gc.py @@ -31,7 +31,7 @@ from app import apitokens, logs, sysconfig from app.config import settings from app.downloads import quota, service, storage from app.downloads.status import JobStatus -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset from app.notifications import create_notification log = logs.get_logger(__name__) diff --git a/backend/app/downloads/jobdb.py b/backend/app/downloads/jobdb.py index a8e95b29..14b34b4a 100644 --- a/backend/app/downloads/jobdb.py +++ b/backend/app/downloads/jobdb.py @@ -6,7 +6,7 @@ from sqlalchemy import text from app.config import settings from app.db import SessionLocal -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset # --- small DB helpers (short-lived sessions; the download itself holds no transaction) ------- diff --git a/backend/app/downloads/links.py b/backend/app/downloads/links.py index 80a0ac77..5d46335a 100644 --- a/backend/app/downloads/links.py +++ b/backend/app/downloads/links.py @@ -24,7 +24,7 @@ from app import security from app.config import settings from app.downloads import storage from app.downloads.status import JobStatus -from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset +from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset _GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks # The most files a folder link lists: as many as the owner's own Files list shows (`routes.files. diff --git a/backend/app/downloads/models.py b/backend/app/downloads/models.py new file mode 100644 index 00000000..79661895 --- /dev/null +++ b/backend/app/downloads/models.py @@ -0,0 +1,385 @@ +"""Capture's tables: the downloaded files (`MediaAsset`), each user's jobs over them and the folders +they sit in, the shares and public links that hand them on, a user's quota, cookies and format +profiles, and where they stopped playing one. Only this unit writes them; another unit reads them +along a declared edge, or asks this unit's service.""" + +from datetime import date, datetime, timezone + +from sqlalchemy import ( + BigInteger, + Boolean, + CheckConstraint, + Date, + DateTime, + ForeignKey, + Index, + Integer, + JSON, + String, + Text, + UniqueConstraint, + func, + text, +) +from sqlalchemy.orm import Mapped, mapped_column, validates + +from app.db import Base +from app.models import DISPLAY_TEXT_MAX, TimestampMixin, UpdatedAtMixin + + +class LibraryState(Base): + """Where a user stopped in a library item (their own download, or one shared with them) — the + library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry, + a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive + both. A revoked share leaves its recipient's row behind, unread — only a job the user can still + reach is ever read or written — and it is back if the item is shared again. Rows exist only for a + position worth keeping (see `routes._common.kept_position`); no row means "from the start".""" + + __tablename__ = "library_states" + __table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + job_id: Mapped[int] = mapped_column( + ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True + ) + # int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500). + position_seconds: Mapped[int] = mapped_column(Integer, nullable=False) + progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class MediaAsset(Base, TimestampMixin): + """A physical downloaded media file — the SHARED cache layer of the download center. + + Mirrors the app's "shared catalog, private per-user state" model: one row per unique + ``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests + the same video with a format that hashes to the same `format_sig`, their `download_job` + links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is + shared; per-user edited derivatives (phase 2) are never cached here. + + `rel_path` is relative to `settings.files_root` (the mount), so the physical filename is + system-decided and bound to the source id — never renamed (the user's custom name is display + metadata on `download_job`, applied only when downloading to their own machine). The naming + fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built + uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees + the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file + stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's).""" + + __tablename__ = "media_assets" + __table_args__ = ( + UniqueConstraint( + "source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key" + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" + source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc + format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec + status: Mapped[str] = mapped_column( + String(16), default="pending", server_default="pending", index=True + ) # "pending" | "ready" | "error" + rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root + storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic + # A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the + # card / watch page / link preview can still show an image. Relative to files_root. + poster_path: Mapped[str | None] = mapped_column(Text) + size_bytes: Mapped[int | None] = mapped_column(BigInteger) + duration_s: Mapped[int | None] = mapped_column(Integer) + width: Mapped[int | None] = mapped_column(Integer) + height: Mapped[int | None] = mapped_column(Integer) + container: Mapped[str | None] = mapped_column(String(16)) + vcodec: Mapped[str | None] = mapped_column(String(32)) + acodec: Mapped[str | None] = mapped_column(String(32)) + # "video" | "audio" | "file" — the media-kind discriminator that drives which library actions + # apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is + # still pending (kind is known once the file is probed at finalize). It is the single stored home + # of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an + # earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here). + media_kind: Mapped[str | None] = mapped_column(String(16)) + nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + # Denormalized naming/metadata for the Plex tree + ad-hoc display. + title: Mapped[str | None] = mapped_column(Text) + uploader: Mapped[str | None] = mapped_column(String(255)) + # yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel + # render as a clickable link. Null for sources without one (e.g. reddit) or older rows. + uploader_url: Mapped[str | None] = mapped_column(Text) + upload_date: Mapped[date | None] = mapped_column(Date) + thumbnail_url: Mapped[str | None] = mapped_column(Text) + # yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from" + # reference shown in the Files library. Null until the worker extracts it (older/queued rows + # fall back to a URL derived from source_kind+source_ref). + source_webpage_url: Mapped[str | None] = mapped_column(Text) + error: Mapped[str | None] = mapped_column(Text) + ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + # The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload + # streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet. + sha256: Mapped[str | None] = mapped_column(String(64)) + + @validates("rel_path") + def _forget_hash(self, _key: str, rel_path: str | None) -> str | None: + # A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them. + self.sha256 = None + return rel_path + + +class DownloadProfile(Base, TimestampMixin): + """A named download preset — a reusable format profile. + + `user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …); + a user's saved custom profiles carry their `user_id`. `spec` is the format definition + (mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/ + thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache + `format_sig`.""" + + __tablename__ = "download_profiles" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) # NULL for a builtin preset + name: Mapped[str] = mapped_column(String(80)) + spec: Mapped[dict] = mapped_column(JSON) + is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class FileFolder(Base, TimestampMixin): + """A folder of the Files library (`app.downloads.folders`): a row, never a directory. + + A SOURCE folder (`source_key` set: youtube / plex / upload / site:) is made the first time a + user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder + (neither set, one per user) is where they swap files with a machine of theirs. Every file — + a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a + file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it.""" + + __tablename__ = "file_folders" + __table_args__ = ( + UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"), + Index( + "uq_file_folder_own_name", + "user_id", + func.lower(text("name")), + unique=True, + postgresql_where=text("name IS NOT NULL"), + ), + # One Hub folder per user. + Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")), + CheckConstraint( + "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)" + " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)" + " OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)", + name="ck_file_folder_kind", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind + # "site:" + a host name (DNS caps one at 253 characters). + source_key: Mapped[str | None] = mapped_column(String(260)) + name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) + retention_days: Mapped[int | None] = mapped_column(Integer) + + +def _ready_on_insert(context) -> datetime | None: + """A job inserted already done (an upload kept as it came) was ready the moment it was made; one + that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`).""" + done = context.get_current_parameters().get("status") == "done" + return datetime.now(timezone.utc) if done else None + + +class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): + """A user's request to download one source — the per-user layer over `MediaAsset`. + + The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live + `progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is + no server push channel, and the worker is a separate process anyway). `profile_snapshot` + freezes the spec at enqueue so later edits/deletion of the profile don't change this job. + `display_name` is the user's own name for the file (display + the Content-Disposition on + local download); the on-disk asset is never renamed. `asset_id` links to the shared file + (SET NULL if the asset is GC'd — the job then shows as expired). + + Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded + ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key) + so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the + asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id` + point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe.""" + + __tablename__ = "download_jobs" + __table_args__ = ( + Index("ix_download_jobs_claim", "status", "queue_pos"), + # Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`). + CheckConstraint( + "status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at" + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + asset_id: Mapped[int | None] = mapped_column( + ForeignKey("media_assets.id", ondelete="SET NULL"), index=True + ) + source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" + source_ref: Mapped[str] = mapped_column(String(512)) + # "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a + # user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or + # "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`). + job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download") + # For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted). + source_job_id: Mapped[int | None] = mapped_column( + ForeignKey("download_jobs.id", ondelete="SET NULL") + ) + source_asset_id: Mapped[int | None] = mapped_column( + ForeignKey("media_assets.id", ondelete="SET NULL") + ) + # For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip + # jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop + # in fractions of the box it was drawn in (`downloads.clip.normalize`). + edit_spec: Mapped[dict | None] = mapped_column(JSON) + profile_id: Mapped[int | None] = mapped_column( + ForeignKey("download_profiles.id", ondelete="SET NULL") + ) + profile_snapshot: Mapped[dict] = mapped_column(JSON) + display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) + # Per-user display overrides + extras (like display_name — metadata only, never the file). + # `display_uploader`/`display_uploader_url` override the asset's auto channel name/link; + # `extra_links` is an optional list of extra reference URLs shown on the card + watch page. + display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) + display_uploader_url: Mapped[str | None] = mapped_column(Text) + extra_links: Mapped[list | None] = mapped_column(JSON) + status: Mapped[str] = mapped_column( + String(16), default="queued", server_default="queued", index=True + ) # queued | running | paused | done | error | canceled + progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + speed_bps: Mapped[int | None] = mapped_column(BigInteger) + eta_s: Mapped[int | None] = mapped_column(Integer) + phase: Mapped[str | None] = mapped_column(String(32)) + error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail) + error_code: Mapped[str | None] = mapped_column( + String(24) + ) # category, see downloads.errors.classify + # The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL. + error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose + queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + # The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own + # folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether + # the user keeps it past its folder's retention, and when it goes (stamped once done; NULL = + # never). `expiry_warned` marks the one warning the GC sends before it deletes the file. + folder_id: Mapped[int | None] = mapped_column( + ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True + ) + # The origin the file came from (the source folder's key), recorded where it is placed: an edit + # traces its origin through its parent, and that parent may be deleted or expire later — the + # file must still know which source folder it may go back to. NULL only for a file placed before + # the column existed outside a source folder; `folders.source_key_for` works those out. + source_key: Mapped[str | None] = mapped_column(String(260)) + kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) + expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + # When the file became done — what the library's "Added" means: a download that failed for days + # and then went through is new to its user, so it sorts as new. `created_at` stays when it was + # asked for (the queue's order). Set again when a file is made anew (a retry, a re-download). + ready_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), default=_ready_on_insert + ) + + +class DownloadShare(Base, TimestampMixin): + """A user→user grant: `to_user` may access `from_user`'s downloaded file (private by + default). Since assets are already deduped/shared on disk, a share is just an ACL row.""" + + __tablename__ = "download_shares" + __table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),) + + id: Mapped[int] = mapped_column(primary_key=True) + job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE")) + from_user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + + +class DownloadQuota(Base, UpdatedAtMixin): + """Per-user download limits (admin-managed). A row exists only when an admin customizes a + user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the + byte cap entirely (e.g. for the admin's own account).""" + + __tablename__ = "download_quotas" + + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + max_bytes: Mapped[int] = mapped_column(BigInteger) + max_concurrent: Mapped[int] = mapped_column(Integer) + max_jobs: Mapped[int] = mapped_column(Integer) + unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + + +class DownloadLink(Base, TimestampMixin): + """A public capability link to one download, or to one of its owner's own folders — the "share + by link" surface. + + Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}` + player page (Google-Drive style); a folder link opens the folder's list there, with what is in + the folder at the time it is opened. The token itself is never stored as written: a link is + found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner + copies the link from again — absent without a key (`downloads/links.py`). Optional per-link + controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash` + (argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which + grants a *registered* user access via the in-app "Shared with me" list.""" + + __tablename__ = "download_links" + __table_args__ = ( + # A link opens exactly one thing: a file or a folder. + CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + job_id: Mapped[int | None] = mapped_column( + ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True + ) + folder_id: Mapped[int | None] = mapped_column( + ForeignKey("file_folders.id", ondelete="CASCADE"), index=True + ) + token_hash: Mapped[str] = mapped_column(String(64), unique=True) + token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged + created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + password_hash: Mapped[str | None] = mapped_column(Text) + # Bumped whenever the password changes; folded into the signed watch grant so rotating the + # password immediately invalidates any grants issued under the old one (instead of them living + # out the 6h grant TTL). + password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin): + """A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …). + + Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host + where they're logged in (e.g. a closed Facebook group they belong to), and a download of a + yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The + blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit — + the routes are write-only (host + updated_at only). See `downloads.credentials`. + + Two load-bearing security controls live around this row, NOT on it: + * **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the + user id, so the authed asset is private to that user and can never be cache-served to + another user (`downloads.credentials.cookie_salt`). + * **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches + this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=` + can't exfiltrate the user's session to a look-alike host. + + `host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the + raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch).""" + + __tablename__ = "download_credentials" + __table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + host_key: Mapped[str] = mapped_column(String(64)) + cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged diff --git a/backend/app/downloads/og.py b/backend/app/downloads/og.py index 10a0e37e..f2d4d29c 100644 --- a/backend/app/downloads/og.py +++ b/backend/app/downloads/og.py @@ -16,7 +16,7 @@ from sqlalchemy.orm import Session from app.config import settings from app.downloads import links as linksmod from app.downloads import storage -from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset +from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset _OG_BLOCK = re.compile(r".*?", re.DOTALL) diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index 307d3886..c6ae8622 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -28,7 +28,8 @@ from app.downloads.jobdb import ( ) from app.downloads.stated import StatedFailure from app.downloads.status import JobStatus -from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem +from app.models import DISPLAY_TEXT_MAX, PlexItem +from app.downloads.models import DownloadJob, MediaAsset from app.plex import lookups as plex_lookups from app.plex import paths as plex_paths from app.notifications import create_notification diff --git a/backend/app/downloads/quota.py b/backend/app/downloads/quota.py index d5832910..ef58665a 100644 --- a/backend/app/downloads/quota.py +++ b/backend/app/downloads/quota.py @@ -26,7 +26,7 @@ from sqlalchemy.orm import Session from app import sysconfig from app.downloads.status import JobKind, JobStatus -from app.models import DownloadJob, DownloadQuota, MediaAsset +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset # Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures # (error, canceled) don't count. diff --git a/backend/app/downloads/runner.py b/backend/app/downloads/runner.py index 9af835ef..bbcf3db3 100644 --- a/backend/app/downloads/runner.py +++ b/backend/app/downloads/runner.py @@ -23,7 +23,7 @@ from app.db import SessionLocal from app.downloads import edit as editmod from app.downloads import formats from app.downloads.status import JobKind -from app.models import DownloadJob +from app.downloads.models import DownloadJob log = logs.get_logger(__name__) diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index bbe5d82c..54169d71 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -21,15 +21,8 @@ from app.downloads import credentials from app.downloads import edit as editmod from app.downloads import folders, formats, mediakind, quota, storage from app.downloads.status import HOLDING, JobKind, JobStatus -from app.models import ( - DISPLAY_TEXT_MAX, - Channel, - DownloadJob, - MediaAsset, - PlexItem, - PlexShow, - Video, -) +from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, Video +from app.downloads.models import DownloadJob, MediaAsset from app.plex import lookups as plex_lookups from app.plex import paths as plex_paths diff --git a/backend/app/downloads/upload.py b/backend/app/downloads/upload.py index 49291c39..6ef5e159 100644 --- a/backend/app/downloads/upload.py +++ b/backend/app/downloads/upload.py @@ -37,7 +37,8 @@ from app.config import settings from app.downloads import edit as editmod from app.downloads import folders, mediakind, playable, probe, quota, service, storage from app.downloads.status import JobKind, JobStatus -from app.models import DownloadJob, MediaAsset, display_text +from app.models import display_text +from app.downloads.models import DownloadJob, MediaAsset def _title_from_filename(filename: str) -> str: diff --git a/backend/app/models.py b/backend/app/models.py index 9ebfbe74..000baa44 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,4 +1,4 @@ -from datetime import date, datetime, timezone +from datetime import date, datetime from sqlalchemy import ( BigInteger, @@ -18,7 +18,6 @@ from sqlalchemy import ( Text, UniqueConstraint, func, - text, ) from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR from sqlalchemy.orm import Mapped, mapped_column, relationship, validates @@ -465,27 +464,6 @@ class VideoState(Base, UpdatedAtMixin): progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) -class LibraryState(Base): - """Where a user stopped in a library item (their own download, or one shared with them) — the - library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry, - a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive - both. A revoked share leaves its recipient's row behind, unread — only a job the user can still - reach is ever read or written — and it is back if the item is shared again. Rows exist only for a - position worth keeping (see `routes._common.kept_position`); no row means "from the start".""" - - __tablename__ = "library_states" - __table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) - job_id: Mapped[int] = mapped_column( - ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True - ) - # int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500). - position_seconds: Mapped[int] = mapped_column(Integer, nullable=False) - progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - - class SearchFind(Base, TimestampMixin): """Per-user record of a video the user surfaced via their own live YouTube search. @@ -861,141 +839,6 @@ class Notification(Base): ) -class MediaAsset(Base, TimestampMixin): - """A physical downloaded media file — the SHARED cache layer of the download center. - - Mirrors the app's "shared catalog, private per-user state" model: one row per unique - ``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests - the same video with a format that hashes to the same `format_sig`, their `download_job` - links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is - shared; per-user edited derivatives (phase 2) are never cached here. - - `rel_path` is relative to `settings.files_root` (the mount), so the physical filename is - system-decided and bound to the source id — never renamed (the user's custom name is display - metadata on `download_job`, applied only when downloading to their own machine). The naming - fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built - uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees - the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file - stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's).""" - - __tablename__ = "media_assets" - __table_args__ = ( - UniqueConstraint( - "source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key" - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" - source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc - format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec - status: Mapped[str] = mapped_column( - String(16), default="pending", server_default="pending", index=True - ) # "pending" | "ready" | "error" - rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root - storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic - # A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the - # card / watch page / link preview can still show an image. Relative to files_root. - poster_path: Mapped[str | None] = mapped_column(Text) - size_bytes: Mapped[int | None] = mapped_column(BigInteger) - duration_s: Mapped[int | None] = mapped_column(Integer) - width: Mapped[int | None] = mapped_column(Integer) - height: Mapped[int | None] = mapped_column(Integer) - container: Mapped[str | None] = mapped_column(String(16)) - vcodec: Mapped[str | None] = mapped_column(String(32)) - acodec: Mapped[str | None] = mapped_column(String(32)) - # "video" | "audio" | "file" — the media-kind discriminator that drives which library actions - # apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is - # still pending (kind is known once the file is probed at finalize). It is the single stored home - # of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an - # earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here). - media_kind: Mapped[str | None] = mapped_column(String(16)) - nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - # Denormalized naming/metadata for the Plex tree + ad-hoc display. - title: Mapped[str | None] = mapped_column(Text) - uploader: Mapped[str | None] = mapped_column(String(255)) - # yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel - # render as a clickable link. Null for sources without one (e.g. reddit) or older rows. - uploader_url: Mapped[str | None] = mapped_column(Text) - upload_date: Mapped[date | None] = mapped_column(Date) - thumbnail_url: Mapped[str | None] = mapped_column(Text) - # yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from" - # reference shown in the Files library. Null until the worker extracts it (older/queued rows - # fall back to a URL derived from source_kind+source_ref). - source_webpage_url: Mapped[str | None] = mapped_column(Text) - error: Mapped[str | None] = mapped_column(Text) - ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - # The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload - # streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet. - sha256: Mapped[str | None] = mapped_column(String(64)) - - @validates("rel_path") - def _forget_hash(self, _key: str, rel_path: str | None) -> str | None: - # A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them. - self.sha256 = None - return rel_path - - -class DownloadProfile(Base, TimestampMixin): - """A named download preset — a reusable format profile. - - `user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …); - a user's saved custom profiles carry their `user_id`. `spec` is the format definition - (mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/ - thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache - `format_sig`.""" - - __tablename__ = "download_profiles" - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int | None] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), index=True - ) # NULL for a builtin preset - name: Mapped[str] = mapped_column(String(80)) - spec: Mapped[dict] = mapped_column(JSON) - is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - - -class FileFolder(Base, TimestampMixin): - """A folder of the Files library (`app.downloads.folders`): a row, never a directory. - - A SOURCE folder (`source_key` set: youtube / plex / upload / site:) is made the first time a - user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder - (neither set, one per user) is where they swap files with a machine of theirs. Every file — - a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a - file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it.""" - - __tablename__ = "file_folders" - __table_args__ = ( - UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"), - Index( - "uq_file_folder_own_name", - "user_id", - func.lower(text("name")), - unique=True, - postgresql_where=text("name IS NOT NULL"), - ), - # One Hub folder per user. - Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")), - CheckConstraint( - "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)" - " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)" - " OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)", - name="ck_file_folder_kind", - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) - kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind - # "site:" + a host name (DNS caps one at 253 characters). - source_key: Mapped[str | None] = mapped_column(String(260)) - name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) - retention_days: Mapped[int | None] = mapped_column(Integer) - - class ApiToken(Base, TimestampMixin): """A user's token for a machine to reach their Files library with (`app.apitokens`): sent as `Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its @@ -1037,208 +880,6 @@ class ApiTokenGrant(Base): can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") -def _ready_on_insert(context) -> datetime | None: - """A job inserted already done (an upload kept as it came) was ready the moment it was made; one - that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`).""" - done = context.get_current_parameters().get("status") == "done" - return datetime.now(timezone.utc) if done else None - - -class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): - """A user's request to download one source — the per-user layer over `MediaAsset`. - - The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live - `progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is - no server push channel, and the worker is a separate process anyway). `profile_snapshot` - freezes the spec at enqueue so later edits/deletion of the profile don't change this job. - `display_name` is the user's own name for the file (display + the Content-Disposition on - local download); the on-disk asset is never renamed. `asset_id` links to the shared file - (SET NULL if the asset is GC'd — the job then shows as expired). - - Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded - ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key) - so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the - asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id` - point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe.""" - - __tablename__ = "download_jobs" - __table_args__ = ( - Index("ix_download_jobs_claim", "status", "queue_pos"), - # Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`). - CheckConstraint( - "status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at" - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) - asset_id: Mapped[int | None] = mapped_column( - ForeignKey("media_assets.id", ondelete="SET NULL"), index=True - ) - source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url" - source_ref: Mapped[str] = mapped_column(String(512)) - # "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a - # user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or - # "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`). - job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download") - # For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted). - source_job_id: Mapped[int | None] = mapped_column( - ForeignKey("download_jobs.id", ondelete="SET NULL") - ) - source_asset_id: Mapped[int | None] = mapped_column( - ForeignKey("media_assets.id", ondelete="SET NULL") - ) - # For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip - # jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop - # in fractions of the box it was drawn in (`downloads.clip.normalize`). - edit_spec: Mapped[dict | None] = mapped_column(JSON) - profile_id: Mapped[int | None] = mapped_column( - ForeignKey("download_profiles.id", ondelete="SET NULL") - ) - profile_snapshot: Mapped[dict] = mapped_column(JSON) - display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) - # Per-user display overrides + extras (like display_name — metadata only, never the file). - # `display_uploader`/`display_uploader_url` override the asset's auto channel name/link; - # `extra_links` is an optional list of extra reference URLs shown on the card + watch page. - display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX)) - display_uploader_url: Mapped[str | None] = mapped_column(Text) - extra_links: Mapped[list | None] = mapped_column(JSON) - status: Mapped[str] = mapped_column( - String(16), default="queued", server_default="queued", index=True - ) # queued | running | paused | done | error | canceled - progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - speed_bps: Mapped[int | None] = mapped_column(BigInteger) - eta_s: Mapped[int | None] = mapped_column(Integer) - phase: Mapped[str | None] = mapped_column(String(32)) - error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail) - error_code: Mapped[str | None] = mapped_column( - String(24) - ) # category, see downloads.errors.classify - # The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL. - error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose - queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - # The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own - # folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether - # the user keeps it past its folder's retention, and when it goes (stamped once done; NULL = - # never). `expiry_warned` marks the one warning the GC sends before it deletes the file. - folder_id: Mapped[int | None] = mapped_column( - ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True - ) - # The origin the file came from (the source folder's key), recorded where it is placed: an edit - # traces its origin through its parent, and that parent may be deleted or expire later — the - # file must still know which source folder it may go back to. NULL only for a file placed before - # the column existed outside a source folder; `folders.source_key_for` works those out. - source_key: Mapped[str | None] = mapped_column(String(260)) - kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) - expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - # When the file became done — what the library's "Added" means: a download that failed for days - # and then went through is new to its user, so it sorts as new. `created_at` stays when it was - # asked for (the queue's order). Set again when a file is made anew (a retry, a re-download). - ready_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), default=_ready_on_insert - ) - - -class DownloadShare(Base, TimestampMixin): - """A user→user grant: `to_user` may access `from_user`'s downloaded file (private by - default). Since assets are already deduped/shared on disk, a share is just an ACL row.""" - - __tablename__ = "download_shares" - __table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),) - - id: Mapped[int] = mapped_column(primary_key=True) - job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE")) - from_user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), index=True - ) - to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) - - -class DownloadQuota(Base, UpdatedAtMixin): - """Per-user download limits (admin-managed). A row exists only when an admin customizes a - user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the - byte cap entirely (e.g. for the admin's own account).""" - - __tablename__ = "download_quotas" - - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), primary_key=True - ) - max_bytes: Mapped[int] = mapped_column(BigInteger) - max_concurrent: Mapped[int] = mapped_column(Integer) - max_jobs: Mapped[int] = mapped_column(Integer) - unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - - -class DownloadLink(Base, TimestampMixin): - """A public capability link to one download, or to one of its owner's own folders — the "share - by link" surface. - - Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}` - player page (Google-Drive style); a folder link opens the folder's list there, with what is in - the folder at the time it is opened. The token itself is never stored as written: a link is - found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner - copies the link from again — absent without a key (`downloads/links.py`). Optional per-link - controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash` - (argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which - grants a *registered* user access via the in-app "Shared with me" list.""" - - __tablename__ = "download_links" - __table_args__ = ( - # A link opens exactly one thing: a file or a folder. - CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - job_id: Mapped[int | None] = mapped_column( - ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True - ) - folder_id: Mapped[int | None] = mapped_column( - ForeignKey("file_folders.id", ondelete="CASCADE"), index=True - ) - token_hash: Mapped[str] = mapped_column(String(64), unique=True) - token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged - created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) - allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") - password_hash: Mapped[str | None] = mapped_column(Text) - # Bumped whenever the password changes; folded into the signed watch grant so rotating the - # password immediately invalidates any grants issued under the old one (instead of them living - # out the 6h grant TTL). - password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0") - - -class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin): - """A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …). - - Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host - where they're logged in (e.g. a closed Facebook group they belong to), and a download of a - yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The - blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit — - the routes are write-only (host + updated_at only). See `downloads.credentials`. - - Two load-bearing security controls live around this row, NOT on it: - * **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the - user id, so the authed asset is private to that user and can never be cache-served to - another user (`downloads.credentials.cookie_salt`). - * **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches - this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=` - can't exfiltrate the user's session to a look-alike host. - - `host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the - raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch).""" - - __tablename__ = "download_credentials" - __table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) - host_key: Mapped[str] = mapped_column(String(64)) - cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged - - # --- Plex integration (optional) ------------------------------------------------------------- # A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is # from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py index 2c897fe1..31d68988 100644 --- a/backend/app/routes/downloads.py +++ b/backend/app/routes/downloads.py @@ -33,7 +33,8 @@ from app.downloads import quota, service, storage from app.downloads import urlguard from app.downloads.folders import FolderKind from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus -from app.models import ( +from app.models import User, display_text +from app.downloads.models import ( DownloadJob, DownloadLink, FileFolder, @@ -42,8 +43,6 @@ from app.models import ( DownloadShare, LibraryState, MediaAsset, - User, - display_text, ) from app.routes._common import ( ProgressIn, diff --git a/backend/app/routes/files.py b/backend/app/routes/files.py index d777b79b..fcf385d8 100644 --- a/backend/app/routes/files.py +++ b/backend/app/routes/files.py @@ -44,16 +44,14 @@ from app.downloads import folders, preview, quota, service, storage from app.downloads import upload as uploadmod from app.downloads.status import JobStatus from app.mailtext import lang_for_user -from app.models import ( - DISPLAY_TEXT_MAX, +from app.models import DISPLAY_TEXT_MAX, User, display_text +from app.downloads.models import ( DownloadJob, DownloadLink, DownloadShare, FileFolder, LibraryState, MediaAsset, - User, - display_text, ) from app.refusal import refused from app.routes._common import capped_stream_to, not_found, owned_or_404 diff --git a/backend/app/routes/public.py b/backend/app/routes/public.py index d0d69ff8..1fc6949c 100644 --- a/backend/app/routes/public.py +++ b/backend/app/routes/public.py @@ -17,7 +17,8 @@ from app.db import get_db from app.downloads import links as linksmod from app.downloads import storage from app.downloads.status import JobStatus -from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset from app.ratelimit import RateLimiter, client_key from app.security import verify_password from app.refusal import refused diff --git a/backend/app/routes/tokens.py b/backend/app/routes/tokens.py index 40e3edc2..34426c2c 100644 --- a/backend/app/routes/tokens.py +++ b/backend/app/routes/tokens.py @@ -22,7 +22,8 @@ from app.audit import AuditAction from app.auth import current_user from app.db import get_db from app.downloads import folders -from app.models import ApiToken, ApiTokenGrant, FileFolder, User +from app.models import ApiToken, ApiTokenGrant, User +from app.downloads.models import FileFolder from app.refusal import refused from app.routes._common import owned_or_404 from app.security import hash_token diff --git a/backend/app/tables.py b/backend/app/tables.py new file mode 100644 index 00000000..5727fe9b --- /dev/null +++ b/backend/app/tables.py @@ -0,0 +1,9 @@ +"""Every table, registered on `Base.metadata`. + +Each unit declares its tables in its own `models` module, so importing `app.models` alone no longer +names the whole schema. Whatever needs all of it — the migrations, the test database, a check over +every mapper — imports this module. +""" + +import app.downloads.models # Capture's tables +import app.models # noqa: F401 — the core tables diff --git a/backend/app/worker.py b/backend/app/worker.py index b8180456..11db8327 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -46,7 +46,8 @@ from app.downloads.jobdb import ( _staging_dir, ) from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus -from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset +from app.models import DISPLAY_TEXT_MAX +from app.downloads.models import DownloadJob, MediaAsset log = logs.get_logger(__name__) _stop = threading.Event() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 32e614ae..c00ea44f 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -25,7 +25,7 @@ from fastapi.testclient import TestClient from sqlalchemy import create_engine, event, text from starlette.requests import HTTPConnection -import app.models # register every model on Base.metadata +import app.tables # every table on Base.metadata import app.state as app_state from app import apitokens, security from app.auth import current_user diff --git a/backend/tests/test_api_tokens.py b/backend/tests/test_api_tokens.py index aa7e64c9..ca70144d 100644 --- a/backend/tests/test_api_tokens.py +++ b/backend/tests/test_api_tokens.py @@ -25,16 +25,8 @@ from app.config import settings from app.db import SessionLocal, get_db from app.main import app from app.notifications import create_notification -from app.models import ( - ApiToken, - ApiTokenGrant, - AuditLog, - DownloadJob, - FileFolder, - MediaAsset, - Notification, - User, -) +from app.models import ApiToken, ApiTokenGrant, AuditLog, Notification, User +from app.downloads.models import DownloadJob, FileFolder, MediaAsset from app.ratelimit import RateLimiter from app.routes import tokens as tokens_routes from app.routes.auth import bump_session_epoch diff --git a/backend/tests/test_clip.py b/backend/tests/test_clip.py index a0af274c..6b492959 100644 --- a/backend/tests/test_clip.py +++ b/backend/tests/test_clip.py @@ -18,7 +18,8 @@ from app.downloads import errors as dlerrors from app.downloads import formats, pipeline, quota, runner, service from app.downloads import probe as probemod from app.downloads.edit import INPUT_GUARD -from app.models import Channel, DownloadJob, MediaAsset, PlexItem, PlexLibrary, PlexShow, Video +from app.models import Channel, PlexItem, PlexLibrary, PlexShow, Video +from app.downloads.models import DownloadJob, MediaAsset from app.plex import paths as plex_paths # --- the spec ----------------------------------------------------------------------------------- diff --git a/backend/tests/test_download_credentials.py b/backend/tests/test_download_credentials.py index 33314592..f5f4c2ed 100644 --- a/backend/tests/test_download_credentials.py +++ b/backend/tests/test_download_credentials.py @@ -18,7 +18,8 @@ from cryptography.fernet import Fernet from app import security from app.downloads import credentials, formats, service -from app.models import DownloadCredential, DownloadJob, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadCredential, DownloadJob, MediaAsset @pytest.fixture(autouse=True) diff --git a/backend/tests/test_download_failure_catalog.py b/backend/tests/test_download_failure_catalog.py index 10ea902d..9f221473 100644 --- a/backend/tests/test_download_failure_catalog.py +++ b/backend/tests/test_download_failure_catalog.py @@ -24,7 +24,8 @@ from app import security from app.config import settings from app.db import engine from app.downloads import credentials, formats, pipeline, service -from app.models import DownloadJob, DownloadQuota, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset from app.worker import _process_job _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e" diff --git a/backend/tests/test_download_job_locks.py b/backend/tests/test_download_job_locks.py index 0d69faf5..a053dda0 100644 --- a/backend/tests/test_download_job_locks.py +++ b/backend/tests/test_download_job_locks.py @@ -18,7 +18,8 @@ from sqlalchemy import select from app.apitokens import FilesCaller from app.db import SessionLocal from app.downloads import service -from app.models import DownloadJob, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, MediaAsset from app.routes.downloads import cancel_download, pause_download from app.routes.files import delete_file, own_job_for_update diff --git a/backend/tests/test_download_limits.py b/backend/tests/test_download_limits.py index e2d97fe9..40422dc1 100644 --- a/backend/tests/test_download_limits.py +++ b/backend/tests/test_download_limits.py @@ -13,7 +13,8 @@ its OWN session, so every test COMMITS its setup before calling it. from app.config import settings from app.downloads import errors as dlerrors from app.downloads import formats, pipeline, quota -from app.models import DownloadJob, DownloadQuota, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset from app.worker import _claim_job diff --git a/backend/tests/test_download_ref_count.py b/backend/tests/test_download_ref_count.py index 3b8a52ae..534b613b 100644 --- a/backend/tests/test_download_ref_count.py +++ b/backend/tests/test_download_ref_count.py @@ -13,7 +13,8 @@ DB lane (`db` fixture) — skipped without Postgres. from sqlalchemy import select, text from app.downloads import service -from app.models import DownloadJob, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, MediaAsset def _user(db, email="ref@example.com"): diff --git a/backend/tests/test_download_share_notice.py b/backend/tests/test_download_share_notice.py index 85c12376..3833a75d 100644 --- a/backend/tests/test_download_share_notice.py +++ b/backend/tests/test_download_share_notice.py @@ -4,7 +4,8 @@ you") — the only place the title lived, so a Hungarian reader got the English. from sqlalchemy import select -from app.models import DownloadJob, MediaAsset, Notification +from app.models import Notification +from app.downloads.models import DownloadJob, MediaAsset def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user): diff --git a/backend/tests/test_download_transitions.py b/backend/tests/test_download_transitions.py index 212e22d6..057a93a2 100644 --- a/backend/tests/test_download_transitions.py +++ b/backend/tests/test_download_transitions.py @@ -8,7 +8,8 @@ behaviour-equivalently. DB lane (`db` fixture) — skipped without Postgres. from sqlalchemy import text -from app.models import DownloadJob, MediaAsset, User +from app.models import User +from app.downloads.models import DownloadJob, MediaAsset def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None): diff --git a/backend/tests/test_downloads_finalize.py b/backend/tests/test_downloads_finalize.py index 3b793e2e..b93f5ef1 100644 --- a/backend/tests/test_downloads_finalize.py +++ b/backend/tests/test_downloads_finalize.py @@ -11,7 +11,8 @@ from sqlalchemy import text from app.downloads import service from app.downloads.status import JobStatus -from app.models import DownloadJob, User +from app.models import User +from app.downloads.models import DownloadJob def _running_job(db) -> DownloadJob: diff --git a/backend/tests/test_edit_cap.py b/backend/tests/test_edit_cap.py index fbc52efa..f88fc339 100644 --- a/backend/tests/test_edit_cap.py +++ b/backend/tests/test_edit_cap.py @@ -13,7 +13,7 @@ from app.config import settings from app.downloads import errors as dlerrors from app.downloads import pipeline, quota, service from app.downloads import upload as uploadmod -from app.models import DownloadJob, DownloadQuota, MediaAsset +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") diff --git a/backend/tests/test_failure_lines.py b/backend/tests/test_failure_lines.py index 3ade4eee..4e43e501 100644 --- a/backend/tests/test_failure_lines.py +++ b/backend/tests/test_failure_lines.py @@ -20,7 +20,7 @@ import yt_dlp from app import logs, sysconfig, worker from app.downloads import pipeline from app.downloads.stated import StatedFailure -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset from app.plex import stream as plex_stream diff --git a/backend/tests/test_file_folders.py b/backend/tests/test_file_folders.py index a3b71fc1..b13b6f79 100644 --- a/backend/tests/test_file_folders.py +++ b/backend/tests/test_file_folders.py @@ -17,7 +17,8 @@ from app.downloads import folders, formats, service from app.downloads import gc as gcmod from app.db import SessionLocal from app.downloads.status import JobStatus -from app.models import DownloadJob, FileFolder, MediaAsset, Notification +from app.models import Notification +from app.downloads.models import DownloadJob, FileFolder, MediaAsset NOW = datetime.now(timezone.utc) diff --git a/backend/tests/test_files_api.py b/backend/tests/test_files_api.py index 1f4f4395..543b8eb3 100644 --- a/backend/tests/test_files_api.py +++ b/backend/tests/test_files_api.py @@ -18,7 +18,8 @@ from app.config import settings from app.downloads import folders, service from app.downloads import upload as uploadmod from app.downloads.status import JobStatus -from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset +from app.models import AuditLog +from app.downloads.models import DownloadJob, DownloadShare, FileFolder, MediaAsset NOW = datetime.now(timezone.utc) diff --git a/backend/tests/test_files_preview.py b/backend/tests/test_files_preview.py index 904ede98..0002314a 100644 --- a/backend/tests/test_files_preview.py +++ b/backend/tests/test_files_preview.py @@ -11,7 +11,7 @@ from sqlalchemy import select from app.config import settings from app.downloads import folders, preview, service from app.downloads.status import JobStatus -from app.models import DownloadJob, DownloadShare, MediaAsset +from app.downloads.models import DownloadJob, DownloadShare, MediaAsset from app.routes.files import TRUNCATED_HEADER diff --git a/backend/tests/test_folder_links.py b/backend/tests/test_folder_links.py index 2bbcf4a5..b8fb98a8 100644 --- a/backend/tests/test_folder_links.py +++ b/backend/tests/test_folder_links.py @@ -18,7 +18,8 @@ from app.config import settings from app.db import engine from app.downloads import folders, og from app.downloads import probe as probemod -from app.models import AuditLog, DownloadJob, DownloadLink, FileFolder, MediaAsset +from app.models import AuditLog +from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset _VIDEO = { "streams": [ @@ -317,7 +318,7 @@ def test_a_password_file_link_says_it_opens_a_file(shelf, client): def test_a_file_whose_bytes_are_gone_is_not_offered(shelf, client, db): from pathlib import Path - from app.models import MediaAsset + from app.downloads.models import MediaAsset token = _link(shelf)["token"] gone = shelf["inside"][0] diff --git a/backend/tests/test_library_progress.py b/backend/tests/test_library_progress.py index 3c033f05..a2ee9b1e 100644 --- a/backend/tests/test_library_progress.py +++ b/backend/tests/test_library_progress.py @@ -9,7 +9,7 @@ cannot reach, is ever read or written. DB-backed (real Postgres). import pytest from sqlalchemy import select -from app.models import DownloadJob, DownloadShare, LibraryState, MediaAsset +from app.downloads.models import DownloadJob, DownloadShare, LibraryState, MediaAsset from app.routes._common import ProgressIn, kept_position diff --git a/backend/tests/test_link_token_at_rest.py b/backend/tests/test_link_token_at_rest.py index d57bfb0b..ceab6dd6 100644 --- a/backend/tests/test_link_token_at_rest.py +++ b/backend/tests/test_link_token_at_rest.py @@ -18,7 +18,8 @@ from app import security from app.config import settings from app.db import engine from app.downloads import probe as probemod -from app.models import AuditLog, DownloadLink +from app.models import AuditLog +from app.downloads.models import DownloadLink _VIDEO = { "streams": [ diff --git a/backend/tests/test_not_found_detail.py b/backend/tests/test_not_found_detail.py index 0351d71e..7fbb5e50 100644 --- a/backend/tests/test_not_found_detail.py +++ b/backend/tests/test_not_found_detail.py @@ -7,7 +7,7 @@ the shared one, so every other owner check kept the English. Needs the DB. """ from app.auth import require_write_scope -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user): diff --git a/backend/tests/test_odd_frame.py b/backend/tests/test_odd_frame.py index 78b1cffe..f1f9b8e5 100644 --- a/backend/tests/test_odd_frame.py +++ b/backend/tests/test_odd_frame.py @@ -17,7 +17,7 @@ import pytest from app.config import settings from app.downloads import pipeline, playable, probe, service, storage from app.downloads import upload as uploadmod -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") diff --git a/backend/tests/test_phone_safe_contract.py b/backend/tests/test_phone_safe_contract.py index 1a0f659c..3ee4b469 100644 --- a/backend/tests/test_phone_safe_contract.py +++ b/backend/tests/test_phone_safe_contract.py @@ -17,7 +17,7 @@ import pytest from app.config import settings from app.downloads import pipeline, playable, probe -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset APP = Path(__file__).resolve().parents[1] / "app" diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py index f933560d..096c46c9 100644 --- a/backend/tests/test_unit_fences.py +++ b/backend/tests/test_unit_fences.py @@ -17,7 +17,8 @@ still not reach into a unit's tables. import ast from pathlib import Path -from app.models import Base +import app.tables # noqa: F401 — every unit's tables, for the ownership check +from app.db import Base APP = Path(__file__).parent.parent / "app" ROUTES = APP / "routes" diff --git a/backend/tests/test_upload.py b/backend/tests/test_upload.py index 90f11046..0b385594 100644 --- a/backend/tests/test_upload.py +++ b/backend/tests/test_upload.py @@ -33,7 +33,8 @@ from app.downloads import mediakind, pipeline, playable, quota, service, storage from app.downloads import probe as probemod from app.downloads import upload as uploadmod from app.downloads.stated import StatedFailure -from app.models import DISPLAY_TEXT_MAX, DownloadJob, DownloadQuota, MediaAsset +from app.models import DISPLAY_TEXT_MAX +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset # --- pure: the kind invariant --------------------------------------------------------------- diff --git a/backend/tests/test_upload_conversion.py b/backend/tests/test_upload_conversion.py index aeb850b9..c7ec930f 100644 --- a/backend/tests/test_upload_conversion.py +++ b/backend/tests/test_upload_conversion.py @@ -21,7 +21,8 @@ from app.downloads import pipeline, quota, storage from app.downloads import probe as probemod from app.downloads import upload as uploadmod from app.downloads.stated import StatedFailure -from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification +from app.models import Notification +from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset from sqlalchemy import select, update from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session diff --git a/backend/tests/test_worker_dispatch.py b/backend/tests/test_worker_dispatch.py index 2593a306..3b726fba 100644 --- a/backend/tests/test_worker_dispatch.py +++ b/backend/tests/test_worker_dispatch.py @@ -12,7 +12,7 @@ from app.config import settings from app.downloads import pipeline, runner, service, storage from app.downloads import probe as probemod from app.downloads import upload as uploadmod -from app.models import DownloadJob, MediaAsset +from app.downloads.models import DownloadJob, MediaAsset @pytest.mark.parametrize( diff --git a/frontend/src/lib/api/downloads.contract.test.ts b/frontend/src/lib/api/downloads.contract.test.ts index 8110bcdb..369a5e8c 100644 --- a/frontend/src/lib/api/downloads.contract.test.ts +++ b/frontend/src/lib/api/downloads.contract.test.ts @@ -29,8 +29,10 @@ describe("display-name bound", () => { it("is the width the columns are actually declared with", () => { // The mutation guard for the check above: both constants could agree on a number neither // column uses. `String(DISPLAY_TEXT_MAX)` is how the three display columns are declared, so a - // literal creeping back into one of them fails here. - const models = src("../../../../backend/app/models.py"); + // literal creeping back into one of them fails here. `User.display_name` is a core table; the + // job's two are Capture's. + const models = + src("../../../../backend/app/models.py") + src("../../../../backend/app/downloads/models.py"); const declared = [...models.matchAll(/display_(?:name|uploader): Mapped.*?String\((.*?)\)/g)]; expect(declared).toHaveLength(3); for (const m of declared) expect(m[1]).toBe("DISPLAY_TEXT_MAX");