refactor(capture): the unit's tables move into downloads/models

The nine Capture tables (files, jobs, folders, shares, links, quota, cookies, profiles, library
resume points) leave app/models.py for the unit's own module; app.tables registers every unit's
tables for the migrations, the test database and the ownership check. No schema change.
This commit is contained in:
2026-10-18 23:45:53 +02:00
parent d66174ed16
commit 3a688fa0f5
48 changed files with 469 additions and 429 deletions
+1 -1
View File
@@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config, pool
from app.config import settings from app.config import settings
from app.db import Base from app.db import Base
from app.logs import configure_logging from app.logs import configure_logging
import app.models # noqa: F401 (import so models register on Base.metadata) import app.tables # noqa: F401 (import so every table registers on Base.metadata)
config = context.config config = context.config
config.set_main_option("sqlalchemy.url", settings.database_url) config.set_main_option("sqlalchemy.url", settings.database_url)
+1 -1
View File
@@ -22,7 +22,7 @@ from app.db import SessionLocal
from app.downloads import adapters from app.downloads import adapters
from app.downloads import edit as editmod from app.downloads import edit as editmod
from app.downloads import formats, service from app.downloads import formats, service
from app.models import MediaAsset from app.downloads.models import MediaAsset
log = logs.get_logger(__name__) log = logs.get_logger(__name__)
+1 -1
View File
@@ -33,7 +33,7 @@ from app import logs
from app.downloads import clip as clipmod from app.downloads import clip as clipmod
from app.downloads import formats from app.downloads import formats
from app.downloads.adapters import _host_is from app.downloads.adapters import _host_is
from app.models import DownloadCredential, MediaAsset from app.downloads.models import DownloadCredential, MediaAsset
from app.security import decrypt_optional, encrypt from app.security import decrypt_optional, encrypt
log = logs.get_logger(__name__) log = logs.get_logger(__name__)
+1 -1
View File
@@ -32,7 +32,7 @@ from sqlalchemy.orm import Session
from app import sysconfig from app import sysconfig
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset from app.downloads.models import DownloadJob, FileFolder, MediaAsset
class FolderKind(StrEnum): class FolderKind(StrEnum):
+1 -1
View File
@@ -31,7 +31,7 @@ from app import apitokens, logs, sysconfig
from app.config import settings from app.config import settings
from app.downloads import quota, service, storage from app.downloads import quota, service, storage
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DownloadJob, MediaAsset from app.downloads.models import DownloadJob, MediaAsset
from app.notifications import create_notification from app.notifications import create_notification
log = logs.get_logger(__name__) log = logs.get_logger(__name__)
+1 -1
View File
@@ -6,7 +6,7 @@ from sqlalchemy import text
from app.config import settings from app.config import settings
from app.db import SessionLocal from app.db import SessionLocal
from app.models import DownloadJob, MediaAsset from app.downloads.models import DownloadJob, MediaAsset
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) ------- # --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
+1 -1
View File
@@ -24,7 +24,7 @@ from app import security
from app.config import settings from app.config import settings
from app.downloads import storage from app.downloads import storage
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks _GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files. # The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
+385
View File
@@ -0,0 +1,385 @@
"""Capture's tables: the downloaded files (`MediaAsset`), each user's jobs over them and the folders
they sit in, the shares and public links that hand them on, a user's quota, cookies and format
profiles, and where they stopped playing one. Only this unit writes them; another unit reads them
along a declared edge, or asks this unit's service."""
from datetime import date, datetime, timezone
from sqlalchemy import (
BigInteger,
Boolean,
CheckConstraint,
Date,
DateTime,
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
UniqueConstraint,
func,
text,
)
from sqlalchemy.orm import Mapped, mapped_column, validates
from app.db import Base
from app.models import DISPLAY_TEXT_MAX, TimestampMixin, UpdatedAtMixin
class LibraryState(Base):
"""Where a user stopped in a library item (their own download, or one shared with them) — the
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
__tablename__ = "library_states"
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
job_id: Mapped[int] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class MediaAsset(Base, TimestampMixin):
"""A physical downloaded media file — the SHARED cache layer of the download center.
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
the same video with a format that hashes to the same `format_sig`, their `download_job`
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
shared; per-user edited derivatives (phase 2) are never cached here.
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
system-decided and bound to the source id — never renamed (the user's custom name is display
metadata on `download_job`, applied only when downloading to their own machine). The naming
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
__tablename__ = "media_assets"
__table_args__ = (
UniqueConstraint(
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
status: Mapped[str] = mapped_column(
String(16), default="pending", server_default="pending", index=True
) # "pending" | "ready" | "error"
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
# card / watch page / link preview can still show an image. Relative to files_root.
poster_path: Mapped[str | None] = mapped_column(Text)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
duration_s: Mapped[int | None] = mapped_column(Integer)
width: Mapped[int | None] = mapped_column(Integer)
height: Mapped[int | None] = mapped_column(Integer)
container: Mapped[str | None] = mapped_column(String(16))
vcodec: Mapped[str | None] = mapped_column(String(32))
acodec: Mapped[str | None] = mapped_column(String(32))
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
# still pending (kind is known once the file is probed at finalize). It is the single stored home
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
media_kind: Mapped[str | None] = mapped_column(String(16))
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
title: Mapped[str | None] = mapped_column(Text)
uploader: Mapped[str | None] = mapped_column(String(255))
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
uploader_url: Mapped[str | None] = mapped_column(Text)
upload_date: Mapped[date | None] = mapped_column(Date)
thumbnail_url: Mapped[str | None] = mapped_column(Text)
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
# fall back to a URL derived from source_kind+source_ref).
source_webpage_url: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
sha256: Mapped[str | None] = mapped_column(String(64))
@validates("rel_path")
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
self.sha256 = None
return rel_path
class DownloadProfile(Base, TimestampMixin):
"""A named download preset — a reusable format profile.
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
`format_sig`."""
__tablename__ = "download_profiles"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
) # NULL for a builtin preset
name: Mapped[str] = mapped_column(String(80))
spec: Mapped[dict] = mapped_column(JSON)
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class FileFolder(Base, TimestampMixin):
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
__tablename__ = "file_folders"
__table_args__ = (
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
Index(
"uq_file_folder_own_name",
"user_id",
func.lower(text("name")),
unique=True,
postgresql_where=text("name IS NOT NULL"),
),
# One Hub folder per user.
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
CheckConstraint(
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
name="ck_file_folder_kind",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
# "site:" + a host name (DNS caps one at 253 characters).
source_key: Mapped[str | None] = mapped_column(String(260))
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
retention_days: Mapped[int | None] = mapped_column(Integer)
def _ready_on_insert(context) -> datetime | None:
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
done = context.get_current_parameters().get("status") == "done"
return datetime.now(timezone.utc) if done else None
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
"""A user's request to download one source — the per-user layer over `MediaAsset`.
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
`display_name` is the user's own name for the file (display + the Content-Disposition on
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
(SET NULL if the asset is GC'd — the job then shows as expired).
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
__tablename__ = "download_jobs"
__table_args__ = (
Index("ix_download_jobs_claim", "status", "queue_pos"),
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
CheckConstraint(
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512))
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
source_job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="SET NULL")
)
source_asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL")
)
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
edit_spec: Mapped[dict | None] = mapped_column(JSON)
profile_id: Mapped[int | None] = mapped_column(
ForeignKey("download_profiles.id", ondelete="SET NULL")
)
profile_snapshot: Mapped[dict] = mapped_column(JSON)
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
# Per-user display overrides + extras (like display_name — metadata only, never the file).
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
display_uploader_url: Mapped[str | None] = mapped_column(Text)
extra_links: Mapped[list | None] = mapped_column(JSON)
status: Mapped[str] = mapped_column(
String(16), default="queued", server_default="queued", index=True
) # queued | running | paused | done | error | canceled
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
eta_s: Mapped[int | None] = mapped_column(Integer)
phase: Mapped[str | None] = mapped_column(String(32))
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
error_code: Mapped[str | None] = mapped_column(
String(24)
) # category, see downloads.errors.classify
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
)
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
# traces its origin through its parent, and that parent may be deleted or expire later — the
# file must still know which source folder it may go back to. NULL only for a file placed before
# the column existed outside a source folder; `folders.source_key_for` works those out.
source_key: Mapped[str | None] = mapped_column(String(260))
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# When the file became done — what the library's "Added" means: a download that failed for days
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
ready_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), default=_ready_on_insert
)
class DownloadShare(Base, TimestampMixin):
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
__tablename__ = "download_shares"
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
from_user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
class DownloadQuota(Base, UpdatedAtMixin):
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
byte cap entirely (e.g. for the admin's own account)."""
__tablename__ = "download_quotas"
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
max_bytes: Mapped[int] = mapped_column(BigInteger)
max_concurrent: Mapped[int] = mapped_column(Integer)
max_jobs: Mapped[int] = mapped_column(Integer)
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class DownloadLink(Base, TimestampMixin):
"""A public capability link to one download, or to one of its owner's own folders — the "share
by link" surface.
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
the folder at the time it is opened. The token itself is never stored as written: a link is
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
grants a *registered* user access via the in-app "Shared with me" list."""
__tablename__ = "download_links"
__table_args__ = (
# A link opens exactly one thing: a file or a folder.
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
)
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
password_hash: Mapped[str | None] = mapped_column(Text)
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
# password immediately invalidates any grants issued under the old one (instead of them living
# out the 6h grant TTL).
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
the routes are write-only (host + updated_at only). See `downloads.credentials`.
Two load-bearing security controls live around this row, NOT on it:
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
user id, so the authed asset is private to that user and can never be cache-served to
another user (`downloads.credentials.cookie_salt`).
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
can't exfiltrate the user's session to a look-alike host.
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
__tablename__ = "download_credentials"
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
host_key: Mapped[str] = mapped_column(String(64))
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
+1 -1
View File
@@ -16,7 +16,7 @@ from sqlalchemy.orm import Session
from app.config import settings from app.config import settings
from app.downloads import links as linksmod from app.downloads import links as linksmod
from app.downloads import storage from app.downloads import storage
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL) _OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
+2 -1
View File
@@ -28,7 +28,8 @@ from app.downloads.jobdb import (
) )
from app.downloads.stated import StatedFailure from app.downloads.stated import StatedFailure
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem from app.models import DISPLAY_TEXT_MAX, PlexItem
from app.downloads.models import DownloadJob, MediaAsset
from app.plex import lookups as plex_lookups from app.plex import lookups as plex_lookups
from app.plex import paths as plex_paths from app.plex import paths as plex_paths
from app.notifications import create_notification from app.notifications import create_notification
+1 -1
View File
@@ -26,7 +26,7 @@ from sqlalchemy.orm import Session
from app import sysconfig from app import sysconfig
from app.downloads.status import JobKind, JobStatus from app.downloads.status import JobKind, JobStatus
from app.models import DownloadJob, DownloadQuota, MediaAsset from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures # Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
# (error, canceled) don't count. # (error, canceled) don't count.
+1 -1
View File
@@ -23,7 +23,7 @@ from app.db import SessionLocal
from app.downloads import edit as editmod from app.downloads import edit as editmod
from app.downloads import formats from app.downloads import formats
from app.downloads.status import JobKind from app.downloads.status import JobKind
from app.models import DownloadJob from app.downloads.models import DownloadJob
log = logs.get_logger(__name__) log = logs.get_logger(__name__)
+2 -9
View File
@@ -21,15 +21,8 @@ from app.downloads import credentials
from app.downloads import edit as editmod from app.downloads import edit as editmod
from app.downloads import folders, formats, mediakind, quota, storage from app.downloads import folders, formats, mediakind, quota, storage
from app.downloads.status import HOLDING, JobKind, JobStatus from app.downloads.status import HOLDING, JobKind, JobStatus
from app.models import ( from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, Video
DISPLAY_TEXT_MAX, from app.downloads.models import DownloadJob, MediaAsset
Channel,
DownloadJob,
MediaAsset,
PlexItem,
PlexShow,
Video,
)
from app.plex import lookups as plex_lookups from app.plex import lookups as plex_lookups
from app.plex import paths as plex_paths from app.plex import paths as plex_paths
+2 -1
View File
@@ -37,7 +37,8 @@ from app.config import settings
from app.downloads import edit as editmod from app.downloads import edit as editmod
from app.downloads import folders, mediakind, playable, probe, quota, service, storage from app.downloads import folders, mediakind, playable, probe, quota, service, storage
from app.downloads.status import JobKind, JobStatus from app.downloads.status import JobKind, JobStatus
from app.models import DownloadJob, MediaAsset, display_text from app.models import display_text
from app.downloads.models import DownloadJob, MediaAsset
def _title_from_filename(filename: str) -> str: def _title_from_filename(filename: str) -> str:
+1 -360
View File
@@ -1,4 +1,4 @@
from datetime import date, datetime, timezone from datetime import date, datetime
from sqlalchemy import ( from sqlalchemy import (
BigInteger, BigInteger,
@@ -18,7 +18,6 @@ from sqlalchemy import (
Text, Text,
UniqueConstraint, UniqueConstraint,
func, func,
text,
) )
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
@@ -465,27 +464,6 @@ class VideoState(Base, UpdatedAtMixin):
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class LibraryState(Base):
"""Where a user stopped in a library item (their own download, or one shared with them) — the
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
__tablename__ = "library_states"
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
job_id: Mapped[int] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class SearchFind(Base, TimestampMixin): class SearchFind(Base, TimestampMixin):
"""Per-user record of a video the user surfaced via their own live YouTube search. """Per-user record of a video the user surfaced via their own live YouTube search.
@@ -861,141 +839,6 @@ class Notification(Base):
) )
class MediaAsset(Base, TimestampMixin):
"""A physical downloaded media file — the SHARED cache layer of the download center.
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
the same video with a format that hashes to the same `format_sig`, their `download_job`
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
shared; per-user edited derivatives (phase 2) are never cached here.
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
system-decided and bound to the source id — never renamed (the user's custom name is display
metadata on `download_job`, applied only when downloading to their own machine). The naming
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
__tablename__ = "media_assets"
__table_args__ = (
UniqueConstraint(
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
status: Mapped[str] = mapped_column(
String(16), default="pending", server_default="pending", index=True
) # "pending" | "ready" | "error"
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
# card / watch page / link preview can still show an image. Relative to files_root.
poster_path: Mapped[str | None] = mapped_column(Text)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
duration_s: Mapped[int | None] = mapped_column(Integer)
width: Mapped[int | None] = mapped_column(Integer)
height: Mapped[int | None] = mapped_column(Integer)
container: Mapped[str | None] = mapped_column(String(16))
vcodec: Mapped[str | None] = mapped_column(String(32))
acodec: Mapped[str | None] = mapped_column(String(32))
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
# still pending (kind is known once the file is probed at finalize). It is the single stored home
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
media_kind: Mapped[str | None] = mapped_column(String(16))
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
title: Mapped[str | None] = mapped_column(Text)
uploader: Mapped[str | None] = mapped_column(String(255))
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
uploader_url: Mapped[str | None] = mapped_column(Text)
upload_date: Mapped[date | None] = mapped_column(Date)
thumbnail_url: Mapped[str | None] = mapped_column(Text)
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
# fall back to a URL derived from source_kind+source_ref).
source_webpage_url: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
sha256: Mapped[str | None] = mapped_column(String(64))
@validates("rel_path")
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
self.sha256 = None
return rel_path
class DownloadProfile(Base, TimestampMixin):
"""A named download preset — a reusable format profile.
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
`format_sig`."""
__tablename__ = "download_profiles"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
) # NULL for a builtin preset
name: Mapped[str] = mapped_column(String(80))
spec: Mapped[dict] = mapped_column(JSON)
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class FileFolder(Base, TimestampMixin):
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
__tablename__ = "file_folders"
__table_args__ = (
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
Index(
"uq_file_folder_own_name",
"user_id",
func.lower(text("name")),
unique=True,
postgresql_where=text("name IS NOT NULL"),
),
# One Hub folder per user.
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
CheckConstraint(
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
name="ck_file_folder_kind",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
# "site:" + a host name (DNS caps one at 253 characters).
source_key: Mapped[str | None] = mapped_column(String(260))
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
retention_days: Mapped[int | None] = mapped_column(Integer)
class ApiToken(Base, TimestampMixin): class ApiToken(Base, TimestampMixin):
"""A user's token for a machine to reach their Files library with (`app.apitokens`): sent as """A user's token for a machine to reach their Files library with (`app.apitokens`): sent as
`Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its `Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its
@@ -1037,208 +880,6 @@ class ApiTokenGrant(Base):
can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
def _ready_on_insert(context) -> datetime | None:
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
done = context.get_current_parameters().get("status") == "done"
return datetime.now(timezone.utc) if done else None
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
"""A user's request to download one source — the per-user layer over `MediaAsset`.
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
`display_name` is the user's own name for the file (display + the Content-Disposition on
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
(SET NULL if the asset is GC'd — the job then shows as expired).
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
__tablename__ = "download_jobs"
__table_args__ = (
Index("ix_download_jobs_claim", "status", "queue_pos"),
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
CheckConstraint(
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512))
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
source_job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="SET NULL")
)
source_asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL")
)
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
edit_spec: Mapped[dict | None] = mapped_column(JSON)
profile_id: Mapped[int | None] = mapped_column(
ForeignKey("download_profiles.id", ondelete="SET NULL")
)
profile_snapshot: Mapped[dict] = mapped_column(JSON)
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
# Per-user display overrides + extras (like display_name — metadata only, never the file).
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
display_uploader_url: Mapped[str | None] = mapped_column(Text)
extra_links: Mapped[list | None] = mapped_column(JSON)
status: Mapped[str] = mapped_column(
String(16), default="queued", server_default="queued", index=True
) # queued | running | paused | done | error | canceled
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
eta_s: Mapped[int | None] = mapped_column(Integer)
phase: Mapped[str | None] = mapped_column(String(32))
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
error_code: Mapped[str | None] = mapped_column(
String(24)
) # category, see downloads.errors.classify
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
)
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
# traces its origin through its parent, and that parent may be deleted or expire later — the
# file must still know which source folder it may go back to. NULL only for a file placed before
# the column existed outside a source folder; `folders.source_key_for` works those out.
source_key: Mapped[str | None] = mapped_column(String(260))
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# When the file became done — what the library's "Added" means: a download that failed for days
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
ready_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), default=_ready_on_insert
)
class DownloadShare(Base, TimestampMixin):
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
__tablename__ = "download_shares"
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
from_user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
class DownloadQuota(Base, UpdatedAtMixin):
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
byte cap entirely (e.g. for the admin's own account)."""
__tablename__ = "download_quotas"
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
max_bytes: Mapped[int] = mapped_column(BigInteger)
max_concurrent: Mapped[int] = mapped_column(Integer)
max_jobs: Mapped[int] = mapped_column(Integer)
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class DownloadLink(Base, TimestampMixin):
"""A public capability link to one download, or to one of its owner's own folders — the "share
by link" surface.
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
the folder at the time it is opened. The token itself is never stored as written: a link is
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
grants a *registered* user access via the in-app "Shared with me" list."""
__tablename__ = "download_links"
__table_args__ = (
# A link opens exactly one thing: a file or a folder.
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
)
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
password_hash: Mapped[str | None] = mapped_column(Text)
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
# password immediately invalidates any grants issued under the old one (instead of them living
# out the 6h grant TTL).
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
the routes are write-only (host + updated_at only). See `downloads.credentials`.
Two load-bearing security controls live around this row, NOT on it:
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
user id, so the authed asset is private to that user and can never be cache-served to
another user (`downloads.credentials.cookie_salt`).
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
can't exfiltrate the user's session to a look-alike host.
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
__tablename__ = "download_credentials"
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
host_key: Mapped[str] = mapped_column(String(64))
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
# --- Plex integration (optional) ------------------------------------------------------------- # --- Plex integration (optional) -------------------------------------------------------------
# A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is # A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is
# from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states # from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states
+2 -3
View File
@@ -33,7 +33,8 @@ from app.downloads import quota, service, storage
from app.downloads import urlguard from app.downloads import urlguard
from app.downloads.folders import FolderKind from app.downloads.folders import FolderKind
from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus
from app.models import ( from app.models import User, display_text
from app.downloads.models import (
DownloadJob, DownloadJob,
DownloadLink, DownloadLink,
FileFolder, FileFolder,
@@ -42,8 +43,6 @@ from app.models import (
DownloadShare, DownloadShare,
LibraryState, LibraryState,
MediaAsset, MediaAsset,
User,
display_text,
) )
from app.routes._common import ( from app.routes._common import (
ProgressIn, ProgressIn,
+2 -4
View File
@@ -44,16 +44,14 @@ from app.downloads import folders, preview, quota, service, storage
from app.downloads import upload as uploadmod from app.downloads import upload as uploadmod
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.mailtext import lang_for_user from app.mailtext import lang_for_user
from app.models import ( from app.models import DISPLAY_TEXT_MAX, User, display_text
DISPLAY_TEXT_MAX, from app.downloads.models import (
DownloadJob, DownloadJob,
DownloadLink, DownloadLink,
DownloadShare, DownloadShare,
FileFolder, FileFolder,
LibraryState, LibraryState,
MediaAsset, MediaAsset,
User,
display_text,
) )
from app.refusal import refused from app.refusal import refused
from app.routes._common import capped_stream_to, not_found, owned_or_404 from app.routes._common import capped_stream_to, not_found, owned_or_404
+2 -1
View File
@@ -17,7 +17,8 @@ from app.db import get_db
from app.downloads import links as linksmod from app.downloads import links as linksmod
from app.downloads import storage from app.downloads import storage
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset, User from app.models import User
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
from app.ratelimit import RateLimiter, client_key from app.ratelimit import RateLimiter, client_key
from app.security import verify_password from app.security import verify_password
from app.refusal import refused from app.refusal import refused
+2 -1
View File
@@ -22,7 +22,8 @@ from app.audit import AuditAction
from app.auth import current_user from app.auth import current_user
from app.db import get_db from app.db import get_db
from app.downloads import folders from app.downloads import folders
from app.models import ApiToken, ApiTokenGrant, FileFolder, User from app.models import ApiToken, ApiTokenGrant, User
from app.downloads.models import FileFolder
from app.refusal import refused from app.refusal import refused
from app.routes._common import owned_or_404 from app.routes._common import owned_or_404
from app.security import hash_token from app.security import hash_token
+9
View File
@@ -0,0 +1,9 @@
"""Every table, registered on `Base.metadata`.
Each unit declares its tables in its own `models` module, so importing `app.models` alone no longer
names the whole schema. Whatever needs all of it — the migrations, the test database, a check over
every mapper — imports this module.
"""
import app.downloads.models # Capture's tables
import app.models # noqa: F401 — the core tables
+2 -1
View File
@@ -46,7 +46,8 @@ from app.downloads.jobdb import (
_staging_dir, _staging_dir,
) )
from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset from app.models import DISPLAY_TEXT_MAX
from app.downloads.models import DownloadJob, MediaAsset
log = logs.get_logger(__name__) log = logs.get_logger(__name__)
_stop = threading.Event() _stop = threading.Event()
+1 -1
View File
@@ -25,7 +25,7 @@ from fastapi.testclient import TestClient
from sqlalchemy import create_engine, event, text from sqlalchemy import create_engine, event, text
from starlette.requests import HTTPConnection from starlette.requests import HTTPConnection
import app.models # register every model on Base.metadata import app.tables # every table on Base.metadata
import app.state as app_state import app.state as app_state
from app import apitokens, security from app import apitokens, security
from app.auth import current_user from app.auth import current_user
+2 -10
View File
@@ -25,16 +25,8 @@ from app.config import settings
from app.db import SessionLocal, get_db from app.db import SessionLocal, get_db
from app.main import app from app.main import app
from app.notifications import create_notification from app.notifications import create_notification
from app.models import ( from app.models import ApiToken, ApiTokenGrant, AuditLog, Notification, User
ApiToken, from app.downloads.models import DownloadJob, FileFolder, MediaAsset
ApiTokenGrant,
AuditLog,
DownloadJob,
FileFolder,
MediaAsset,
Notification,
User,
)
from app.ratelimit import RateLimiter from app.ratelimit import RateLimiter
from app.routes import tokens as tokens_routes from app.routes import tokens as tokens_routes
from app.routes.auth import bump_session_epoch from app.routes.auth import bump_session_epoch
+2 -1
View File
@@ -18,7 +18,8 @@ from app.downloads import errors as dlerrors
from app.downloads import formats, pipeline, quota, runner, service from app.downloads import formats, pipeline, quota, runner, service
from app.downloads import probe as probemod from app.downloads import probe as probemod
from app.downloads.edit import INPUT_GUARD from app.downloads.edit import INPUT_GUARD
from app.models import Channel, DownloadJob, MediaAsset, PlexItem, PlexLibrary, PlexShow, Video from app.models import Channel, PlexItem, PlexLibrary, PlexShow, Video
from app.downloads.models import DownloadJob, MediaAsset
from app.plex import paths as plex_paths from app.plex import paths as plex_paths
# --- the spec ----------------------------------------------------------------------------------- # --- the spec -----------------------------------------------------------------------------------
+2 -1
View File
@@ -18,7 +18,8 @@ from cryptography.fernet import Fernet
from app import security from app import security
from app.downloads import credentials, formats, service from app.downloads import credentials, formats, service
from app.models import DownloadCredential, DownloadJob, MediaAsset, User from app.models import User
from app.downloads.models import DownloadCredential, DownloadJob, MediaAsset
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
@@ -24,7 +24,8 @@ from app import security
from app.config import settings from app.config import settings
from app.db import engine from app.db import engine
from app.downloads import credentials, formats, pipeline, service from app.downloads import credentials, formats, pipeline, service
from app.models import DownloadJob, DownloadQuota, MediaAsset, User from app.models import User
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
from app.worker import _process_job from app.worker import _process_job
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e" _REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
+2 -1
View File
@@ -18,7 +18,8 @@ from sqlalchemy import select
from app.apitokens import FilesCaller from app.apitokens import FilesCaller
from app.db import SessionLocal from app.db import SessionLocal
from app.downloads import service from app.downloads import service
from app.models import DownloadJob, MediaAsset, User from app.models import User
from app.downloads.models import DownloadJob, MediaAsset
from app.routes.downloads import cancel_download, pause_download from app.routes.downloads import cancel_download, pause_download
from app.routes.files import delete_file, own_job_for_update from app.routes.files import delete_file, own_job_for_update
+2 -1
View File
@@ -13,7 +13,8 @@ its OWN session, so every test COMMITS its setup before calling it.
from app.config import settings from app.config import settings
from app.downloads import errors as dlerrors from app.downloads import errors as dlerrors
from app.downloads import formats, pipeline, quota from app.downloads import formats, pipeline, quota
from app.models import DownloadJob, DownloadQuota, MediaAsset, User from app.models import User
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
from app.worker import _claim_job from app.worker import _claim_job
+2 -1
View File
@@ -13,7 +13,8 @@ DB lane (`db` fixture) — skipped without Postgres.
from sqlalchemy import select, text from sqlalchemy import select, text
from app.downloads import service from app.downloads import service
from app.models import DownloadJob, MediaAsset, User from app.models import User
from app.downloads.models import DownloadJob, MediaAsset
def _user(db, email="ref@example.com"): def _user(db, email="ref@example.com"):
+2 -1
View File
@@ -4,7 +4,8 @@ you") — the only place the title lived, so a Hungarian reader got the English.
from sqlalchemy import select from sqlalchemy import select
from app.models import DownloadJob, MediaAsset, Notification from app.models import Notification
from app.downloads.models import DownloadJob, MediaAsset
def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user): def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user):
+2 -1
View File
@@ -8,7 +8,8 @@ behaviour-equivalently. DB lane (`db` fixture) — skipped without Postgres.
from sqlalchemy import text from sqlalchemy import text
from app.models import DownloadJob, MediaAsset, User from app.models import User
from app.downloads.models import DownloadJob, MediaAsset
def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None): def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None):
+2 -1
View File
@@ -11,7 +11,8 @@ from sqlalchemy import text
from app.downloads import service from app.downloads import service
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DownloadJob, User from app.models import User
from app.downloads.models import DownloadJob
def _running_job(db) -> DownloadJob: def _running_job(db) -> DownloadJob:
+1 -1
View File
@@ -13,7 +13,7 @@ from app.config import settings
from app.downloads import errors as dlerrors from app.downloads import errors as dlerrors
from app.downloads import pipeline, quota, service from app.downloads import pipeline, quota, service
from app.downloads import upload as uploadmod from app.downloads import upload as uploadmod
from app.models import DownloadJob, DownloadQuota, MediaAsset from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
+1 -1
View File
@@ -20,7 +20,7 @@ import yt_dlp
from app import logs, sysconfig, worker from app import logs, sysconfig, worker
from app.downloads import pipeline from app.downloads import pipeline
from app.downloads.stated import StatedFailure from app.downloads.stated import StatedFailure
from app.models import DownloadJob, MediaAsset from app.downloads.models import DownloadJob, MediaAsset
from app.plex import stream as plex_stream from app.plex import stream as plex_stream
+2 -1
View File
@@ -17,7 +17,8 @@ from app.downloads import folders, formats, service
from app.downloads import gc as gcmod from app.downloads import gc as gcmod
from app.db import SessionLocal from app.db import SessionLocal
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset, Notification from app.models import Notification
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
NOW = datetime.now(timezone.utc) NOW = datetime.now(timezone.utc)
+2 -1
View File
@@ -18,7 +18,8 @@ from app.config import settings
from app.downloads import folders, service from app.downloads import folders, service
from app.downloads import upload as uploadmod from app.downloads import upload as uploadmod
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset from app.models import AuditLog
from app.downloads.models import DownloadJob, DownloadShare, FileFolder, MediaAsset
NOW = datetime.now(timezone.utc) NOW = datetime.now(timezone.utc)
+1 -1
View File
@@ -11,7 +11,7 @@ from sqlalchemy import select
from app.config import settings from app.config import settings
from app.downloads import folders, preview, service from app.downloads import folders, preview, service
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadShare, MediaAsset from app.downloads.models import DownloadJob, DownloadShare, MediaAsset
from app.routes.files import TRUNCATED_HEADER from app.routes.files import TRUNCATED_HEADER
+3 -2
View File
@@ -18,7 +18,8 @@ from app.config import settings
from app.db import engine from app.db import engine
from app.downloads import folders, og from app.downloads import folders, og
from app.downloads import probe as probemod from app.downloads import probe as probemod
from app.models import AuditLog, DownloadJob, DownloadLink, FileFolder, MediaAsset from app.models import AuditLog
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
_VIDEO = { _VIDEO = {
"streams": [ "streams": [
@@ -317,7 +318,7 @@ def test_a_password_file_link_says_it_opens_a_file(shelf, client):
def test_a_file_whose_bytes_are_gone_is_not_offered(shelf, client, db): def test_a_file_whose_bytes_are_gone_is_not_offered(shelf, client, db):
from pathlib import Path from pathlib import Path
from app.models import MediaAsset from app.downloads.models import MediaAsset
token = _link(shelf)["token"] token = _link(shelf)["token"]
gone = shelf["inside"][0] gone = shelf["inside"][0]
+1 -1
View File
@@ -9,7 +9,7 @@ cannot reach, is ever read or written. DB-backed (real Postgres).
import pytest import pytest
from sqlalchemy import select from sqlalchemy import select
from app.models import DownloadJob, DownloadShare, LibraryState, MediaAsset from app.downloads.models import DownloadJob, DownloadShare, LibraryState, MediaAsset
from app.routes._common import ProgressIn, kept_position from app.routes._common import ProgressIn, kept_position
+2 -1
View File
@@ -18,7 +18,8 @@ from app import security
from app.config import settings from app.config import settings
from app.db import engine from app.db import engine
from app.downloads import probe as probemod from app.downloads import probe as probemod
from app.models import AuditLog, DownloadLink from app.models import AuditLog
from app.downloads.models import DownloadLink
_VIDEO = { _VIDEO = {
"streams": [ "streams": [
+1 -1
View File
@@ -7,7 +7,7 @@ the shared one, so every other owner check kept the English. Needs the DB.
""" """
from app.auth import require_write_scope from app.auth import require_write_scope
from app.models import DownloadJob, MediaAsset from app.downloads.models import DownloadJob, MediaAsset
def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user): def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user):
+1 -1
View File
@@ -17,7 +17,7 @@ import pytest
from app.config import settings from app.config import settings
from app.downloads import pipeline, playable, probe, service, storage from app.downloads import pipeline, playable, probe, service, storage
from app.downloads import upload as uploadmod from app.downloads import upload as uploadmod
from app.models import DownloadJob, MediaAsset from app.downloads.models import DownloadJob, MediaAsset
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
+1 -1
View File
@@ -17,7 +17,7 @@ import pytest
from app.config import settings from app.config import settings
from app.downloads import pipeline, playable, probe from app.downloads import pipeline, playable, probe
from app.models import DownloadJob, MediaAsset from app.downloads.models import DownloadJob, MediaAsset
APP = Path(__file__).resolve().parents[1] / "app" APP = Path(__file__).resolve().parents[1] / "app"
+2 -1
View File
@@ -17,7 +17,8 @@ still not reach into a unit's tables.
import ast import ast
from pathlib import Path from pathlib import Path
from app.models import Base import app.tables # noqa: F401 — every unit's tables, for the ownership check
from app.db import Base
APP = Path(__file__).parent.parent / "app" APP = Path(__file__).parent.parent / "app"
ROUTES = APP / "routes" ROUTES = APP / "routes"
+2 -1
View File
@@ -33,7 +33,8 @@ from app.downloads import mediakind, pipeline, playable, quota, service, storage
from app.downloads import probe as probemod from app.downloads import probe as probemod
from app.downloads import upload as uploadmod from app.downloads import upload as uploadmod
from app.downloads.stated import StatedFailure from app.downloads.stated import StatedFailure
from app.models import DISPLAY_TEXT_MAX, DownloadJob, DownloadQuota, MediaAsset from app.models import DISPLAY_TEXT_MAX
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
# --- pure: the kind invariant --------------------------------------------------------------- # --- pure: the kind invariant ---------------------------------------------------------------
+2 -1
View File
@@ -21,7 +21,8 @@ from app.downloads import pipeline, quota, storage
from app.downloads import probe as probemod from app.downloads import probe as probemod
from app.downloads import upload as uploadmod from app.downloads import upload as uploadmod
from app.downloads.stated import StatedFailure from app.downloads.stated import StatedFailure
from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification from app.models import Notification
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
from sqlalchemy import select, update from sqlalchemy import select, update
from sqlalchemy.exc import OperationalError from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
+1 -1
View File
@@ -12,7 +12,7 @@ from app.config import settings
from app.downloads import pipeline, runner, service, storage from app.downloads import pipeline, runner, service, storage
from app.downloads import probe as probemod from app.downloads import probe as probemod
from app.downloads import upload as uploadmod from app.downloads import upload as uploadmod
from app.models import DownloadJob, MediaAsset from app.downloads.models import DownloadJob, MediaAsset
@pytest.mark.parametrize( @pytest.mark.parametrize(
@@ -29,8 +29,10 @@ describe("display-name bound", () => {
it("is the width the columns are actually declared with", () => { it("is the width the columns are actually declared with", () => {
// The mutation guard for the check above: both constants could agree on a number neither // The mutation guard for the check above: both constants could agree on a number neither
// column uses. `String(DISPLAY_TEXT_MAX)` is how the three display columns are declared, so a // column uses. `String(DISPLAY_TEXT_MAX)` is how the three display columns are declared, so a
// literal creeping back into one of them fails here. // literal creeping back into one of them fails here. `User.display_name` is a core table; the
const models = src("../../../../backend/app/models.py"); // job's two are Capture's.
const models =
src("../../../../backend/app/models.py") + src("../../../../backend/app/downloads/models.py");
const declared = [...models.matchAll(/display_(?:name|uploader): Mapped.*?String\((.*?)\)/g)]; const declared = [...models.matchAll(/display_(?:name|uploader): Mapped.*?String\((.*?)\)/g)];
expect(declared).toHaveLength(3); expect(declared).toHaveLength(3);
for (const m of declared) expect(m[1]).toBe("DISPLAY_TEXT_MAX"); for (const m of declared) expect(m[1]).toBe("DISPLAY_TEXT_MAX");