refactor(capture): the unit's tables move into downloads/models
The nine Capture tables (files, jobs, folders, shares, links, quota, cookies, profiles, library resume points) leave app/models.py for the unit's own module; app.tables registers every unit's tables for the migrations, the test database and the ownership check. No schema change.
This commit is contained in:
@@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config, pool
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import Base
|
from app.db import Base
|
||||||
from app.logs import configure_logging
|
from app.logs import configure_logging
|
||||||
import app.models # noqa: F401 (import so models register on Base.metadata)
|
import app.tables # noqa: F401 (import so every table registers on Base.metadata)
|
||||||
|
|
||||||
config = context.config
|
config = context.config
|
||||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from app.db import SessionLocal
|
|||||||
from app.downloads import adapters
|
from app.downloads import adapters
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import formats, service
|
from app.downloads import formats, service
|
||||||
from app.models import MediaAsset
|
from app.downloads.models import MediaAsset
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ from app import logs
|
|||||||
from app.downloads import clip as clipmod
|
from app.downloads import clip as clipmod
|
||||||
from app.downloads import formats
|
from app.downloads import formats
|
||||||
from app.downloads.adapters import _host_is
|
from app.downloads.adapters import _host_is
|
||||||
from app.models import DownloadCredential, MediaAsset
|
from app.downloads.models import DownloadCredential, MediaAsset
|
||||||
from app.security import decrypt_optional, encrypt
|
from app.security import decrypt_optional, encrypt
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app import sysconfig
|
from app import sysconfig
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DownloadJob, FileFolder, MediaAsset
|
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
class FolderKind(StrEnum):
|
class FolderKind(StrEnum):
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ from app import apitokens, logs, sysconfig
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import quota, service, storage
|
from app.downloads import quota, service, storage
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from sqlalchemy import text
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from app import security
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import storage
|
from app.downloads import storage
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||||
|
|
||||||
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
|
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
|
||||||
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
|
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
|
||||||
|
|||||||
@@ -0,0 +1,385 @@
|
|||||||
|
"""Capture's tables: the downloaded files (`MediaAsset`), each user's jobs over them and the folders
|
||||||
|
they sit in, the shares and public links that hand them on, a user's quota, cookies and format
|
||||||
|
profiles, and where they stopped playing one. Only this unit writes them; another unit reads them
|
||||||
|
along a declared edge, or asks this unit's service."""
|
||||||
|
|
||||||
|
from datetime import date, datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
BigInteger,
|
||||||
|
Boolean,
|
||||||
|
CheckConstraint,
|
||||||
|
Date,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Index,
|
||||||
|
Integer,
|
||||||
|
JSON,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
func,
|
||||||
|
text,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column, validates
|
||||||
|
|
||||||
|
from app.db import Base
|
||||||
|
from app.models import DISPLAY_TEXT_MAX, TimestampMixin, UpdatedAtMixin
|
||||||
|
|
||||||
|
|
||||||
|
class LibraryState(Base):
|
||||||
|
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
||||||
|
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
||||||
|
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
||||||
|
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
||||||
|
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
||||||
|
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
||||||
|
|
||||||
|
__tablename__ = "library_states"
|
||||||
|
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
|
job_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||||
|
)
|
||||||
|
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||||
|
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class MediaAsset(Base, TimestampMixin):
|
||||||
|
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
||||||
|
|
||||||
|
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
|
||||||
|
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
|
||||||
|
the same video with a format that hashes to the same `format_sig`, their `download_job`
|
||||||
|
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||||
|
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||||
|
|
||||||
|
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
||||||
|
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||||
|
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||||
|
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||||
|
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
||||||
|
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
||||||
|
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
||||||
|
|
||||||
|
__tablename__ = "media_assets"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||||
|
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
|
||||||
|
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), default="pending", server_default="pending", index=True
|
||||||
|
) # "pending" | "ready" | "error"
|
||||||
|
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
||||||
|
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||||
|
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||||
|
# card / watch page / link preview can still show an image. Relative to files_root.
|
||||||
|
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||||
|
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
|
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
width: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
height: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
container: Mapped[str | None] = mapped_column(String(16))
|
||||||
|
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
|
||||||
|
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
|
||||||
|
# still pending (kind is known once the file is probed at finalize). It is the single stored home
|
||||||
|
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
|
||||||
|
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||||
|
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||||
|
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||||
|
title: Mapped[str | None] = mapped_column(Text)
|
||||||
|
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||||
|
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
|
||||||
|
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
|
||||||
|
uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||||
|
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||||
|
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||||
|
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
||||||
|
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
||||||
|
# fall back to a URL derived from source_kind+source_ref).
|
||||||
|
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text)
|
||||||
|
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
|
||||||
|
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
|
||||||
|
sha256: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
|
||||||
|
@validates("rel_path")
|
||||||
|
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
|
||||||
|
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
|
||||||
|
self.sha256 = None
|
||||||
|
return rel_path
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadProfile(Base, TimestampMixin):
|
||||||
|
"""A named download preset — a reusable format profile.
|
||||||
|
|
||||||
|
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
|
||||||
|
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
|
||||||
|
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
|
||||||
|
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
|
||||||
|
`format_sig`."""
|
||||||
|
|
||||||
|
__tablename__ = "download_profiles"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||||
|
) # NULL for a builtin preset
|
||||||
|
name: Mapped[str] = mapped_column(String(80))
|
||||||
|
spec: Mapped[dict] = mapped_column(JSON)
|
||||||
|
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
|
||||||
|
|
||||||
|
class FileFolder(Base, TimestampMixin):
|
||||||
|
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
||||||
|
|
||||||
|
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
||||||
|
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
|
||||||
|
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
|
||||||
|
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
||||||
|
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
||||||
|
|
||||||
|
__tablename__ = "file_folders"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||||
|
Index(
|
||||||
|
"uq_file_folder_own_name",
|
||||||
|
"user_id",
|
||||||
|
func.lower(text("name")),
|
||||||
|
unique=True,
|
||||||
|
postgresql_where=text("name IS NOT NULL"),
|
||||||
|
),
|
||||||
|
# One Hub folder per user.
|
||||||
|
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
|
||||||
|
CheckConstraint(
|
||||||
|
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||||
|
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||||
|
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
|
||||||
|
name="ck_file_folder_kind",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
||||||
|
# "site:" + a host name (DNS caps one at 253 characters).
|
||||||
|
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||||
|
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||||
|
retention_days: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_on_insert(context) -> datetime | None:
|
||||||
|
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
|
||||||
|
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
|
||||||
|
done = context.get_current_parameters().get("status") == "done"
|
||||||
|
return datetime.now(timezone.utc) if done else None
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||||
|
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||||
|
|
||||||
|
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
||||||
|
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
|
||||||
|
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
|
||||||
|
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
||||||
|
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
||||||
|
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
||||||
|
(SET NULL if the asset is GC'd — the job then shows as expired).
|
||||||
|
|
||||||
|
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
|
||||||
|
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
|
||||||
|
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
|
||||||
|
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
|
||||||
|
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
||||||
|
|
||||||
|
__tablename__ = "download_jobs"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_download_jobs_claim", "status", "queue_pos"),
|
||||||
|
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
|
||||||
|
CheckConstraint(
|
||||||
|
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
|
asset_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
||||||
|
)
|
||||||
|
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||||
|
source_ref: Mapped[str] = mapped_column(String(512))
|
||||||
|
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
|
||||||
|
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
|
||||||
|
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
|
||||||
|
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||||
|
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||||
|
source_job_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("download_jobs.id", ondelete="SET NULL")
|
||||||
|
)
|
||||||
|
source_asset_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("media_assets.id", ondelete="SET NULL")
|
||||||
|
)
|
||||||
|
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
|
||||||
|
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
|
||||||
|
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
|
||||||
|
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
||||||
|
profile_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||||
|
)
|
||||||
|
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||||
|
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||||
|
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||||
|
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||||
|
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||||
|
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||||
|
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||||
|
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), default="queued", server_default="queued", index=True
|
||||||
|
) # queued | running | paused | done | error | canceled
|
||||||
|
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
|
eta_s: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
phase: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
||||||
|
error_code: Mapped[str | None] = mapped_column(
|
||||||
|
String(24)
|
||||||
|
) # category, see downloads.errors.classify
|
||||||
|
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||||
|
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||||
|
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
||||||
|
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
||||||
|
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
||||||
|
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
||||||
|
folder_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||||
|
)
|
||||||
|
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
||||||
|
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
||||||
|
# file must still know which source folder it may go back to. NULL only for a file placed before
|
||||||
|
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
||||||
|
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||||
|
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||||
|
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
# When the file became done — what the library's "Added" means: a download that failed for days
|
||||||
|
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
|
||||||
|
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
|
||||||
|
ready_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), default=_ready_on_insert
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadShare(Base, TimestampMixin):
|
||||||
|
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
|
||||||
|
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
||||||
|
|
||||||
|
__tablename__ = "download_shares"
|
||||||
|
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
|
||||||
|
from_user_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||||
|
)
|
||||||
|
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadQuota(Base, UpdatedAtMixin):
|
||||||
|
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
|
||||||
|
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
|
||||||
|
byte cap entirely (e.g. for the admin's own account)."""
|
||||||
|
|
||||||
|
__tablename__ = "download_quotas"
|
||||||
|
|
||||||
|
user_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||||
|
)
|
||||||
|
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
max_concurrent: Mapped[int] = mapped_column(Integer)
|
||||||
|
max_jobs: Mapped[int] = mapped_column(Integer)
|
||||||
|
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadLink(Base, TimestampMixin):
|
||||||
|
"""A public capability link to one download, or to one of its owner's own folders — the "share
|
||||||
|
by link" surface.
|
||||||
|
|
||||||
|
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
|
||||||
|
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
|
||||||
|
the folder at the time it is opened. The token itself is never stored as written: a link is
|
||||||
|
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
|
||||||
|
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
|
||||||
|
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
|
||||||
|
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
|
||||||
|
grants a *registered* user access via the in-app "Shared with me" list."""
|
||||||
|
|
||||||
|
__tablename__ = "download_links"
|
||||||
|
__table_args__ = (
|
||||||
|
# A link opens exactly one thing: a file or a folder.
|
||||||
|
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
job_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||||
|
)
|
||||||
|
folder_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||||
|
)
|
||||||
|
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||||
|
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||||
|
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
|
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
password_hash: Mapped[str | None] = mapped_column(Text)
|
||||||
|
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
|
||||||
|
# password immediately invalidates any grants issued under the old one (instead of them living
|
||||||
|
# out the 6h grant TTL).
|
||||||
|
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||||
|
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
|
||||||
|
|
||||||
|
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
|
||||||
|
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
|
||||||
|
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
|
||||||
|
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
|
||||||
|
the routes are write-only (host + updated_at only). See `downloads.credentials`.
|
||||||
|
|
||||||
|
Two load-bearing security controls live around this row, NOT on it:
|
||||||
|
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
|
||||||
|
user id, so the authed asset is private to that user and can never be cache-served to
|
||||||
|
another user (`downloads.credentials.cookie_salt`).
|
||||||
|
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
|
||||||
|
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
|
||||||
|
can't exfiltrate the user's session to a look-alike host.
|
||||||
|
|
||||||
|
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
|
||||||
|
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
|
||||||
|
|
||||||
|
__tablename__ = "download_credentials"
|
||||||
|
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
|
host_key: Mapped[str] = mapped_column(String(64))
|
||||||
|
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||||
@@ -16,7 +16,7 @@ from sqlalchemy.orm import Session
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import links as linksmod
|
from app.downloads import links as linksmod
|
||||||
from app.downloads import storage
|
from app.downloads import storage
|
||||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||||
|
|
||||||
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
|
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,8 @@ from app.downloads.jobdb import (
|
|||||||
)
|
)
|
||||||
from app.downloads.stated import StatedFailure
|
from app.downloads.stated import StatedFailure
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem
|
from app.models import DISPLAY_TEXT_MAX, PlexItem
|
||||||
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
from app.plex import lookups as plex_lookups
|
from app.plex import lookups as plex_lookups
|
||||||
from app.plex import paths as plex_paths
|
from app.plex import paths as plex_paths
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app import sysconfig
|
from app import sysconfig
|
||||||
from app.downloads.status import JobKind, JobStatus
|
from app.downloads.status import JobKind, JobStatus
|
||||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||||
|
|
||||||
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
||||||
# (error, canceled) don't count.
|
# (error, canceled) don't count.
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ from app.db import SessionLocal
|
|||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import formats
|
from app.downloads import formats
|
||||||
from app.downloads.status import JobKind
|
from app.downloads.status import JobKind
|
||||||
from app.models import DownloadJob
|
from app.downloads.models import DownloadJob
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -21,15 +21,8 @@ from app.downloads import credentials
|
|||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import folders, formats, mediakind, quota, storage
|
from app.downloads import folders, formats, mediakind, quota, storage
|
||||||
from app.downloads.status import HOLDING, JobKind, JobStatus
|
from app.downloads.status import HOLDING, JobKind, JobStatus
|
||||||
from app.models import (
|
from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, Video
|
||||||
DISPLAY_TEXT_MAX,
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
Channel,
|
|
||||||
DownloadJob,
|
|
||||||
MediaAsset,
|
|
||||||
PlexItem,
|
|
||||||
PlexShow,
|
|
||||||
Video,
|
|
||||||
)
|
|
||||||
from app.plex import lookups as plex_lookups
|
from app.plex import lookups as plex_lookups
|
||||||
from app.plex import paths as plex_paths
|
from app.plex import paths as plex_paths
|
||||||
|
|
||||||
|
|||||||
@@ -37,7 +37,8 @@ from app.config import settings
|
|||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
|
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
|
||||||
from app.downloads.status import JobKind, JobStatus
|
from app.downloads.status import JobKind, JobStatus
|
||||||
from app.models import DownloadJob, MediaAsset, display_text
|
from app.models import display_text
|
||||||
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
def _title_from_filename(filename: str) -> str:
|
def _title_from_filename(filename: str) -> str:
|
||||||
|
|||||||
+1
-360
@@ -1,4 +1,4 @@
|
|||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime
|
||||||
|
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
BigInteger,
|
BigInteger,
|
||||||
@@ -18,7 +18,6 @@ from sqlalchemy import (
|
|||||||
Text,
|
Text,
|
||||||
UniqueConstraint,
|
UniqueConstraint,
|
||||||
func,
|
func,
|
||||||
text,
|
|
||||||
)
|
)
|
||||||
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||||
@@ -465,27 +464,6 @@ class VideoState(Base, UpdatedAtMixin):
|
|||||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
|
||||||
class LibraryState(Base):
|
|
||||||
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
|
||||||
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
|
||||||
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
|
||||||
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
|
||||||
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
|
||||||
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
|
||||||
|
|
||||||
__tablename__ = "library_states"
|
|
||||||
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
||||||
job_id: Mapped[int] = mapped_column(
|
|
||||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
|
||||||
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class SearchFind(Base, TimestampMixin):
|
class SearchFind(Base, TimestampMixin):
|
||||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||||
|
|
||||||
@@ -861,141 +839,6 @@ class Notification(Base):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class MediaAsset(Base, TimestampMixin):
|
|
||||||
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
|
||||||
|
|
||||||
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
|
|
||||||
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
|
|
||||||
the same video with a format that hashes to the same `format_sig`, their `download_job`
|
|
||||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
|
||||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
|
||||||
|
|
||||||
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
|
||||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
|
||||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
|
||||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
|
||||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
|
||||||
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
|
||||||
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
|
||||||
|
|
||||||
__tablename__ = "media_assets"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint(
|
|
||||||
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
|
||||||
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
|
|
||||||
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
|
|
||||||
status: Mapped[str] = mapped_column(
|
|
||||||
String(16), default="pending", server_default="pending", index=True
|
|
||||||
) # "pending" | "ready" | "error"
|
|
||||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
|
||||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
|
||||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
|
||||||
# card / watch page / link preview can still show an image. Relative to files_root.
|
|
||||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
|
||||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
|
||||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
width: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
height: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
container: Mapped[str | None] = mapped_column(String(16))
|
|
||||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
|
|
||||||
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
|
|
||||||
# still pending (kind is known once the file is probed at finalize). It is the single stored home
|
|
||||||
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
|
|
||||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
|
||||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
|
||||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
||||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
|
||||||
title: Mapped[str | None] = mapped_column(Text)
|
|
||||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
|
||||||
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
|
|
||||||
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
|
|
||||||
uploader_url: Mapped[str | None] = mapped_column(Text)
|
|
||||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
|
||||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
|
||||||
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
|
||||||
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
|
||||||
# fall back to a URL derived from source_kind+source_ref).
|
|
||||||
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
|
||||||
error: Mapped[str | None] = mapped_column(Text)
|
|
||||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
||||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
|
|
||||||
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
|
|
||||||
sha256: Mapped[str | None] = mapped_column(String(64))
|
|
||||||
|
|
||||||
@validates("rel_path")
|
|
||||||
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
|
|
||||||
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
|
|
||||||
self.sha256 = None
|
|
||||||
return rel_path
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadProfile(Base, TimestampMixin):
|
|
||||||
"""A named download preset — a reusable format profile.
|
|
||||||
|
|
||||||
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
|
|
||||||
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
|
|
||||||
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
|
|
||||||
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
|
|
||||||
`format_sig`."""
|
|
||||||
|
|
||||||
__tablename__ = "download_profiles"
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
user_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
|
||||||
) # NULL for a builtin preset
|
|
||||||
name: Mapped[str] = mapped_column(String(80))
|
|
||||||
spec: Mapped[dict] = mapped_column(JSON)
|
|
||||||
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
||||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
||||||
|
|
||||||
|
|
||||||
class FileFolder(Base, TimestampMixin):
|
|
||||||
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
|
||||||
|
|
||||||
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
|
||||||
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
|
|
||||||
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
|
|
||||||
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
|
||||||
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
|
||||||
|
|
||||||
__tablename__ = "file_folders"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
|
||||||
Index(
|
|
||||||
"uq_file_folder_own_name",
|
|
||||||
"user_id",
|
|
||||||
func.lower(text("name")),
|
|
||||||
unique=True,
|
|
||||||
postgresql_where=text("name IS NOT NULL"),
|
|
||||||
),
|
|
||||||
# One Hub folder per user.
|
|
||||||
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
|
|
||||||
CheckConstraint(
|
|
||||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
|
||||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
|
||||||
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
|
|
||||||
name="ck_file_folder_kind",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
|
||||||
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
|
||||||
# "site:" + a host name (DNS caps one at 253 characters).
|
|
||||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
|
||||||
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
|
||||||
retention_days: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
|
|
||||||
|
|
||||||
class ApiToken(Base, TimestampMixin):
|
class ApiToken(Base, TimestampMixin):
|
||||||
"""A user's token for a machine to reach their Files library with (`app.apitokens`): sent as
|
"""A user's token for a machine to reach their Files library with (`app.apitokens`): sent as
|
||||||
`Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its
|
`Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its
|
||||||
@@ -1037,208 +880,6 @@ class ApiTokenGrant(Base):
|
|||||||
can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
|
||||||
|
|
||||||
def _ready_on_insert(context) -> datetime | None:
|
|
||||||
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
|
|
||||||
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
|
|
||||||
done = context.get_current_parameters().get("status") == "done"
|
|
||||||
return datetime.now(timezone.utc) if done else None
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
|
||||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
|
||||||
|
|
||||||
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
|
||||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
|
|
||||||
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
|
|
||||||
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
|
||||||
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
|
||||||
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
|
||||||
(SET NULL if the asset is GC'd — the job then shows as expired).
|
|
||||||
|
|
||||||
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
|
|
||||||
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
|
|
||||||
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
|
|
||||||
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
|
|
||||||
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
|
||||||
|
|
||||||
__tablename__ = "download_jobs"
|
|
||||||
__table_args__ = (
|
|
||||||
Index("ix_download_jobs_claim", "status", "queue_pos"),
|
|
||||||
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
|
|
||||||
CheckConstraint(
|
|
||||||
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
|
||||||
asset_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
|
||||||
)
|
|
||||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
|
||||||
source_ref: Mapped[str] = mapped_column(String(512))
|
|
||||||
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
|
|
||||||
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
|
|
||||||
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
|
|
||||||
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
|
||||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
|
||||||
source_job_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("download_jobs.id", ondelete="SET NULL")
|
|
||||||
)
|
|
||||||
source_asset_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("media_assets.id", ondelete="SET NULL")
|
|
||||||
)
|
|
||||||
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
|
|
||||||
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
|
|
||||||
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
|
|
||||||
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
|
||||||
profile_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
|
||||||
)
|
|
||||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
|
||||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
|
||||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
|
||||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
|
||||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
|
||||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
|
||||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
|
||||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
|
||||||
status: Mapped[str] = mapped_column(
|
|
||||||
String(16), default="queued", server_default="queued", index=True
|
|
||||||
) # queued | running | paused | done | error | canceled
|
|
||||||
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
||||||
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
|
|
||||||
eta_s: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
phase: Mapped[str | None] = mapped_column(String(32))
|
|
||||||
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
|
||||||
error_code: Mapped[str | None] = mapped_column(
|
|
||||||
String(24)
|
|
||||||
) # category, see downloads.errors.classify
|
|
||||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
|
||||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
|
||||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
||||||
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
|
||||||
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
|
||||||
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
|
||||||
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
|
||||||
folder_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
|
||||||
)
|
|
||||||
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
|
||||||
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
|
||||||
# file must still know which source folder it may go back to. NULL only for a file placed before
|
|
||||||
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
|
||||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
|
||||||
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
||||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
|
||||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
||||||
# When the file became done — what the library's "Added" means: a download that failed for days
|
|
||||||
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
|
|
||||||
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
|
|
||||||
ready_at: Mapped[datetime | None] = mapped_column(
|
|
||||||
DateTime(timezone=True), default=_ready_on_insert
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadShare(Base, TimestampMixin):
|
|
||||||
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
|
|
||||||
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
|
||||||
|
|
||||||
__tablename__ = "download_shares"
|
|
||||||
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
|
|
||||||
from_user_id: Mapped[int] = mapped_column(
|
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadQuota(Base, UpdatedAtMixin):
|
|
||||||
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
|
|
||||||
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
|
|
||||||
byte cap entirely (e.g. for the admin's own account)."""
|
|
||||||
|
|
||||||
__tablename__ = "download_quotas"
|
|
||||||
|
|
||||||
user_id: Mapped[int] = mapped_column(
|
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
|
||||||
)
|
|
||||||
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
|
||||||
max_concurrent: Mapped[int] = mapped_column(Integer)
|
|
||||||
max_jobs: Mapped[int] = mapped_column(Integer)
|
|
||||||
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadLink(Base, TimestampMixin):
|
|
||||||
"""A public capability link to one download, or to one of its owner's own folders — the "share
|
|
||||||
by link" surface.
|
|
||||||
|
|
||||||
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
|
|
||||||
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
|
|
||||||
the folder at the time it is opened. The token itself is never stored as written: a link is
|
|
||||||
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
|
|
||||||
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
|
|
||||||
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
|
|
||||||
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
|
|
||||||
grants a *registered* user access via the in-app "Shared with me" list."""
|
|
||||||
|
|
||||||
__tablename__ = "download_links"
|
|
||||||
__table_args__ = (
|
|
||||||
# A link opens exactly one thing: a file or a folder.
|
|
||||||
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
job_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
folder_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
|
||||||
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
|
||||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
||||||
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
||||||
password_hash: Mapped[str | None] = mapped_column(Text)
|
|
||||||
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
|
|
||||||
# password immediately invalidates any grants issued under the old one (instead of them living
|
|
||||||
# out the 6h grant TTL).
|
|
||||||
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
||||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
|
||||||
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
|
|
||||||
|
|
||||||
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
|
|
||||||
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
|
|
||||||
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
|
|
||||||
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
|
|
||||||
the routes are write-only (host + updated_at only). See `downloads.credentials`.
|
|
||||||
|
|
||||||
Two load-bearing security controls live around this row, NOT on it:
|
|
||||||
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
|
|
||||||
user id, so the authed asset is private to that user and can never be cache-served to
|
|
||||||
another user (`downloads.credentials.cookie_salt`).
|
|
||||||
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
|
|
||||||
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
|
|
||||||
can't exfiltrate the user's session to a look-alike host.
|
|
||||||
|
|
||||||
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
|
|
||||||
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
|
|
||||||
|
|
||||||
__tablename__ = "download_credentials"
|
|
||||||
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
|
||||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
|
||||||
host_key: Mapped[str] = mapped_column(String(64))
|
|
||||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
|
||||||
|
|
||||||
|
|
||||||
# --- Plex integration (optional) -------------------------------------------------------------
|
# --- Plex integration (optional) -------------------------------------------------------------
|
||||||
# A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is
|
# A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is
|
||||||
# from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states
|
# from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states
|
||||||
|
|||||||
@@ -33,7 +33,8 @@ from app.downloads import quota, service, storage
|
|||||||
from app.downloads import urlguard
|
from app.downloads import urlguard
|
||||||
from app.downloads.folders import FolderKind
|
from app.downloads.folders import FolderKind
|
||||||
from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus
|
from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus
|
||||||
from app.models import (
|
from app.models import User, display_text
|
||||||
|
from app.downloads.models import (
|
||||||
DownloadJob,
|
DownloadJob,
|
||||||
DownloadLink,
|
DownloadLink,
|
||||||
FileFolder,
|
FileFolder,
|
||||||
@@ -42,8 +43,6 @@ from app.models import (
|
|||||||
DownloadShare,
|
DownloadShare,
|
||||||
LibraryState,
|
LibraryState,
|
||||||
MediaAsset,
|
MediaAsset,
|
||||||
User,
|
|
||||||
display_text,
|
|
||||||
)
|
)
|
||||||
from app.routes._common import (
|
from app.routes._common import (
|
||||||
ProgressIn,
|
ProgressIn,
|
||||||
|
|||||||
@@ -44,16 +44,14 @@ from app.downloads import folders, preview, quota, service, storage
|
|||||||
from app.downloads import upload as uploadmod
|
from app.downloads import upload as uploadmod
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.mailtext import lang_for_user
|
from app.mailtext import lang_for_user
|
||||||
from app.models import (
|
from app.models import DISPLAY_TEXT_MAX, User, display_text
|
||||||
DISPLAY_TEXT_MAX,
|
from app.downloads.models import (
|
||||||
DownloadJob,
|
DownloadJob,
|
||||||
DownloadLink,
|
DownloadLink,
|
||||||
DownloadShare,
|
DownloadShare,
|
||||||
FileFolder,
|
FileFolder,
|
||||||
LibraryState,
|
LibraryState,
|
||||||
MediaAsset,
|
MediaAsset,
|
||||||
User,
|
|
||||||
display_text,
|
|
||||||
)
|
)
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
from app.routes._common import capped_stream_to, not_found, owned_or_404
|
from app.routes._common import capped_stream_to, not_found, owned_or_404
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ from app.db import get_db
|
|||||||
from app.downloads import links as linksmod
|
from app.downloads import links as linksmod
|
||||||
from app.downloads import storage
|
from app.downloads import storage
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset, User
|
from app.models import User
|
||||||
|
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||||
from app.ratelimit import RateLimiter, client_key
|
from app.ratelimit import RateLimiter, client_key
|
||||||
from app.security import verify_password
|
from app.security import verify_password
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
|
|||||||
@@ -22,7 +22,8 @@ from app.audit import AuditAction
|
|||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
from app.downloads import folders
|
from app.downloads import folders
|
||||||
from app.models import ApiToken, ApiTokenGrant, FileFolder, User
|
from app.models import ApiToken, ApiTokenGrant, User
|
||||||
|
from app.downloads.models import FileFolder
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
from app.routes._common import owned_or_404
|
from app.routes._common import owned_or_404
|
||||||
from app.security import hash_token
|
from app.security import hash_token
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
"""Every table, registered on `Base.metadata`.
|
||||||
|
|
||||||
|
Each unit declares its tables in its own `models` module, so importing `app.models` alone no longer
|
||||||
|
names the whole schema. Whatever needs all of it — the migrations, the test database, a check over
|
||||||
|
every mapper — imports this module.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import app.downloads.models # Capture's tables
|
||||||
|
import app.models # noqa: F401 — the core tables
|
||||||
@@ -46,7 +46,8 @@ from app.downloads.jobdb import (
|
|||||||
_staging_dir,
|
_staging_dir,
|
||||||
)
|
)
|
||||||
from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus
|
from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus
|
||||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
from app.models import DISPLAY_TEXT_MAX
|
||||||
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
_stop = threading.Event()
|
_stop = threading.Event()
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ from fastapi.testclient import TestClient
|
|||||||
from sqlalchemy import create_engine, event, text
|
from sqlalchemy import create_engine, event, text
|
||||||
from starlette.requests import HTTPConnection
|
from starlette.requests import HTTPConnection
|
||||||
|
|
||||||
import app.models # register every model on Base.metadata
|
import app.tables # every table on Base.metadata
|
||||||
import app.state as app_state
|
import app.state as app_state
|
||||||
from app import apitokens, security
|
from app import apitokens, security
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
|
|||||||
@@ -25,16 +25,8 @@ from app.config import settings
|
|||||||
from app.db import SessionLocal, get_db
|
from app.db import SessionLocal, get_db
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
from app.models import (
|
from app.models import ApiToken, ApiTokenGrant, AuditLog, Notification, User
|
||||||
ApiToken,
|
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||||
ApiTokenGrant,
|
|
||||||
AuditLog,
|
|
||||||
DownloadJob,
|
|
||||||
FileFolder,
|
|
||||||
MediaAsset,
|
|
||||||
Notification,
|
|
||||||
User,
|
|
||||||
)
|
|
||||||
from app.ratelimit import RateLimiter
|
from app.ratelimit import RateLimiter
|
||||||
from app.routes import tokens as tokens_routes
|
from app.routes import tokens as tokens_routes
|
||||||
from app.routes.auth import bump_session_epoch
|
from app.routes.auth import bump_session_epoch
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ from app.downloads import errors as dlerrors
|
|||||||
from app.downloads import formats, pipeline, quota, runner, service
|
from app.downloads import formats, pipeline, quota, runner, service
|
||||||
from app.downloads import probe as probemod
|
from app.downloads import probe as probemod
|
||||||
from app.downloads.edit import INPUT_GUARD
|
from app.downloads.edit import INPUT_GUARD
|
||||||
from app.models import Channel, DownloadJob, MediaAsset, PlexItem, PlexLibrary, PlexShow, Video
|
from app.models import Channel, PlexItem, PlexLibrary, PlexShow, Video
|
||||||
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
from app.plex import paths as plex_paths
|
from app.plex import paths as plex_paths
|
||||||
|
|
||||||
# --- the spec -----------------------------------------------------------------------------------
|
# --- the spec -----------------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ from cryptography.fernet import Fernet
|
|||||||
|
|
||||||
from app import security
|
from app import security
|
||||||
from app.downloads import credentials, formats, service
|
from app.downloads import credentials, formats, service
|
||||||
from app.models import DownloadCredential, DownloadJob, MediaAsset, User
|
from app.models import User
|
||||||
|
from app.downloads.models import DownloadCredential, DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ from app import security
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import engine
|
from app.db import engine
|
||||||
from app.downloads import credentials, formats, pipeline, service
|
from app.downloads import credentials, formats, pipeline, service
|
||||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
from app.models import User
|
||||||
|
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||||
from app.worker import _process_job
|
from app.worker import _process_job
|
||||||
|
|
||||||
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
|
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ from sqlalchemy import select
|
|||||||
from app.apitokens import FilesCaller
|
from app.apitokens import FilesCaller
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads import service
|
from app.downloads import service
|
||||||
from app.models import DownloadJob, MediaAsset, User
|
from app.models import User
|
||||||
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
from app.routes.downloads import cancel_download, pause_download
|
from app.routes.downloads import cancel_download, pause_download
|
||||||
from app.routes.files import delete_file, own_job_for_update
|
from app.routes.files import delete_file, own_job_for_update
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ its OWN session, so every test COMMITS its setup before calling it.
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import errors as dlerrors
|
from app.downloads import errors as dlerrors
|
||||||
from app.downloads import formats, pipeline, quota
|
from app.downloads import formats, pipeline, quota
|
||||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
from app.models import User
|
||||||
|
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||||
from app.worker import _claim_job
|
from app.worker import _claim_job
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ DB lane (`db` fixture) — skipped without Postgres.
|
|||||||
from sqlalchemy import select, text
|
from sqlalchemy import select, text
|
||||||
|
|
||||||
from app.downloads import service
|
from app.downloads import service
|
||||||
from app.models import DownloadJob, MediaAsset, User
|
from app.models import User
|
||||||
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
def _user(db, email="ref@example.com"):
|
def _user(db, email="ref@example.com"):
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ you") — the only place the title lived, so a Hungarian reader got the English.
|
|||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from app.models import DownloadJob, MediaAsset, Notification
|
from app.models import Notification
|
||||||
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user):
|
def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user):
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ behaviour-equivalently. DB lane (`db` fixture) — skipped without Postgres.
|
|||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
from app.models import DownloadJob, MediaAsset, User
|
from app.models import User
|
||||||
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None):
|
def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None):
|
||||||
|
|||||||
@@ -11,7 +11,8 @@ from sqlalchemy import text
|
|||||||
|
|
||||||
from app.downloads import service
|
from app.downloads import service
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DownloadJob, User
|
from app.models import User
|
||||||
|
from app.downloads.models import DownloadJob
|
||||||
|
|
||||||
|
|
||||||
def _running_job(db) -> DownloadJob:
|
def _running_job(db) -> DownloadJob:
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from app.config import settings
|
|||||||
from app.downloads import errors as dlerrors
|
from app.downloads import errors as dlerrors
|
||||||
from app.downloads import pipeline, quota, service
|
from app.downloads import pipeline, quota, service
|
||||||
from app.downloads import upload as uploadmod
|
from app.downloads import upload as uploadmod
|
||||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||||
|
|
||||||
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import yt_dlp
|
|||||||
from app import logs, sysconfig, worker
|
from app import logs, sysconfig, worker
|
||||||
from app.downloads import pipeline
|
from app.downloads import pipeline
|
||||||
from app.downloads.stated import StatedFailure
|
from app.downloads.stated import StatedFailure
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
from app.plex import stream as plex_stream
|
from app.plex import stream as plex_stream
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ from app.downloads import folders, formats, service
|
|||||||
from app.downloads import gc as gcmod
|
from app.downloads import gc as gcmod
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DownloadJob, FileFolder, MediaAsset, Notification
|
from app.models import Notification
|
||||||
|
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||||
|
|
||||||
NOW = datetime.now(timezone.utc)
|
NOW = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ from app.config import settings
|
|||||||
from app.downloads import folders, service
|
from app.downloads import folders, service
|
||||||
from app.downloads import upload as uploadmod
|
from app.downloads import upload as uploadmod
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset
|
from app.models import AuditLog
|
||||||
|
from app.downloads.models import DownloadJob, DownloadShare, FileFolder, MediaAsset
|
||||||
|
|
||||||
NOW = datetime.now(timezone.utc)
|
NOW = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from sqlalchemy import select
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import folders, preview, service
|
from app.downloads import folders, preview, service
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DownloadJob, DownloadShare, MediaAsset
|
from app.downloads.models import DownloadJob, DownloadShare, MediaAsset
|
||||||
from app.routes.files import TRUNCATED_HEADER
|
from app.routes.files import TRUNCATED_HEADER
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ from app.config import settings
|
|||||||
from app.db import engine
|
from app.db import engine
|
||||||
from app.downloads import folders, og
|
from app.downloads import folders, og
|
||||||
from app.downloads import probe as probemod
|
from app.downloads import probe as probemod
|
||||||
from app.models import AuditLog, DownloadJob, DownloadLink, FileFolder, MediaAsset
|
from app.models import AuditLog
|
||||||
|
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||||
|
|
||||||
_VIDEO = {
|
_VIDEO = {
|
||||||
"streams": [
|
"streams": [
|
||||||
@@ -317,7 +318,7 @@ def test_a_password_file_link_says_it_opens_a_file(shelf, client):
|
|||||||
def test_a_file_whose_bytes_are_gone_is_not_offered(shelf, client, db):
|
def test_a_file_whose_bytes_are_gone_is_not_offered(shelf, client, db):
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from app.models import MediaAsset
|
from app.downloads.models import MediaAsset
|
||||||
|
|
||||||
token = _link(shelf)["token"]
|
token = _link(shelf)["token"]
|
||||||
gone = shelf["inside"][0]
|
gone = shelf["inside"][0]
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ cannot reach, is ever read or written. DB-backed (real Postgres).
|
|||||||
import pytest
|
import pytest
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from app.models import DownloadJob, DownloadShare, LibraryState, MediaAsset
|
from app.downloads.models import DownloadJob, DownloadShare, LibraryState, MediaAsset
|
||||||
from app.routes._common import ProgressIn, kept_position
|
from app.routes._common import ProgressIn, kept_position
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ from app import security
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import engine
|
from app.db import engine
|
||||||
from app.downloads import probe as probemod
|
from app.downloads import probe as probemod
|
||||||
from app.models import AuditLog, DownloadLink
|
from app.models import AuditLog
|
||||||
|
from app.downloads.models import DownloadLink
|
||||||
|
|
||||||
_VIDEO = {
|
_VIDEO = {
|
||||||
"streams": [
|
"streams": [
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ the shared one, so every other owner check kept the English. Needs the DB.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from app.auth import require_write_scope
|
from app.auth import require_write_scope
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user):
|
def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user):
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import pytest
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import pipeline, playable, probe, service, storage
|
from app.downloads import pipeline, playable, probe, service, storage
|
||||||
from app.downloads import upload as uploadmod
|
from app.downloads import upload as uploadmod
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import pytest
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import pipeline, playable, probe
|
from app.downloads import pipeline, playable, probe
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
APP = Path(__file__).resolve().parents[1] / "app"
|
APP = Path(__file__).resolve().parents[1] / "app"
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ still not reach into a unit's tables.
|
|||||||
import ast
|
import ast
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from app.models import Base
|
import app.tables # noqa: F401 — every unit's tables, for the ownership check
|
||||||
|
from app.db import Base
|
||||||
|
|
||||||
APP = Path(__file__).parent.parent / "app"
|
APP = Path(__file__).parent.parent / "app"
|
||||||
ROUTES = APP / "routes"
|
ROUTES = APP / "routes"
|
||||||
|
|||||||
@@ -33,7 +33,8 @@ from app.downloads import mediakind, pipeline, playable, quota, service, storage
|
|||||||
from app.downloads import probe as probemod
|
from app.downloads import probe as probemod
|
||||||
from app.downloads import upload as uploadmod
|
from app.downloads import upload as uploadmod
|
||||||
from app.downloads.stated import StatedFailure
|
from app.downloads.stated import StatedFailure
|
||||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, DownloadQuota, MediaAsset
|
from app.models import DISPLAY_TEXT_MAX
|
||||||
|
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
# --- pure: the kind invariant ---------------------------------------------------------------
|
# --- pure: the kind invariant ---------------------------------------------------------------
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ from app.downloads import pipeline, quota, storage
|
|||||||
from app.downloads import probe as probemod
|
from app.downloads import probe as probemod
|
||||||
from app.downloads import upload as uploadmod
|
from app.downloads import upload as uploadmod
|
||||||
from app.downloads.stated import StatedFailure
|
from app.downloads.stated import StatedFailure
|
||||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification
|
from app.models import Notification
|
||||||
|
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||||
from sqlalchemy import select, update
|
from sqlalchemy import select, update
|
||||||
from sqlalchemy.exc import OperationalError
|
from sqlalchemy.exc import OperationalError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from app.config import settings
|
|||||||
from app.downloads import pipeline, runner, service, storage
|
from app.downloads import pipeline, runner, service, storage
|
||||||
from app.downloads import probe as probemod
|
from app.downloads import probe as probemod
|
||||||
from app.downloads import upload as uploadmod
|
from app.downloads import upload as uploadmod
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.downloads.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
|||||||
@@ -29,8 +29,10 @@ describe("display-name bound", () => {
|
|||||||
it("is the width the columns are actually declared with", () => {
|
it("is the width the columns are actually declared with", () => {
|
||||||
// The mutation guard for the check above: both constants could agree on a number neither
|
// The mutation guard for the check above: both constants could agree on a number neither
|
||||||
// column uses. `String(DISPLAY_TEXT_MAX)` is how the three display columns are declared, so a
|
// column uses. `String(DISPLAY_TEXT_MAX)` is how the three display columns are declared, so a
|
||||||
// literal creeping back into one of them fails here.
|
// literal creeping back into one of them fails here. `User.display_name` is a core table; the
|
||||||
const models = src("../../../../backend/app/models.py");
|
// job's two are Capture's.
|
||||||
|
const models =
|
||||||
|
src("../../../../backend/app/models.py") + src("../../../../backend/app/downloads/models.py");
|
||||||
const declared = [...models.matchAll(/display_(?:name|uploader): Mapped.*?String\((.*?)\)/g)];
|
const declared = [...models.matchAll(/display_(?:name|uploader): Mapped.*?String\((.*?)\)/g)];
|
||||||
expect(declared).toHaveLength(3);
|
expect(declared).toHaveLength(3);
|
||||||
for (const m of declared) expect(m[1]).toBe("DISPLAY_TEXT_MAX");
|
for (const m of declared) expect(m[1]).toBe("DISPLAY_TEXT_MAX");
|
||||||
|
|||||||
Reference in New Issue
Block a user