Compare commits

..
7 Commits
Author SHA1 Message Date
peter a35eece652 fix(e2e): the seed imports Reader's tables from reader/models 2026-10-19 01:33:32 +02:00
peter b703234c17 test(reader): the service tests hold each rule their mutant broke
A reorder's left-out item, a playlisted video beside one not asked about, another user's tag, and
the user's own channel link committed before the auto-tagger reads the table.
2026-10-19 01:28:38 +02:00
peter dbb4065e7b refactor(reader): the HTTP layer reads through queries and writes through the service
Feed, playlists, saved views, tags and the Home pins do no database work in their routes any more:
reads go through reader/queries (and feedquery for a feed page, its counts and facets), writes
through reader/service and discover/pins, which flush; the handlers commit. Reader is fenced below
its HTTP layer too: its owner filters go through userscope, and the catalog's jobs ask Reader
instead of joining its tables (the auto-tagger's system tags, the cleanup jobs' keep rule, the
maintenance notice). A playlist sync's catalog ingest moves to sync/videos.
2026-10-19 01:20:14 +02:00
peter e82f87acb0 refactor(reader): the unit's tables move into reader/models
VideoState, Playlist, PlaylistItem, SavedView, ComfortPin, Tag and ChannelTag leave app/models.py
for app/reader/models.py; app.tables registers them. No schema change.
2026-10-19 00:53:44 +02:00
peter e9c60f72cd refactor(capture): the HTTP layer reads through queries and writes through the service
Capture's routes (downloads, files, public watch links) and the token list no longer build SQL, call
the session or set a model's fields: reads are downloads/queries, writes the unit's service (service,
folders, links, quota), and each write flushes for the handler to commit. Whose a row is goes
through userscope (owned_by, owned_row), which owned_or_404 now uses too.

The fence holds Capture below its routes as well: it reads only Catalog's and Plex's tables, writes
no other unit's, filters owners through userscope and commits only in its jobs and the upload
ingest. A test per service use case, among them that the tray lists only the caller's files.
2026-10-19 00:19:03 +02:00
peter 3a688fa0f5 refactor(capture): the unit's tables move into downloads/models
The nine Capture tables (files, jobs, folders, shares, links, quota, cookies, profiles, library
resume points) leave app/models.py for the unit's own module; app.tables registers every unit's
tables for the migrations, the test database and the ownership check. No schema change.
2026-10-18 23:45:53 +02:00
peter d66174ed16 test(arch): a unit's HTTP layer does no database work beyond its pin
The fence is switched on unit by unit: each route file's database work (SQL built, session data
calls, model attributes set, unit tables imported) is pinned at today's count and may only shrink.
A core route may not reach into a unit's tables. Every model has an owner.
2026-10-18 23:01:12 +02:00
95 changed files with 4124 additions and 2288 deletions
+1 -1
View File
@@ -63,7 +63,7 @@ how it gets built. Concretely:
## Architecture shape (don't fight it)
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`,
`downloads/`, `reader/`, `discover/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`,
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
Frontend: `main.tsx` provider tree → `App.tsx` shell (a shared `/watch` link boots from its own
-156
View File
@@ -692,162 +692,6 @@
}
}
],
"./app/routes/downloads.py": [
{
"code": "reportOptionalMemberAccess",
"range": {
"startColumn": 17,
"endColumn": 19,
"lineCount": 1
}
},
{
"code": "reportOptionalMemberAccess",
"range": {
"startColumn": 82,
"endColumn": 84,
"lineCount": 1
}
},
{
"code": "reportOptionalMemberAccess",
"range": {
"startColumn": 87,
"endColumn": 89,
"lineCount": 1
}
},
{
"code": "reportOptionalMemberAccess",
"range": {
"startColumn": 30,
"endColumn": 32,
"lineCount": 1
}
},
{
"code": "reportOptionalMemberAccess",
"range": {
"startColumn": 49,
"endColumn": 54,
"lineCount": 1
}
}
],
"./app/routes/playlists.py": [
{
"code": "reportOptionalOperand",
"range": {
"startColumn": 8,
"endColumn": 9,
"lineCount": 5
}
},
{
"code": "reportOptionalOperand",
"range": {
"startColumn": 8,
"endColumn": 9,
"lineCount": 5
}
},
{
"code": "reportCallIssue",
"range": {
"startColumn": 13,
"endColumn": 5,
"lineCount": 7
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 8,
"endColumn": 15,
"lineCount": 5
}
},
{
"code": "reportCallIssue",
"range": {
"startColumn": 16,
"endColumn": 5,
"lineCount": 11
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 8,
"endColumn": 15,
"lineCount": 9
}
},
{
"code": "reportCallIssue",
"range": {
"startColumn": 13,
"endColumn": 5,
"lineCount": 5
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 8,
"endColumn": 15,
"lineCount": 3
}
},
{
"code": "reportCallIssue",
"range": {
"startColumn": 12,
"endColumn": 31,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 23,
"endColumn": 27,
"lineCount": 1
}
},
{
"code": "reportCallIssue",
"range": {
"startColumn": 12,
"endColumn": 34,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 26,
"endColumn": 30,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 12,
"endColumn": 28,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 23,
"endColumn": 27,
"lineCount": 1
}
}
],
"./app/routes/plex/browse.py": [
{
"code": "reportArgumentType",
+1 -1
View File
@@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config, pool
from app.config import settings
from app.db import Base
from app.logs import configure_logging
import app.models # noqa: F401 (import so models register on Base.metadata)
import app.tables # noqa: F401 (import so every table registers on Base.metadata)
config = context.config
config.set_main_option("sqlalchemy.url", settings.database_url)
+2 -1
View File
@@ -22,7 +22,8 @@ to the client, which has both the native payload and the translations.
"""
from app.discover.budget import remaining_seconds
from app.models import Channel, Playlist
from app.models import Channel
from app.reader.models import Playlist
# Plex artwork is a poster for the things you browse and a landscape still for the things you play.
_POSTER_KINDS = ("movie", "show")
+34 -5
View File
@@ -22,10 +22,12 @@ from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.discover import cards, queries
from app.models import Channel, ComfortPin, PlexItem, PlexShow, Playlist, User, Video
from app.models import Channel, PlexItem, PlexShow, User, Video
from app.reader.models import ComfortPin, Playlist
from app.plex import rollup
from app.plex.cards import SHOW_NEW, leaf_card, show_card
from app.plex.lookups import by_rating_key
from app.userscope import owned_by, owned_row
# kind -> the ComfortPin column that holds it. The client's vocabulary and the schema's, in one place.
PIN_COLUMNS: dict[str, str] = {
@@ -92,7 +94,7 @@ def reorder(db: Session, user: User, ids: list[int]) -> None:
# old slots — and one global lock order means two reorders cannot deadlock.
locked = db.execute(
select(ComfortPin)
.where(ComfortPin.user_id == user.id)
.where(owned_by(ComfortPin.user_id, user.id))
.order_by(ComfortPin.id)
.with_for_update()
.execution_options(populate_existing=True)
@@ -117,7 +119,7 @@ def list_pins(db: Session, user: User) -> list[ComfortPin]:
return list(
db.execute(
select(ComfortPin)
.where(ComfortPin.user_id == user.id)
.where(owned_by(ComfortPin.user_id, user.id))
.order_by(ComfortPin.position.asc(), ComfortPin.id.asc())
)
.scalars()
@@ -132,17 +134,44 @@ def next_position(db: Session, user: User) -> int:
deletes the row, and every FK cascade silently removes a pin when the thing it points at goes
away — so past the first hole a count under-counts and would drop the new pin into the MIDDLE of
the shelf, or onto a position another pin already holds (nothing in the schema forbids that).
Same idiom as `routes/playlists._next_playlist_position`."""
Same idiom as a new playlist's place in the rail (`reader.service`)."""
return (
db.scalar(
select(func.coalesce(func.max(ComfortPin.position), -1)).where(
ComfortPin.user_id == user.id
owned_by(ComfortPin.user_id, user.id)
)
)
+ 1
)
def own_pin(db: Session, user: User, pin_id: int) -> ComfortPin | None:
return owned_row(db, ComfortPin, pin_id, user)
def pin_of(db: Session, user: User, column: str, value: object) -> ComfortPin | None:
"""The user's pin of this thing (`column`/`value` from `resolve_ref`), if it is on the shelf."""
return db.execute(
select(ComfortPin).where(
owned_by(ComfortPin.user_id, user), getattr(ComfortPin, column) == value
)
).scalar_one_or_none()
def add(db: Session, user: User, column: str, value: object) -> ComfortPin:
"""A new pin at the end of the shelf. A second pin of the same thing (another tab, at the same
moment) raises `IntegrityError` at the flush: the unique constraint is the guard."""
pin = ComfortPin(user_id=user.id, position=next_position(db, user), **{column: value})
db.add(pin)
db.flush()
return pin
def remove(db: Session, pin: ComfortPin) -> None:
db.delete(pin)
db.flush()
@dataclass
class _Batch:
"""What the shelf resolves for every pinned channel and playlist in one go."""
+10 -13
View File
@@ -29,19 +29,12 @@ from app.feedquery import (
serialize,
sort_keys,
)
from app.models import (
Channel,
PlaylistItem,
PlexItem,
PlexShow,
PlexState,
User,
Video,
VideoState,
)
from app.models import Channel, PlexItem, PlexShow, PlexState, User, Video
from app.reader.models import PlaylistItem, VideoState
from app.plex import rollup
from app.plex.cards import display_title, leaf_card, movie_card, show_card
from app.plex.query import apply_show_state_filter, content_libs
from app.userscope import owned_by
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
# (`routes/plex/discover.py` imports it): the two surfaces should feel like the same app, and a rail
@@ -209,7 +202,7 @@ def _unseen_movie_conds(uid: int):
.select_from(PlexState)
.where(
PlexState.item_id == PlexItem.id,
PlexState.user_id == uid,
owned_by(PlexState.user_id, uid),
or_(
PlexState.status.in_(("watched", "hidden")),
# Started counts as "not a surprise" too — see the note in `yt_surprise`.
@@ -237,7 +230,9 @@ def plex_in_progress(
if movie_libs:
mq = (
select(PlexItem, PlexState)
.join(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
.join(
PlexState, and_(PlexState.item_id == PlexItem.id, owned_by(PlexState.user_id, uid))
)
.where(
PlexItem.kind == "movie",
PlexItem.library_id.in_(movie_libs),
@@ -331,7 +326,9 @@ def next_episode(db: Session, uid: int, show_id: int, *, lang: str) -> dict | No
)
base = (
select(PlexItem, PlexState)
.outerjoin(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
.outerjoin(
PlexState, and_(PlexState.item_id == PlexItem.id, owned_by(PlexState.user_id, uid))
)
.where(PlexItem.show_id == show_id, PlexItem.kind == "episode")
.limit(1)
)
+1 -1
View File
@@ -22,7 +22,7 @@ from app.db import SessionLocal
from app.downloads import adapters
from app.downloads import edit as editmod
from app.downloads import formats, service
from app.models import MediaAsset
from app.downloads.models import MediaAsset
log = logs.get_logger(__name__)
+4 -3
View File
@@ -33,8 +33,9 @@ from app import logs
from app.downloads import clip as clipmod
from app.downloads import formats
from app.downloads.adapters import _host_is
from app.models import DownloadCredential, MediaAsset
from app.downloads.models import DownloadCredential, MediaAsset
from app.security import decrypt_optional, encrypt
from app.userscope import owned_by
log = logs.get_logger(__name__)
@@ -157,7 +158,7 @@ def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None:
return db.execute(
select(DownloadCredential).where(
DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key
owned_by(DownloadCredential.user_id, user_id), DownloadCredential.host_key == host_key
)
).scalar_one_or_none()
@@ -214,7 +215,7 @@ def status_list(db: Session, user_id: int) -> list[dict]:
have = {
c.host_key: c
for c in db.execute(
select(DownloadCredential).where(DownloadCredential.user_id == user_id)
select(DownloadCredential).where(owned_by(DownloadCredential.user_id, user_id))
).scalars()
}
out = []
+31 -3
View File
@@ -32,7 +32,8 @@ from sqlalchemy.orm import Session
from app import sysconfig
from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
from app.userscope import owned_by
class FolderKind(StrEnum):
@@ -106,7 +107,9 @@ def source_key_for(db: Session, job: DownloadJob) -> str:
def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
"""The user's source folder for `key`, made on first use. A race on the unique (user, key) is
settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
stmt = select(FileFolder).where(
owned_by(FileFolder.user_id, user_id), FileFolder.source_key == key
)
folder = db.execute(stmt).scalar_one_or_none()
if folder is not None:
return folder
@@ -131,7 +134,7 @@ def hub_folder(db: Session, user_id: int) -> FileFolder:
"""The user's Hub folder, made on first use; a race on its one-per-user index is settled like
`source_folder`'s."""
stmt = select(FileFolder).where(
FileFolder.user_id == user_id, FileFolder.kind == FolderKind.HUB
owned_by(FileFolder.user_id, user_id), FileFolder.kind == FolderKind.HUB
)
folder = db.execute(stmt).scalar_one_or_none()
if folder is not None:
@@ -310,3 +313,28 @@ def delete_own(db: Session, folder: FileFolder) -> int:
db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete
db.delete(folder)
return len(jobs)
def create_own(db: Session, user_id: int, name: str) -> FileFolder:
"""A new folder of the user's own; FolderError("file_folder_name_taken") when they already have
one by that name (the unique (user, lower(name)))."""
folder = FileFolder(user_id=user_id, kind=FolderKind.OWN, name=name)
try:
with db.begin_nested():
db.add(folder)
db.flush()
except IntegrityError as e:
raise FolderError("file_folder_name_taken") from e
return folder
def rename(db: Session, folder: FileFolder, name: str) -> None:
"""Rename one of the user's own folders (a source folder's name is its origin's)."""
if folder.kind != FolderKind.OWN:
raise FolderError("file_folder_not_own")
folder.name = name
try:
with db.begin_nested():
db.flush()
except IntegrityError as e:
raise FolderError("file_folder_name_taken") from e
+1 -1
View File
@@ -31,7 +31,7 @@ from app import apitokens, logs, sysconfig
from app.config import settings
from app.downloads import quota, service, storage
from app.downloads.status import JobStatus
from app.models import DownloadJob, MediaAsset
from app.downloads.models import DownloadJob, MediaAsset
from app.notifications import create_notification
log = logs.get_logger(__name__)
+1 -1
View File
@@ -6,7 +6,7 @@ from sqlalchemy import text
from app.config import settings
from app.db import SessionLocal
from app.models import DownloadJob, MediaAsset
from app.downloads.models import DownloadJob, MediaAsset
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
+124 -6
View File
@@ -15,20 +15,24 @@ grant appended to the file URL, so the raw password never rides in the media URL
import hashlib
import hmac
import secrets
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy import update as sql_update
from sqlalchemy.orm import Session
from app import security
from app import audit, security
from app.config import settings
from app.downloads import storage
from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
from app.models import User
from app.utils import iso
from app.userscope import owned_by
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
# MAX_LIST`; a test holds the two equal — this layer may not import that one).
# The most files a folder link lists: as many as the owner's own Files list shows
# (`queries.MAX_LIST`; a test holds the two equal — `queries` imports this module).
FOLDER_MAX = 500
@@ -59,7 +63,7 @@ def in_folder(folder: FileFolder) -> tuple:
moved out stops opening through the link at its next request; one moved in starts."""
return (
DownloadJob.folder_id == folder.id,
DownloadJob.user_id == folder.user_id,
owned_by(DownloadJob.user_id, folder.user_id),
DownloadJob.status == JobStatus.DONE,
)
@@ -170,3 +174,117 @@ def public_meta(
"file_url": file_url,
"poster_url": poster_url,
}
# --- the owner's writes --------------------------------------------------------------------
# Each flushes and records its audit row; the handler commits.
def expiry(days: int | None) -> datetime | None:
"""None/0/negative → no expiry; otherwise N days out, capped at ~10 years."""
if not days or days <= 0:
return None
return datetime.now(timezone.utc) + timedelta(days=min(days, 3650))
def audit_view(link: DownloadLink) -> dict:
"""A link's settings as the audit records them: whether it has a password, never which."""
return {
"allow_download": link.allow_download,
"has_password": link.password_hash is not None,
"expires_at": iso(link.expires_at),
}
def create(
db: Session,
user: User,
*,
password: str,
allow_download: bool,
expires_days: int | None,
**target: int,
) -> tuple[DownloadLink, str]:
"""A link to `target` (`job_id=` or `folder_id=`, already checked to be the user's), and the
token it is opened by — which is in hand only now."""
token = new_token()
link = DownloadLink(
**target,
created_by=user.id,
allow_download=allow_download,
password_hash=security.hash_password(password) if password else None,
expires_at=expiry(expires_days),
)
seal(link, token)
db.add(link)
db.flush()
audit.record(
db,
user.id,
audit.AuditAction.DOWNLOAD_LINK_CREATE,
target_type="download_link",
target_id=link.id,
after={**target, **audit_view(link)},
)
return link, token
def update(
db: Session,
user: User,
link: DownloadLink,
*,
allow_download: bool | None = None,
expires_days: int | None = None,
password: str | None = None,
sent: frozenset[str] = frozenset(),
) -> None:
"""Change what the caller sent (`sent` names the fields, so `expires_days=None` can mean "never
expires"). Setting, clearing or changing the password bumps its version, so any grant issued
under the previous one stops working now rather than living out its TTL."""
before = audit_view(link)
if "allow_download" in sent:
link.allow_download = bool(allow_download)
if "expires_days" in sent:
link.expires_at = expiry(expires_days)
if "password" in sent:
pw = (password or "").strip()
link.password_hash = security.hash_password(pw) if pw else None
link.password_version += 1
after = audit_view(link)
if after != before or "password" in sent:
audit.record(
db,
user.id,
audit.AuditAction.DOWNLOAD_LINK_UPDATE,
target_type="download_link",
target_id=link.id,
before=before,
after=after,
)
db.flush()
def revoke(db: Session, user: User, link: DownloadLink) -> None:
link_id = link.id
db.delete(link)
audit.record(
db,
user.id,
audit.AuditAction.DOWNLOAD_LINK_REVOKE,
target_type="download_link",
target_id=link_id,
after={"job_id": link.job_id} if link.job_id is not None else {"folder_id": link.folder_id},
)
db.flush()
def count_view(db: Session, link: DownloadLink) -> None:
"""One more opening of the link. In SQL, not `+= 1` on the loaded row: two opens at once would
both write the same count."""
db.execute(
sql_update(DownloadLink)
.where(DownloadLink.id == link.id)
.values(view_count=DownloadLink.view_count + 1)
.execution_options(synchronize_session=False)
)
+385
View File
@@ -0,0 +1,385 @@
"""Capture's tables: the downloaded files (`MediaAsset`), each user's jobs over them and the folders
they sit in, the shares and public links that hand them on, a user's quota, cookies and format
profiles, and where they stopped playing one. Only this unit writes them; another unit reads them
along a declared edge, or asks this unit's service."""
from datetime import date, datetime, timezone
from sqlalchemy import (
BigInteger,
Boolean,
CheckConstraint,
Date,
DateTime,
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
UniqueConstraint,
func,
text,
)
from sqlalchemy.orm import Mapped, mapped_column, validates
from app.db import Base
from app.models import DISPLAY_TEXT_MAX, TimestampMixin, UpdatedAtMixin
class LibraryState(Base):
"""Where a user stopped in a library item (their own download, or one shared with them) — the
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
__tablename__ = "library_states"
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
job_id: Mapped[int] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class MediaAsset(Base, TimestampMixin):
"""A physical downloaded media file — the SHARED cache layer of the download center.
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
the same video with a format that hashes to the same `format_sig`, their `download_job`
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
shared; per-user edited derivatives (phase 2) are never cached here.
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
system-decided and bound to the source id — never renamed (the user's custom name is display
metadata on `download_job`, applied only when downloading to their own machine). The naming
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
__tablename__ = "media_assets"
__table_args__ = (
UniqueConstraint(
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
status: Mapped[str] = mapped_column(
String(16), default="pending", server_default="pending", index=True
) # "pending" | "ready" | "error"
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
# card / watch page / link preview can still show an image. Relative to files_root.
poster_path: Mapped[str | None] = mapped_column(Text)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
duration_s: Mapped[int | None] = mapped_column(Integer)
width: Mapped[int | None] = mapped_column(Integer)
height: Mapped[int | None] = mapped_column(Integer)
container: Mapped[str | None] = mapped_column(String(16))
vcodec: Mapped[str | None] = mapped_column(String(32))
acodec: Mapped[str | None] = mapped_column(String(32))
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
# still pending (kind is known once the file is probed at finalize). It is the single stored home
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
media_kind: Mapped[str | None] = mapped_column(String(16))
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
title: Mapped[str | None] = mapped_column(Text)
uploader: Mapped[str | None] = mapped_column(String(255))
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
uploader_url: Mapped[str | None] = mapped_column(Text)
upload_date: Mapped[date | None] = mapped_column(Date)
thumbnail_url: Mapped[str | None] = mapped_column(Text)
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
# fall back to a URL derived from source_kind+source_ref).
source_webpage_url: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
sha256: Mapped[str | None] = mapped_column(String(64))
@validates("rel_path")
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
self.sha256 = None
return rel_path
class DownloadProfile(Base, TimestampMixin):
"""A named download preset — a reusable format profile.
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
`format_sig`."""
__tablename__ = "download_profiles"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
) # NULL for a builtin preset
name: Mapped[str] = mapped_column(String(80))
spec: Mapped[dict] = mapped_column(JSON)
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class FileFolder(Base, TimestampMixin):
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
__tablename__ = "file_folders"
__table_args__ = (
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
Index(
"uq_file_folder_own_name",
"user_id",
func.lower(text("name")),
unique=True,
postgresql_where=text("name IS NOT NULL"),
),
# One Hub folder per user.
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
CheckConstraint(
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
name="ck_file_folder_kind",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
# "site:" + a host name (DNS caps one at 253 characters).
source_key: Mapped[str | None] = mapped_column(String(260))
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
retention_days: Mapped[int | None] = mapped_column(Integer)
def _ready_on_insert(context) -> datetime | None:
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
done = context.get_current_parameters().get("status") == "done"
return datetime.now(timezone.utc) if done else None
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
"""A user's request to download one source — the per-user layer over `MediaAsset`.
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
`display_name` is the user's own name for the file (display + the Content-Disposition on
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
(SET NULL if the asset is GC'd — the job then shows as expired).
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
__tablename__ = "download_jobs"
__table_args__ = (
Index("ix_download_jobs_claim", "status", "queue_pos"),
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
CheckConstraint(
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512))
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
source_job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="SET NULL")
)
source_asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL")
)
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
edit_spec: Mapped[dict | None] = mapped_column(JSON)
profile_id: Mapped[int | None] = mapped_column(
ForeignKey("download_profiles.id", ondelete="SET NULL")
)
profile_snapshot: Mapped[dict] = mapped_column(JSON)
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
# Per-user display overrides + extras (like display_name — metadata only, never the file).
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
display_uploader_url: Mapped[str | None] = mapped_column(Text)
extra_links: Mapped[list | None] = mapped_column(JSON)
status: Mapped[str] = mapped_column(
String(16), default="queued", server_default="queued", index=True
) # queued | running | paused | done | error | canceled
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
eta_s: Mapped[int | None] = mapped_column(Integer)
phase: Mapped[str | None] = mapped_column(String(32))
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
error_code: Mapped[str | None] = mapped_column(
String(24)
) # category, see downloads.errors.classify
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
)
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
# traces its origin through its parent, and that parent may be deleted or expire later — the
# file must still know which source folder it may go back to. NULL only for a file placed before
# the column existed outside a source folder; `folders.source_key_for` works those out.
source_key: Mapped[str | None] = mapped_column(String(260))
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# When the file became done — what the library's "Added" means: a download that failed for days
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
ready_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), default=_ready_on_insert
)
class DownloadShare(Base, TimestampMixin):
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
__tablename__ = "download_shares"
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
from_user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
class DownloadQuota(Base, UpdatedAtMixin):
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
byte cap entirely (e.g. for the admin's own account)."""
__tablename__ = "download_quotas"
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
max_bytes: Mapped[int] = mapped_column(BigInteger)
max_concurrent: Mapped[int] = mapped_column(Integer)
max_jobs: Mapped[int] = mapped_column(Integer)
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class DownloadLink(Base, TimestampMixin):
"""A public capability link to one download, or to one of its owner's own folders — the "share
by link" surface.
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
the folder at the time it is opened. The token itself is never stored as written: a link is
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
grants a *registered* user access via the in-app "Shared with me" list."""
__tablename__ = "download_links"
__table_args__ = (
# A link opens exactly one thing: a file or a folder.
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
)
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
password_hash: Mapped[str | None] = mapped_column(Text)
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
# password immediately invalidates any grants issued under the old one (instead of them living
# out the 6h grant TTL).
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
the routes are write-only (host + updated_at only). See `downloads.credentials`.
Two load-bearing security controls live around this row, NOT on it:
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
user id, so the authed asset is private to that user and can never be cache-served to
another user (`downloads.credentials.cookie_salt`).
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
can't exfiltrate the user's session to a look-alike host.
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
__tablename__ = "download_credentials"
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
host_key: Mapped[str] = mapped_column(String(64))
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
+1 -1
View File
@@ -16,7 +16,7 @@ from sqlalchemy.orm import Session
from app.config import settings
from app.downloads import links as linksmod
from app.downloads import storage
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
+2 -1
View File
@@ -28,7 +28,8 @@ from app.downloads.jobdb import (
)
from app.downloads.stated import StatedFailure
from app.downloads.status import JobStatus
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem
from app.models import DISPLAY_TEXT_MAX, PlexItem
from app.downloads.models import DownloadJob, MediaAsset
from app.plex import lookups as plex_lookups
from app.plex import paths as plex_paths
from app.notifications import create_notification
+406
View File
@@ -0,0 +1,406 @@
"""Capture's reads: what the library, the download tray, the share dialogs and the public watch pages
show. Each returns rows or plain values and writes nothing; whose a row is goes through
`userscope.owned_by` / `owned_row`, so a row of someone else's reads as a missing one.
The HTTP layer (`routes.downloads`, `routes.files`, `routes.public`) asks here and serializes; the
writes are the service's (`service`, `folders`, `links`, `quota`)."""
from collections.abc import Sequence
from datetime import datetime, timezone
from sqlalchemy import func, or_, select
from sqlalchemy.orm import Session
from app.apitokens import FilesCaller, Right
from app.collation import collated
from app.downloads import folders, links, quota
from app.downloads.models import (
DownloadJob,
DownloadLink,
DownloadProfile,
DownloadQuota,
DownloadShare,
FileFolder,
LibraryState,
MediaAsset,
)
from app.downloads.status import FINISHED, JobStatus
from app.models import User
from app.userscope import owned_by, owned_row
from app.utils import like_escape
# Cap a listing so a heavy library can't return an unbounded result set (the old Download Center's
# own cap; far beyond any real library).
MAX_LIST = 500
# --- jobs and their files ------------------------------------------------------------------
def own_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
return owned_row(db, DownloadJob, job_id, user)
def job(db: Session, job_id: int) -> DownloadJob | None:
"""A job by id, whoever's — for a public link, which already names its job."""
return db.get(DownloadJob, job_id)
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
"""A file the caller may read: their own, or one shared with them."""
row = db.get(DownloadJob, job_id)
if row is None:
return None
if row.user_id == user.id or shared_with(db, job_id, user) is not None:
return row
return None
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob | None:
"""A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's
own in a folder it may read."""
if caller.grants is None:
return accessible_job(db, caller.user, job_id)
row = own_job(db, caller.user, job_id)
if row is None or not caller.may(row.folder_id, Right.READ):
return None
return row
def asset_of(db: Session, job: DownloadJob | None) -> MediaAsset | None:
"""The file under a job, if it has one yet."""
return db.get(MediaAsset, job.asset_id) if job is not None and job.asset_id else None
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
ids = {j.asset_id for j in jobs if j.asset_id}
if not ids:
return {}
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
return {a.id: a for a in rows}
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
ids = [j.id for j in jobs]
if not ids:
return {}
rows = db.execute(
select(LibraryState.job_id, LibraryState.position_seconds).where(
owned_by(LibraryState.user_id, user), LibraryState.job_id.in_(ids)
)
).all()
return {job_id: pos for job_id, pos in rows}
def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
"""Where each file came from (its source folder's key), for the list's Source column, the
"back to its source folder" move and the download tray's site: a file in a source folder has its
folder's key; one moved into an own folder (or someone else's, shared) is traced back by
`folders.source_key_for`."""
ids = {j.folder_id for j in jobs if j.folder_id is not None}
by_id = (
{f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
if ids
else {}
)
out: dict[int, str] = {}
for j in jobs:
folder = by_id.get(j.folder_id) if j.folder_id is not None else None
if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
out[j.id] = folder.source_key
else:
out[j.id] = folders.source_key_for(db, j)
return out
def queue(db: Session, user: User, recent: datetime) -> list[DownloadJob]:
"""The download tray: what is waiting, running, paused or failed, and what finished (done or
canceled) since `recent`. A done job is timed from when it became done, never from its last
write: a rename, a move or the GC's expiry warning touches an old file's row, and the tray would
announce it as just finished."""
return list(
db.execute(
select(DownloadJob)
.where(
owned_by(DownloadJob.user_id, user),
DownloadJob.status.not_in(FINISHED)
| ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent))
| ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)),
)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
.limit(MAX_LIST)
).scalars()
)
def statuses_by_source(db: Session, user: User, statuses: Sequence[str]) -> list[tuple[str, str]]:
"""(source_ref, status) of each of `user`'s jobs in one of `statuses`."""
return list(
db.execute(
select(DownloadJob.source_ref, DownloadJob.status).where(
owned_by(DownloadJob.user_id, user), DownloadJob.status.in_(tuple(statuses))
)
).tuples()
)
def library(
db: Session,
user: User,
*,
folder_id: int | None,
listable: set[int] | None,
shared: bool,
term: str,
sort: str,
order: str,
lang: str,
) -> list[DownloadJob]:
"""The caller's done files (optionally one folder's, or only in the folders a token may list),
or the ones shared with them; searched by name and channel, sorted by when they were added, by
name (in the reader's alphabet) or by size. A file is added to its owner's list when it is done,
not when it was asked for (a download that failed for days and then went through is new); one
shared with the caller, when it was shared."""
stmt = (
select(DownloadJob)
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(DownloadJob.status == JobStatus.DONE)
)
if shared:
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
owned_by(DownloadShare.to_user_id, user)
)
else:
stmt = stmt.where(owned_by(DownloadJob.user_id, user))
if folder_id is not None:
stmt = stmt.where(DownloadJob.folder_id == folder_id)
elif listable is not None:
stmt = stmt.where(DownloadJob.folder_id.in_(listable))
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
if term:
like = f"%{like_escape(term)}%"
stmt = stmt.where(
or_(
name.ilike(like),
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
)
)
key = {
"added": DownloadShare.created_at if shared else DownloadJob.ready_at,
"name": collated(name, lang),
"size": MediaAsset.size_bytes,
}[sort]
first = key.asc() if order == "asc" else key.desc()
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
return list(db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars())
def recent_jobs(db: Session) -> list[DownloadJob]:
"""Everyone's newest jobs, for the admin's list."""
return list(
db.execute(
select(DownloadJob)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
.limit(MAX_LIST)
).scalars()
)
# --- shares --------------------------------------------------------------------------------
def shared_with(db: Session, job_id: int, user: User | int) -> DownloadShare | None:
"""The grant that hands this job to `user`, if there is one."""
return db.execute(
select(DownloadShare).where(
DownloadShare.job_id == job_id, owned_by(DownloadShare.to_user_id, user)
)
).scalar_one_or_none()
def shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
rows = db.execute(
select(DownloadShare.job_id, DownloadShare.created_at).where(
owned_by(DownloadShare.to_user_id, user),
DownloadShare.job_id.in_([j.id for j in jobs]),
)
).all()
return {job_id: at for job_id, at in rows}
# --- folders -------------------------------------------------------------------------------
def own_folder(db: Session, user: User, folder_id: int) -> FileFolder | None:
return owned_row(db, FileFolder, folder_id, user)
def folders_by_id(db: Session, ids: set[int]) -> dict[int, FileFolder]:
"""These folders, by id — for a reader that already knows which ones (a token's grants)."""
if not ids:
return {}
return {f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
def folders_with_totals(
db: Session, user: User, listable: set[int] | None
) -> list[tuple[FileFolder, int, int]]:
"""The user's folders (only `listable` ones, when given) with how many done files each holds and
their size. A source folder its files have all left stays (count 0)."""
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
stmt = select(
FileFolder,
func.count(DownloadJob.id),
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
).where(owned_by(FileFolder.user_id, user))
if listable is not None:
stmt = stmt.where(FileFolder.id.in_(listable))
rows = db.execute(
stmt.outerjoin(DownloadJob, done)
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.group_by(FileFolder.id)
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
).tuples()
return [(f, count, int(size or 0)) for f, count, size in rows]
def folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
count, size = db.execute(
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
.select_from(DownloadJob)
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
).one()
return count, int(size)
def linked_folders(db: Session, folder_ids: list[int]) -> set[int]:
"""Which of these folders a live public link opens (one not past its expiry)."""
now = datetime.now(timezone.utc)
rows = db.execute(
select(DownloadLink.folder_id).where(
DownloadLink.folder_id.in_(folder_ids),
or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now),
)
).scalars()
return {folder_id for folder_id in rows if folder_id is not None}
# --- links ---------------------------------------------------------------------------------
def own_link(db: Session, user: User, link_id: int) -> DownloadLink | None:
"""A link to one of the user's files or folders."""
link = db.get(DownloadLink, link_id)
if link is None:
return None
target = (
db.get(DownloadJob, link.job_id)
if link.job_id is not None
else db.get(FileFolder, link.folder_id)
)
if target is None or target.user_id != user.id:
return None
return link
def links_to(
db: Session, *, job_id: int | None = None, folder_id: int | None = None
) -> list[DownloadLink]:
"""The links to one file or one folder, newest first."""
target = (
DownloadLink.job_id == job_id if job_id is not None else DownloadLink.folder_id == folder_id
)
return list(
db.execute(select(DownloadLink).where(target).order_by(DownloadLink.id.desc())).scalars()
)
def link_folder(db: Session, link: DownloadLink) -> FileFolder | None:
return db.get(FileFolder, link.folder_id) if link.folder_id is not None else None
def folder_job(db: Session, folder: FileFolder, job_id: int) -> DownloadJob | None:
"""One of the files a link to `folder` reaches (`links.in_folder`), while it is in it."""
return db.execute(
select(DownloadJob).where(DownloadJob.id == job_id, *links.in_folder(folder))
).scalar_one_or_none()
def owner_suspended(db: Session, job: DownloadJob) -> bool:
"""The job's owner is suspended (or gone): the worker leaves their queued jobs where they are."""
owner = db.get(User, job.user_id)
return owner is None or owner.is_suspended
# --- profiles ------------------------------------------------------------------------------
def profiles_for(db: Session, user: User) -> list[DownloadProfile]:
"""The built-in format profiles, then the user's own."""
return list(
db.execute(
select(DownloadProfile)
.where(DownloadProfile.is_builtin.is_(True) | owned_by(DownloadProfile.user_id, user))
.order_by(
DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id
)
).scalars()
)
def own_profile(db: Session, user: User, profile_id: int) -> DownloadProfile | None:
"""One of the user's own profiles (a built-in is no one's to change)."""
p = owned_row(db, DownloadProfile, profile_id, user)
return None if p is None or p.is_builtin else p
def usable_profile(db: Session, user: User, profile_id: int) -> DownloadProfile | None:
"""A profile the user may enqueue with: a built-in, or their own."""
p = db.get(DownloadProfile, profile_id)
if p is None or (not p.is_builtin and p.user_id != user.id):
return None
return p
def default_profile(db: Session) -> DownloadProfile | None:
"""The first built-in, for an enqueue that named no format."""
return db.execute(
select(DownloadProfile)
.where(DownloadProfile.is_builtin.is_(True))
.order_by(DownloadProfile.sort_order)
.limit(1)
).scalar_one_or_none()
# --- storage and quota (admin) -------------------------------------------------------------
def storage_totals(db: Session) -> tuple[int, int, int]:
"""(files on disk, their bytes, every asset row). On disk is `quota.ON_DISK`: ready files, and
uploads waiting for or failed in their conversion — the same disk the footprint and the GC cap
count."""
on_disk = db.execute(
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
).one()
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar_one()
return on_disk[0], int(on_disk[1]), total_assets
def done_files_per_user(db: Session) -> dict[int, int]:
"""How many finished files each user's Files lists."""
return dict(
db.execute(
select(DownloadJob.user_id, func.count())
.where(DownloadJob.status == JobStatus.DONE)
.group_by(DownloadJob.user_id)
)
.tuples()
.all()
)
def custom_quota(db: Session, user_id: int) -> DownloadQuota | None:
"""The admin's own limits for this user, if any were set."""
return db.get(DownloadQuota, user_id)
+47 -3
View File
@@ -26,7 +26,8 @@ from sqlalchemy.orm import Session
from app import sysconfig
from app.downloads.status import JobKind, JobStatus
from app.models import DownloadJob, DownloadQuota, MediaAsset
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
from app.userscope import owned_by
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
# (error, canceled) don't count.
@@ -85,7 +86,7 @@ def footprint(db: Session, user_id: int) -> int:
held_assets = (
select(DownloadJob.asset_id)
.where(
DownloadJob.user_id == user_id,
owned_by(DownloadJob.user_id, user_id),
or_(
DownloadJob.status.in_(_HOLDING),
and_(DownloadJob.job_kind == JobKind.UPLOAD, DownloadJob.status == JobStatus.ERROR),
@@ -109,7 +110,7 @@ def _count(db: Session, user_id: int, states) -> int:
db.execute(
select(func.count())
.select_from(DownloadJob)
.where(DownloadJob.user_id == user_id, DownloadJob.status.in_(states))
.where(owned_by(DownloadJob.user_id, user_id), DownloadJob.status.in_(states))
).scalar()
or 0
)
@@ -201,3 +202,46 @@ def usage(db: Session, user_id: int) -> dict:
"max_jobs": lim.max_jobs,
"unlimited": lim.unlimited,
}
def set_custom(
db: Session,
user_id: int,
*,
max_bytes: int | None = None,
max_concurrent: int | None = None,
max_jobs: int | None = None,
unlimited: bool | None = None,
) -> None:
"""The admin's own limits for a user: starts from what applies now, then each value given (a
size floored at 0, a count at 1). The handler commits."""
row = db.get(DownloadQuota, user_id)
if row is None:
cur = resolve(db, user_id)
row = DownloadQuota(
user_id=user_id,
max_bytes=cur.max_bytes,
max_concurrent=cur.max_concurrent,
max_jobs=cur.max_jobs,
unlimited=cur.unlimited,
)
db.add(row)
if max_bytes is not None:
row.max_bytes = max(0, max_bytes)
if max_concurrent is not None:
row.max_concurrent = max(1, max_concurrent)
if max_jobs is not None:
row.max_jobs = max(1, max_jobs)
if unlimited is not None:
row.unlimited = unlimited
db.flush()
def reset_custom(db: Session, user_id: int) -> bool:
"""Back to the defaults; False when the user had no limits of their own."""
row = db.get(DownloadQuota, user_id)
if row is None:
return False
db.delete(row)
db.flush()
return True
+1 -1
View File
@@ -23,7 +23,7 @@ from app.db import SessionLocal
from app.downloads import edit as editmod
from app.downloads import formats
from app.downloads.status import JobKind
from app.models import DownloadJob
from app.downloads.models import DownloadJob
log = logs.get_logger(__name__)
+199 -22
View File
@@ -1,16 +1,19 @@
"""Enqueue + cache resolution for the download center (the per-user layer over MediaAsset).
"""Capture's writes: enqueue + cache resolution, the job transitions, and what the library's
dialogs change (profiles, resume points, shares, a file's display fields).
`enqueue` is the single entry point used by the API: it resolves (or creates) the shared
MediaAsset for the requested source+format, links a per-user DownloadJob to it, and — on a
cache hit (asset already ready) — marks the job done instantly with no re-download. The worker
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
handles everything else. Every function here flushes and leaves the commit to its caller (the
handler); the folder, link and quota writes are `folders`, `links` and `quota`.
"""
import uuid
from datetime import datetime, timezone
from sqlalchemy import delete as delete_rows
from sqlalchemy import func, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -21,17 +24,17 @@ from app.downloads import credentials
from app.downloads import edit as editmod
from app.downloads import folders, formats, mediakind, quota, storage
from app.downloads.status import HOLDING, JobKind, JobStatus
from app.models import (
DISPLAY_TEXT_MAX,
Channel,
from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, User, Video
from app.downloads.models import (
DownloadJob,
DownloadProfile,
DownloadShare,
LibraryState,
MediaAsset,
PlexItem,
PlexShow,
Video,
)
from app.plex import lookups as plex_lookups
from app.plex import paths as plex_paths
from app.userscope import owned_by, owned_row
class EditError(Exception):
@@ -230,8 +233,7 @@ def enqueue(
folders.stamp_done(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
db.refresh(job)
db.flush()
return job
@@ -310,8 +312,7 @@ def enqueue_edit(
folders.stamp_done(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
db.refresh(job)
db.flush()
return job
@@ -399,8 +400,7 @@ def enqueue_clip(
folders.stamp_done(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
db.refresh(job)
db.flush()
return job
@@ -508,8 +508,8 @@ def release_asset(db: Session, job: DownloadJob) -> None:
# --- Job state transitions ---------------------------------------------------------------------
# The single home for the pause/resume/cancel/delete/redownload transitions + their asset
# bookkeeping (release, failed-asset reset, fresh-asset rebuild), so the routes are thin callers and
# the worker/GC could share the same definitions (B-sub-8). Each commits its own transition; the
# route keeps only auth + HTTP mapping + serialization.
# the worker/GC could share the same definitions (B-sub-8). Each flushes its transition and the caller
# commits it; the route keeps only auth + HTTP mapping + serialization.
def pause(db: Session, job: DownloadJob) -> None:
@@ -517,7 +517,7 @@ def pause(db: Session, job: DownloadJob) -> None:
in any other state."""
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING):
job.status = JobStatus.PAUSED
db.commit()
db.flush()
def resume(db: Session, job: DownloadJob) -> None:
@@ -541,7 +541,7 @@ def resume(db: Session, job: DownloadJob) -> None:
asset.status = "pending"
asset.error = None
job.status = JobStatus.QUEUED
db.commit()
db.flush()
def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset:
@@ -576,14 +576,14 @@ def cancel(db: Session, job: DownloadJob) -> None:
if job.status not in (JobStatus.DONE, JobStatus.CANCELED):
release_asset(db, job) # release while the job still counts as holding
job.status = JobStatus.CANCELED
db.commit()
db.flush()
def delete(db: Session, job: DownloadJob) -> None:
"""Delete a job, releasing its asset hold first (which frees the file iff this was the last holder)."""
release_asset(db, job)
db.delete(job)
db.commit()
db.flush()
def redownload(db: Session, job: DownloadJob) -> None:
@@ -632,4 +632,181 @@ def redownload(db: Session, job: DownloadJob) -> None:
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
job.expires_at = None # stamped again when the rebuild is done
job.expiry_warned = False
db.commit()
db.flush()
# --- the library's writes ------------------------------------------------------------------
# What the HTTP layer changes for a user: each flushes, the handler commits.
def lock_own_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
"""One of the user's jobs, re-read under a row lock, for a transition that moves its asset's ref
count (resume, cancel, delete, redownload) or writes the status those decide on (pause). Two
overlapping requests for one job run one after the other and the second decides on the first's
committed result — never both releasing the same hold, and never a pause turning a job a cancel
just released back into a holder; either can free a file another job still holds. None: not
theirs, or gone.
The asset is locked BEFORE the job: the order a finishing worker child takes them in
(`finalize_if_running`), so the two can never deadlock."""
while True:
owned = owned_row(db, DownloadJob, job_id, user)
if owned is None:
return None
asset_id = owned.asset_id
if asset_id is not None:
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
job = db.execute(
select(DownloadJob)
.where(DownloadJob.id == job_id)
.with_for_update()
.execution_options(populate_existing=True)
).scalar_one_or_none()
if job is None:
return None
if job.asset_id == asset_id:
return job
db.rollback() # another request moved the job to another asset meanwhile: lock that one
def hold_for_share(db: Session, user: User, job_id: int) -> DownloadJob | None:
"""One of the user's files, held (FOR SHARE) for the rest of the transaction: a delete of it at
the same moment (the user's, or the retention sweep's) waits and takes the new link with it,
rather than failing the link's foreign key — or, done first, leaves None."""
if owned_row(db, DownloadJob, job_id, user) is None:
return None
return db.execute(
select(DownloadJob)
.where(DownloadJob.id == job_id)
.with_for_update(read=True)
.execution_options(populate_existing=True)
).scalar_one_or_none()
def create_profile(db: Session, user: User, name: str, spec: dict) -> DownloadProfile:
"""A format profile of the user's own, after their others."""
maxpos = db.execute(
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
owned_by(DownloadProfile.user_id, user)
)
).scalar_one()
p = DownloadProfile(
user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1
)
db.add(p)
db.flush()
return p
def update_profile(profile: DownloadProfile, *, name: str | None, spec: dict | None) -> None:
"""Rename a profile and/or replace its spec; None leaves that one as it is."""
if name is not None:
profile.name = name[:80]
if spec is not None:
profile.spec = spec
def delete_profile(db: Session, profile: DownloadProfile) -> None:
db.delete(profile)
db.flush()
def set_position(db: Session, user: User, job_id: int, position: int) -> None:
"""The library player's resume point in a file the user can play; 0 clears it.
A clear is a statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon
of one stop may both clear it, and the second then deletes nothing rather than failing. A set is
an upsert, one row per (user, job) even when two checkpoints race, not a read-then-insert that
the second would fail on the unique key."""
if not position:
db.execute(
delete_rows(LibraryState).where(
owned_by(LibraryState.user_id, user), LibraryState.job_id == job_id
)
)
return
now = datetime.now(timezone.utc)
db.execute(
pg_insert(LibraryState)
.values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now)
.on_conflict_do_update(
constraint="uq_library_user_job",
set_={"position_seconds": position, "progress_updated_at": now},
)
)
def share(db: Session, job: DownloadJob, sender: User, recipient: User) -> bool:
"""Hand a file to another user, with a notice for them; False when it was already theirs."""
from app.notifications import create_notification
exists = db.execute(
select(DownloadShare.id).where(
DownloadShare.job_id == job.id, owned_by(DownloadShare.to_user_id, recipient)
)
).first()
if exists is not None:
return False
db.add(DownloadShare(job_id=job.id, from_user_id=sender.id, to_user_id=recipient.id))
create_notification(
db,
user_id=recipient.id,
type="download_shared",
title=(job.display_name or "A download") + " was shared with you",
# The client names the notice in the reader's language from `data`; the title is the
# English fallback only.
data={
"kind": "download_shared",
"from": sender.email,
"job_id": job.id,
"title": job.display_name,
},
commit=False,
)
db.flush()
return True
def unshare(db: Session, job_id: int, recipient: User) -> bool:
"""Take a file back from one user (its owner's act, or the recipient's own dismissal); False
when it was not theirs."""
row = db.execute(
select(DownloadShare).where(
DownloadShare.job_id == job_id, owned_by(DownloadShare.to_user_id, recipient)
)
).scalar_one_or_none()
if row is None:
return False
db.delete(row)
db.flush()
return True
# The display fields a user may set on their file; the file on disk keeps its own name.
DISPLAY_FIELDS = frozenset(
{"display_name", "display_uploader", "display_uploader_url", "extra_links"}
)
def set_display(job: DownloadJob, **fields: object) -> None:
"""Set a file's display fields (each already cleaned by the caller); a field not given is left."""
unknown = set(fields) - DISPLAY_FIELDS
if unknown:
raise ValueError(f"not a display field: {sorted(unknown)}")
for name, value in fields.items():
setattr(job, name, value)
def touch(asset: MediaAsset) -> None:
"""The file was just served: the cache's idle clock starts again."""
asset.last_access_at = datetime.now(timezone.utc)
def store_sha256(db: Session, asset_id: int, digest: str) -> None:
"""Keep a file's hash once it is known — only where none is, so a concurrent hash of the same
file never overwrites the first."""
db.execute(
update(MediaAsset)
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
.values(sha256=digest)
)
+2 -1
View File
@@ -37,7 +37,8 @@ from app.config import settings
from app.downloads import edit as editmod
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
from app.downloads.status import JobKind, JobStatus
from app.models import DownloadJob, MediaAsset, display_text
from app.models import display_text
from app.downloads.models import DownloadJob, MediaAsset
def _title_from_filename(filename: str) -> str:
+114 -21
View File
@@ -1,10 +1,11 @@
"""Feed read-model: the pure query-building and row-serialization layer behind the feed.
"""Feed read-model: Reader's query-building, paging and row serialization behind the feed.
Extracted from `routes/feed.py` so the read-model has one home instead of living inside the
router module — the feed route handlers and the playlist-detail query share these helpers (the
projection, the filter builder, and keyset paging). Nothing here defines a route or reaches for
request state; it's all `(db, user, params) -> query`, and a malformed cursor raises a plain
`ValueError` for the route handler to translate (the tsquery sanitizer lives in `app.utils`).
One home for the read-model rather than the router module: the feed routes, the playlist-detail
query and Home share these helpers (the projection, the filter builder, and keyset paging), and the
feed's routes only validate and answer — a page, its counts and its facets are built here. Nothing
here defines a route or reaches for request state; it's all `(db, user, params) -> query`, and a
malformed cursor raises `BadCursor` for the route handler to translate (the tsquery sanitizer lives
in `app.utils`).
"""
import base64
@@ -20,18 +21,15 @@ from app.models import (
LIVE_OR_UPCOMING,
BlockedChannel,
Channel,
ChannelTag,
ExploredChannel,
Playlist,
PlaylistItem,
SearchFind,
Subscription,
Tag,
User,
Video,
VideoState,
)
from app.reader.models import ChannelTag, Playlist, PlaylistItem, Tag, VideoState
from app.utils import iso, like_escape, to_tsquery_str
from app.userscope import owned_by
def _channel_url(channel_id: str, handle: str | None) -> str:
@@ -48,7 +46,7 @@ def _saved_expr(user: User):
.select_from(PlaylistItem)
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
.where(
Playlist.user_id == user.id,
owned_by(Playlist.user_id, user.id),
Playlist.kind == "watch_later",
PlaylistItem.video_id == Video.id,
)
@@ -103,7 +101,7 @@ def serialize(row) -> dict:
def _tag_visible(user: User):
"""A ChannelTag row this user may see: a shared system tag (user_id NULL) or their own."""
return or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
return or_(ChannelTag.user_id.is_(None), owned_by(ChannelTag.user_id, user.id))
# The pseudo-category of a selected tag id no Tag row answers (a saved view or the feed state outlives
@@ -122,7 +120,7 @@ def tags_by_category(db: Session, user: User, tag_ids: list[int]) -> dict[str, l
if not tag_ids:
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
out: dict[str, list[int]] = {}
visible = or_(Tag.user_id.is_(None), Tag.user_id == user.id)
visible = or_(Tag.user_id.is_(None), owned_by(Tag.user_id, user.id))
for tid, cat in db.execute(
select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids), visible)
).all():
@@ -142,7 +140,7 @@ def not_blocked_cond(user: User):
position — and the one query that copied only PART of this predicate set is exactly how that
video came back as the landing page's suggestion after the user had said "never again"."""
return Video.channel_id.notin_(
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
select(BlockedChannel.channel_id).where(owned_by(BlockedChannel.user_id, user.id))
)
@@ -187,7 +185,7 @@ def filtered_query(
Subscription,
and_(
Subscription.channel_id == Video.channel_id,
Subscription.user_id == user.id,
owned_by(Subscription.user_id, user.id),
),
)
else:
@@ -199,12 +197,12 @@ def filtered_query(
Subscription,
and_(
Subscription.channel_id == Video.channel_id,
Subscription.user_id == user.id,
owned_by(Subscription.user_id, user.id),
Subscription.hidden.is_(False),
),
).outerjoin(
SearchFind,
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
and_(SearchFind.video_id == Video.id, owned_by(SearchFind.user_id, user.id)),
)
# The state-gated views (watched/hidden/in_progress) only ever return videos that HAVE a
@@ -227,8 +225,8 @@ def filtered_query(
# catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by
# asking for scope=all + channel_ids (the channel is in this accessible set).
accessible_explore = (
select(Subscription.channel_id).where(Subscription.user_id == user.id)
).union(select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id))
select(Subscription.channel_id).where(owned_by(Subscription.user_id, user.id))
).union(select(ExploredChannel.channel_id).where(owned_by(ExploredChannel.user_id, user.id)))
query = query.where(
or_(
Video.via_explore.is_(False),
@@ -346,7 +344,7 @@ def filtered_query(
if untagged:
query = query.where(
Video.channel_id.notin_(
select(ChannelTag.channel_id).where(ChannelTag.user_id == user.id)
select(ChannelTag.channel_id).where(owned_by(ChannelTag.user_id, user.id))
)
)
@@ -696,3 +694,98 @@ def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str, datetime |
# Framework-free: the route handler translates this into an HTTP 400 so the
# read-model stays web-layer-agnostic (see routes/feed.get_feed).
raise ValueError("Invalid feed cursor") from exc
class BadCursor(ValueError):
"""A `cursor` this module did not write (or one from another sort): the caller's bad input."""
def feed_page(
db: Session,
user: User,
params: dict,
*,
sort: str,
seed: int,
limit: int,
cursor: str | None,
lang: str,
) -> dict:
"""One page of the feed, and the cursor for the next. A malformed cursor raises `BadCursor`."""
query, rank_expr = filtered_query(db, user, **params)
keys = sort_keys(sort, seed, rank_expr, lang)
vals, cursor_id, window = None, None, None
if cursor:
try:
vals, cursor_id, window = decode_cursor(keys, cursor)
except ValueError as e:
raise BadCursor(str(e)) from e
ranked = sort == "relevance" and rank_expr is not None
if ranked and not cursor:
window = rank_window_start(db, query)
if ranked and window is not None:
keys = sort_keys(sort, seed, windowed_rank(rank_expr, window), lang)
# Expose each key column on the row so we can build the next cursor from the last item.
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
if vals is not None:
for k, v in zip(keys, vals, strict=True):
k["value"] = v
if ranked and window is not None:
rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1)
else:
query = query.order_by(*order_by(keys))
if cursor_id is not None:
query = query.where(keyset_predicate(keys, cursor_id))
rows = db.execute(query.limit(limit + 1)).all()
has_more = len(rows) > limit
page = rows[:limit]
return {
"items": [serialize(r) for r in page],
"next_cursor": (
encode_cursor(keys, page[-1], window if ranked else None)
if (has_more and page)
else None
),
"has_more": has_more,
"limit": limit,
}
def feed_count(db: Session, user: User, **params) -> int:
"""How many videos the filters match — the reference oracle `feed_summary`'s count is tested
against."""
query, _rank = filtered_query(db, user, **params)
return db.scalar(select(func.count()).select_from(query.subquery())) or 0
def facet_counts(db: Session, user: User, **params) -> dict[int, int]:
"""Per-tag channel counts for the current filter context, one pass per category — the reference
oracle `feed_summary`'s facets are tested against.
Each category is counted with every other filter applied but its own selections ignored
(drill-down faceting: selecting one topic doesn't zero out the rest). A conjunctive (AND)
category keeps them applied, so each remaining chip narrows to channels that ALSO have every
selected tag, and tags that can't co-occur drop to zero."""
visible = _tag_visible(user)
counts: dict[int, int] = {}
for category in ("language", "topic", "other"):
conjunctive = params.get("tag_modes", {}).get(category) == "and"
base, _rank = filtered_query(
db,
user,
**{**params, "exclude_tag_category": None if conjunctive else category},
)
channels = base.with_only_columns(Video.channel_id).distinct().subquery()
rows = db.execute(
select(ChannelTag.tag_id, func.count(func.distinct(ChannelTag.channel_id)))
.select_from(channels)
.join(ChannelTag, ChannelTag.channel_id == channels.c.channel_id)
.join(Tag, and_(Tag.id == ChannelTag.tag_id, Tag.category == category))
.where(visible)
.group_by(ChannelTag.tag_id)
).all()
for tag_id, count in rows:
counts[tag_id] = count
return counts
+1 -540
View File
@@ -1,4 +1,4 @@
from datetime import date, datetime, timezone
from datetime import date, datetime
from sqlalchemy import (
BigInteger,
@@ -18,7 +18,6 @@ from sqlalchemy import (
Text,
UniqueConstraint,
func,
text,
)
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
@@ -407,85 +406,6 @@ class Video(Base, TimestampMixin):
channel: Mapped["Channel"] = relationship(back_populates="videos")
class Tag(Base, TimestampMixin):
"""A label. System tags (user_id NULL) are computed automatically and shared by all
users; user tags (user_id set) are personal. category groups them in the UI."""
__tablename__ = "tags"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
name: Mapped[str] = mapped_column(String(64))
color: Mapped[str | None] = mapped_column(String(16))
# language | topic | other
category: Mapped[str] = mapped_column(String(16), default="other", server_default="other")
class ChannelTag(Base):
"""Links a tag to a channel. source='auto' rows are managed by the auto-tagger and
regenerated freely; source='user' rows are never touched by the system."""
__tablename__ = "channel_tags"
__table_args__ = (UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),)
id: Mapped[int] = mapped_column(primary_key=True)
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
tag_id: Mapped[int] = mapped_column(ForeignKey("tags.id", ondelete="CASCADE"), index=True)
# NULL = system/auto link visible to everyone; set = a specific user's link.
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
confidence: Mapped[float | None] = mapped_column(Float)
class VideoState(Base, UpdatedAtMixin):
"""Per-user watch state for a video. Rows exist only for non-default states; the
absence of a row means 'new/unseen'."""
__tablename__ = "video_states"
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
# new | watched | hidden ("saved" is now Watch-later playlist membership, not a status)
status: Mapped[str] = mapped_column(String(12), default="new", server_default="new")
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# Resume position (seconds into the video) for the in-app player. Orthogonal to
# status: a partially-watched video keeps status "new" but a non-zero position, so
# it can render a progress bar and match the "in progress" feed filter. 0 = not
# started / finished (trivially-early and near-finished positions are not stored).
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
position_seconds: Mapped[int] = mapped_column(
Integer, default=0, server_default="0", nullable=False
)
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class LibraryState(Base):
"""Where a user stopped in a library item (their own download, or one shared with them) — the
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
__tablename__ = "library_states"
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
job_id: Mapped[int] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class SearchFind(Base, TimestampMixin):
"""Per-user record of a video the user surfaced via their own live YouTube search.
@@ -761,79 +681,6 @@ class SystemConfig(Base):
)
class Playlist(Base, TimestampMixin, UpdatedAtMixin):
"""A per-user named, ordered collection of videos from the shared catalog.
`kind`: 'user' (normal) or 'watch_later' (built-in, undeletable, one per user).
`source`: 'local' (created here) or 'youtube' (mirrored from the user's YouTube
playlists). `yt_playlist_id` links a mirrored/exported playlist to its YouTube id.
`dirty` marks a YouTube-linked playlist with local edits not yet pushed, so the read
sync won't clobber them. (source/yt_playlist_id/dirty are forward-looking for the YT
sync phases; the foundation phase only uses local playlists.)"""
__tablename__ = "playlists"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
name: Mapped[str] = mapped_column(String(255))
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
source: Mapped[str] = mapped_column(String(16), default="local", server_default="local")
yt_playlist_id: Mapped[str | None] = mapped_column(String(64))
# True when the current items/name differ from `synced_fingerprint` (unpushed edits).
dirty: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# SHA-256 of name + ordered video ids as last synced from / pushed to YouTube; the
# baseline `dirty` is derived against. NULL = no baseline yet (treated as dirty).
synced_fingerprint: Mapped[str | None] = mapped_column(String)
# JSON list of the ordered video ids as of the last sync — the 3-way-merge baseline the
# two-way YouTube sync classifies each item against (local-add vs YT-add vs real delete).
# NULL = no baseline yet; the first merge then does a lossless union (never deletes).
synced_video_ids: Mapped[str | None] = mapped_column(Text)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
items: Mapped[list["PlaylistItem"]] = relationship(
back_populates="playlist",
cascade="all, delete-orphan",
order_by="PlaylistItem.position",
)
class PlaylistItem(Base):
"""A video's membership in a playlist, with its order within that playlist."""
__tablename__ = "playlist_items"
__table_args__ = (UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
playlist_id: Mapped[int] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
playlist: Mapped["Playlist"] = relationship(back_populates="items")
class SavedView(Base, TimestampMixin, UpdatedAtMixin):
"""A per-user named snapshot of a module's filter/sort state (a "smart view").
`source` names the module the view belongs to ("feed" | "plex"); `filters` is that module's
serialized filter JSON the frontend applies with one click. `position` orders them in the sidebar
(drag-reorder, per source); `is_default` marks the one view applied automatically on load — at most
one per (user, source), enforced by the API on set. The name is unique per (user, source)."""
__tablename__ = "saved_views"
__table_args__ = (
UniqueConstraint("user_id", "source", "name", name="uq_user_source_saved_view"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
source: Mapped[str] = mapped_column(String(16), default="feed", server_default="feed")
name: Mapped[str] = mapped_column(String(80))
filters: Mapped[dict] = mapped_column(JSON)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
is_default: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class Notification(Base):
"""A durable, server-backed per-user notification (the inbox center, phase 1).
@@ -861,141 +708,6 @@ class Notification(Base):
)
class MediaAsset(Base, TimestampMixin):
"""A physical downloaded media file — the SHARED cache layer of the download center.
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
the same video with a format that hashes to the same `format_sig`, their `download_job`
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
shared; per-user edited derivatives (phase 2) are never cached here.
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
system-decided and bound to the source id — never renamed (the user's custom name is display
metadata on `download_job`, applied only when downloading to their own machine). The naming
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
__tablename__ = "media_assets"
__table_args__ = (
UniqueConstraint(
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
status: Mapped[str] = mapped_column(
String(16), default="pending", server_default="pending", index=True
) # "pending" | "ready" | "error"
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
# card / watch page / link preview can still show an image. Relative to files_root.
poster_path: Mapped[str | None] = mapped_column(Text)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
duration_s: Mapped[int | None] = mapped_column(Integer)
width: Mapped[int | None] = mapped_column(Integer)
height: Mapped[int | None] = mapped_column(Integer)
container: Mapped[str | None] = mapped_column(String(16))
vcodec: Mapped[str | None] = mapped_column(String(32))
acodec: Mapped[str | None] = mapped_column(String(32))
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
# still pending (kind is known once the file is probed at finalize). It is the single stored home
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
media_kind: Mapped[str | None] = mapped_column(String(16))
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
title: Mapped[str | None] = mapped_column(Text)
uploader: Mapped[str | None] = mapped_column(String(255))
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
uploader_url: Mapped[str | None] = mapped_column(Text)
upload_date: Mapped[date | None] = mapped_column(Date)
thumbnail_url: Mapped[str | None] = mapped_column(Text)
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
# fall back to a URL derived from source_kind+source_ref).
source_webpage_url: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
sha256: Mapped[str | None] = mapped_column(String(64))
@validates("rel_path")
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
self.sha256 = None
return rel_path
class DownloadProfile(Base, TimestampMixin):
"""A named download preset — a reusable format profile.
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
`format_sig`."""
__tablename__ = "download_profiles"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
) # NULL for a builtin preset
name: Mapped[str] = mapped_column(String(80))
spec: Mapped[dict] = mapped_column(JSON)
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class FileFolder(Base, TimestampMixin):
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
__tablename__ = "file_folders"
__table_args__ = (
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
Index(
"uq_file_folder_own_name",
"user_id",
func.lower(text("name")),
unique=True,
postgresql_where=text("name IS NOT NULL"),
),
# One Hub folder per user.
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
CheckConstraint(
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
name="ck_file_folder_kind",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
# "site:" + a host name (DNS caps one at 253 characters).
source_key: Mapped[str | None] = mapped_column(String(260))
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
retention_days: Mapped[int | None] = mapped_column(Integer)
class ApiToken(Base, TimestampMixin):
"""A user's token for a machine to reach their Files library with (`app.apitokens`): sent as
`Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its
@@ -1037,208 +749,6 @@ class ApiTokenGrant(Base):
can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
def _ready_on_insert(context) -> datetime | None:
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
done = context.get_current_parameters().get("status") == "done"
return datetime.now(timezone.utc) if done else None
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
"""A user's request to download one source — the per-user layer over `MediaAsset`.
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
`display_name` is the user's own name for the file (display + the Content-Disposition on
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
(SET NULL if the asset is GC'd — the job then shows as expired).
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
__tablename__ = "download_jobs"
__table_args__ = (
Index("ix_download_jobs_claim", "status", "queue_pos"),
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
CheckConstraint(
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
)
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
source_ref: Mapped[str] = mapped_column(String(512))
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
source_job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="SET NULL")
)
source_asset_id: Mapped[int | None] = mapped_column(
ForeignKey("media_assets.id", ondelete="SET NULL")
)
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
edit_spec: Mapped[dict | None] = mapped_column(JSON)
profile_id: Mapped[int | None] = mapped_column(
ForeignKey("download_profiles.id", ondelete="SET NULL")
)
profile_snapshot: Mapped[dict] = mapped_column(JSON)
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
# Per-user display overrides + extras (like display_name — metadata only, never the file).
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
display_uploader_url: Mapped[str | None] = mapped_column(Text)
extra_links: Mapped[list | None] = mapped_column(JSON)
status: Mapped[str] = mapped_column(
String(16), default="queued", server_default="queued", index=True
) # queued | running | paused | done | error | canceled
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
eta_s: Mapped[int | None] = mapped_column(Integer)
phase: Mapped[str | None] = mapped_column(String(32))
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
error_code: Mapped[str | None] = mapped_column(
String(24)
) # category, see downloads.errors.classify
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
)
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
# traces its origin through its parent, and that parent may be deleted or expire later — the
# file must still know which source folder it may go back to. NULL only for a file placed before
# the column existed outside a source folder; `folders.source_key_for` works those out.
source_key: Mapped[str | None] = mapped_column(String(260))
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# When the file became done — what the library's "Added" means: a download that failed for days
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
ready_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), default=_ready_on_insert
)
class DownloadShare(Base, TimestampMixin):
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
__tablename__ = "download_shares"
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
from_user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
class DownloadQuota(Base, UpdatedAtMixin):
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
byte cap entirely (e.g. for the admin's own account)."""
__tablename__ = "download_quotas"
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
max_bytes: Mapped[int] = mapped_column(BigInteger)
max_concurrent: Mapped[int] = mapped_column(Integer)
max_jobs: Mapped[int] = mapped_column(Integer)
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class DownloadLink(Base, TimestampMixin):
"""A public capability link to one download, or to one of its owner's own folders — the "share
by link" surface.
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
the folder at the time it is opened. The token itself is never stored as written: a link is
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
grants a *registered* user access via the in-app "Shared with me" list."""
__tablename__ = "download_links"
__table_args__ = (
# A link opens exactly one thing: a file or a folder.
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
)
id: Mapped[int] = mapped_column(primary_key=True)
job_id: Mapped[int | None] = mapped_column(
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
)
folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
)
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
password_hash: Mapped[str | None] = mapped_column(Text)
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
# password immediately invalidates any grants issued under the old one (instead of them living
# out the 6h grant TTL).
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
the routes are write-only (host + updated_at only). See `downloads.credentials`.
Two load-bearing security controls live around this row, NOT on it:
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
user id, so the authed asset is private to that user and can never be cache-served to
another user (`downloads.credentials.cookie_salt`).
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
can't exfiltrate the user's session to a look-alike host.
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
__tablename__ = "download_credentials"
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
host_key: Mapped[str] = mapped_column(String(64))
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
# --- Plex integration (optional) -------------------------------------------------------------
# A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is
# from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states
@@ -1673,52 +1183,3 @@ class PlexPlaylistItem(Base):
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
class ComfortPin(Base, TimestampMixin):
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X".
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
usually "put on X", a series or a channel, not one particular video. A string ref would order and
hold anything, but it would also lose the foreign key: a Plex item dropped from the library, or an
unsubscribed channel, would leave a dead tile on the one surface whose whole job is to be
reliable. So instead: one nullable CASCADE-ing FK per pinnable kind, and a check constraint that
exactly one of them is set.
That constraint is the house rule applied to a five-way union — the invariant lives in the schema,
not in a comment every insert site has to remember. `kind`/`ref` below derive FROM the columns, so
there is no second vocabulary to keep in sync.
NULLs are DISTINCT in a Postgres unique index, so the five per-kind unique constraints each stop a
double-pin of the same thing without colliding across kinds."""
__tablename__ = "comfort_pins"
__table_args__ = (
CheckConstraint(
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
name="ck_comfort_pin_one_ref",
),
UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
video_id: Mapped[str | None] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"))
channel_id: Mapped[str | None] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
plex_item_id: Mapped[int | None] = mapped_column(
ForeignKey("plex_items.id", ondelete="CASCADE")
)
plex_show_id: Mapped[int | None] = mapped_column(
ForeignKey("plex_shows.id", ondelete="CASCADE")
)
playlist_id: Mapped[int | None] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
View File
+204
View File
@@ -0,0 +1,204 @@
"""Reader's tables: what a user does with the shared catalog — where they are in a video
(`VideoState`), their playlists, their saved views, their labels on channels (`Tag`, `ChannelTag`)
and the comfort shelf on Home. Only this unit writes them; another unit reads them along a declared
edge, or asks this unit's service."""
from datetime import datetime
from sqlalchemy import (
Boolean,
CheckConstraint,
DateTime,
Float,
ForeignKey,
Integer,
JSON,
String,
Text,
UniqueConstraint,
func,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.db import Base
from app.models import TimestampMixin, UpdatedAtMixin
class Tag(Base, TimestampMixin):
"""A label. System tags (user_id NULL) are computed automatically and shared by all
users; user tags (user_id set) are personal. category groups them in the UI."""
__tablename__ = "tags"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
name: Mapped[str] = mapped_column(String(64))
color: Mapped[str | None] = mapped_column(String(16))
# language | topic | other
category: Mapped[str] = mapped_column(String(16), default="other", server_default="other")
class ChannelTag(Base):
"""Links a tag to a channel. source='auto' rows are managed by the auto-tagger and
regenerated freely; source='user' rows are never touched by the system."""
__tablename__ = "channel_tags"
__table_args__ = (UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),)
id: Mapped[int] = mapped_column(primary_key=True)
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
tag_id: Mapped[int] = mapped_column(ForeignKey("tags.id", ondelete="CASCADE"), index=True)
# NULL = system/auto link visible to everyone; set = a specific user's link.
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
confidence: Mapped[float | None] = mapped_column(Float)
class VideoState(Base, UpdatedAtMixin):
"""Per-user watch state for a video. Rows exist only for non-default states; the
absence of a row means 'new/unseen'."""
__tablename__ = "video_states"
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
# new | watched | hidden ("saved" is now Watch-later playlist membership, not a status)
status: Mapped[str] = mapped_column(String(12), default="new", server_default="new")
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# Resume position (seconds into the video) for the in-app player. Orthogonal to
# status: a partially-watched video keeps status "new" but a non-zero position, so
# it can render a progress bar and match the "in progress" feed filter. 0 = not
# started / finished (trivially-early and near-finished positions are not stored).
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
position_seconds: Mapped[int] = mapped_column(
Integer, default=0, server_default="0", nullable=False
)
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Playlist(Base, TimestampMixin, UpdatedAtMixin):
"""A per-user named, ordered collection of videos from the shared catalog.
`kind`: 'user' (normal) or 'watch_later' (built-in, undeletable, one per user).
`source`: 'local' (created here) or 'youtube' (mirrored from the user's YouTube
playlists). `yt_playlist_id` links a mirrored/exported playlist to its YouTube id.
`dirty` marks a YouTube-linked playlist with local edits not yet pushed, so the read
sync won't clobber them. (source/yt_playlist_id/dirty are forward-looking for the YT
sync phases; the foundation phase only uses local playlists.)"""
__tablename__ = "playlists"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
name: Mapped[str] = mapped_column(String(255))
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
source: Mapped[str] = mapped_column(String(16), default="local", server_default="local")
yt_playlist_id: Mapped[str | None] = mapped_column(String(64))
# True when the current items/name differ from `synced_fingerprint` (unpushed edits).
dirty: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# SHA-256 of name + ordered video ids as last synced from / pushed to YouTube; the
# baseline `dirty` is derived against. NULL = no baseline yet (treated as dirty).
synced_fingerprint: Mapped[str | None] = mapped_column(String)
# JSON list of the ordered video ids as of the last sync — the 3-way-merge baseline the
# two-way YouTube sync classifies each item against (local-add vs YT-add vs real delete).
# NULL = no baseline yet; the first merge then does a lossless union (never deletes).
synced_video_ids: Mapped[str | None] = mapped_column(Text)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
items: Mapped[list["PlaylistItem"]] = relationship(
back_populates="playlist",
cascade="all, delete-orphan",
order_by="PlaylistItem.position",
)
class PlaylistItem(Base):
"""A video's membership in a playlist, with its order within that playlist."""
__tablename__ = "playlist_items"
__table_args__ = (UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
playlist_id: Mapped[int] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
playlist: Mapped["Playlist"] = relationship(back_populates="items")
class SavedView(Base, TimestampMixin, UpdatedAtMixin):
"""A per-user named snapshot of a module's filter/sort state (a "smart view").
`source` names the module the view belongs to ("feed" | "plex"); `filters` is that module's
serialized filter JSON the frontend applies with one click. `position` orders them in the sidebar
(drag-reorder, per source); `is_default` marks the one view applied automatically on load — at most
one per (user, source), enforced by the API on set. The name is unique per (user, source)."""
__tablename__ = "saved_views"
__table_args__ = (
UniqueConstraint("user_id", "source", "name", name="uq_user_source_saved_view"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
source: Mapped[str] = mapped_column(String(16), default="feed", server_default="feed")
name: Mapped[str] = mapped_column(String(80))
filters: Mapped[dict] = mapped_column(JSON)
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
is_default: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class ComfortPin(Base, TimestampMixin):
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X".
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
usually "put on X", a series or a channel, not one particular video. A string ref would order and
hold anything, but it would also lose the foreign key: a Plex item dropped from the library, or an
unsubscribed channel, would leave a dead tile on the one surface whose whole job is to be
reliable. So instead: one nullable CASCADE-ing FK per pinnable kind, and a check constraint that
exactly one of them is set.
That constraint is the house rule applied to a five-way union — the invariant lives in the schema,
not in a comment every insert site has to remember. `kind`/`ref` below derive FROM the columns, so
there is no second vocabulary to keep in sync.
NULLs are DISTINCT in a Postgres unique index, so the five per-kind unique constraints each stop a
double-pin of the same thing without colliding across kinds."""
__tablename__ = "comfort_pins"
__table_args__ = (
CheckConstraint(
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
name="ck_comfort_pin_one_ref",
),
UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
video_id: Mapped[str | None] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"))
channel_id: Mapped[str | None] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
plex_item_id: Mapped[int | None] = mapped_column(
ForeignKey("plex_items.id", ondelete="CASCADE")
)
plex_show_id: Mapped[int | None] = mapped_column(
ForeignKey("plex_shows.id", ondelete="CASCADE")
)
playlist_id: Mapped[int | None] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
+270
View File
@@ -0,0 +1,270 @@
"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and
for the other units that ask about them.
Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as
a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and
length, whether a video exists); another unit asks the questions at the end of this module rather
than joining Reader's tables itself.
"""
from sqlalchemy import ColumnElement, and_, func, or_, select
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
from app.feedquery import feed_columns
from app.models import Channel, User, Video
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
from app.userscope import owned_by, owned_row
# --- catalog, read along the edge ------------------------------------------------------------
def video(db: Session, video_id: str) -> Video | None:
return db.get(Video, video_id)
# --- tags ------------------------------------------------------------------------------------
def tags_with_counts(db: Session, user: User) -> list[tuple[Tag, int]]:
"""The tags this user sees (the shared system ones and their own), each with how many channel
links it has, by category then name."""
return [
(tag, count)
for tag, count in db.execute(
select(Tag, func.count(ChannelTag.id))
.outerjoin(ChannelTag, ChannelTag.tag_id == Tag.id)
.where(or_(Tag.user_id.is_(None), owned_by(Tag.user_id, user)))
.group_by(Tag.id)
.order_by(Tag.category, Tag.name)
).tuples()
]
def own_tag(db: Session, user: User, tag_id: int) -> Tag | None:
"""One of the user's own tags. A system tag (no owner) is nobody's to edit, so it reads as
missing too."""
return owned_row(db, Tag, tag_id, user)
# --- saved views -----------------------------------------------------------------------------
def views(db: Session, user: User, source: str) -> list[SavedView]:
return list(
db.execute(
select(SavedView)
.where(owned_by(SavedView.user_id, user), SavedView.source == source)
.order_by(SavedView.position, SavedView.id)
).scalars()
)
def all_views(db: Session, user: User) -> list[SavedView]:
return list(db.execute(select(SavedView).where(owned_by(SavedView.user_id, user))).scalars())
def own_view(db: Session, user: User, view_id: int) -> SavedView | None:
return owned_row(db, SavedView, view_id, user)
# --- watch state -----------------------------------------------------------------------------
def video_state(db: Session, user: User, video_id: str) -> VideoState | None:
return db.execute(
select(VideoState).where(
owned_by(VideoState.user_id, user), VideoState.video_id == video_id
)
).scalar_one_or_none()
# --- playlists -------------------------------------------------------------------------------
def own_playlist(
db: Session, user: User, playlist_id: int, *, lock: bool = False
) -> Playlist | None:
return owned_row(db, Playlist, playlist_id, user, lock=lock)
def playlists_for(db: Session, user: User) -> list[Playlist]:
"""The user's playlists in their rail order."""
return list(
db.execute(
select(Playlist)
.where(owned_by(Playlist.user_id, user))
.order_by(Playlist.position, Playlist.created_at)
).scalars()
)
def watch_later(db: Session, user: User) -> Playlist | None:
return db.scalar(
select(Playlist).where(owned_by(Playlist.user_id, user), Playlist.kind == "watch_later")
)
def item_counts(db: Session, ids: list[int]) -> dict[int, int]:
return dict(
db.execute(
select(PlaylistItem.playlist_id, func.count())
.where(PlaylistItem.playlist_id.in_(ids))
.group_by(PlaylistItem.playlist_id)
)
.tuples()
.all()
)
def durations(db: Session, ids: list[int]) -> dict[int, int]:
"""Each playlist's running time: the sum of its videos' lengths."""
return {
pid: int(total or 0)
for pid, total in db.execute(
select(PlaylistItem.playlist_id, func.coalesce(func.sum(Video.duration_seconds), 0))
.join(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id.in_(ids))
.group_by(PlaylistItem.playlist_id)
).tuples()
}
def containing(db: Session, ids: list[int], video_id: str) -> set[int]:
"""Which of these playlists already hold the video (the add-to-playlist popover's ticks)."""
return set(
db.execute(
select(PlaylistItem.playlist_id).where(
PlaylistItem.playlist_id.in_(ids), PlaylistItem.video_id == video_id
)
).scalars()
)
def covers(db: Session, ids: list[int]) -> dict[int, str | None]:
"""Each playlist's cover: its first item's thumbnail, for every list in ONE window query rather
than a lookup per playlist."""
rn = (
func.row_number()
.over(partition_by=PlaylistItem.playlist_id, order_by=PlaylistItem.position)
.label("rn")
)
first_items = (
select(PlaylistItem.playlist_id.label("pid"), Video.thumbnail_url.label("thumb"), rn)
.join(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id.in_(ids))
.subquery()
)
return dict(
db.execute(select(first_items.c.pid, first_items.c.thumb).where(first_items.c.rn == 1))
.tuples()
.all()
)
def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None]:
"""One playlist's item count, running time and cover. An outer join, so an empty list still
answers (0, 0, None)."""
count, total = db.execute(
select(func.count(PlaylistItem.id), func.coalesce(func.sum(Video.duration_seconds), 0))
.select_from(PlaylistItem)
.outerjoin(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id == playlist_id)
).one()
cover = db.scalar(
select(Video.thumbnail_url)
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
.where(PlaylistItem.playlist_id == playlist_id)
.order_by(PlaylistItem.position)
.limit(1)
)
return count or 0, total or 0, cover
def playlist_rows(db: Session, user: User, playlist_id: int) -> list:
"""A playlist's videos as feed rows (the same columns the feed serializes), in list order."""
state = aliased(VideoState)
return list(
db.execute(
select(*feed_columns(user, state))
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
.join(Channel, Channel.id == Video.channel_id)
.outerjoin(state, and_(state.video_id == Video.id, owned_by(state.user_id, user)))
.where(PlaylistItem.playlist_id == playlist_id)
# `id` breaks a position tie the same way on every read (the renumbering add does too).
.order_by(PlaylistItem.position, PlaylistItem.id)
).all()
)
def items(db: Session, playlist_id: int) -> list[PlaylistItem]:
"""A playlist's item rows, in list order."""
return list(
db.execute(
select(PlaylistItem)
.where(PlaylistItem.playlist_id == playlist_id)
.order_by(PlaylistItem.position, PlaylistItem.id)
).scalars()
)
def item(db: Session, playlist_id: int, video_id: str) -> PlaylistItem | None:
return db.scalar(
select(PlaylistItem).where(
PlaylistItem.playlist_id == playlist_id, PlaylistItem.video_id == video_id
)
)
# --- for the other units ---------------------------------------------------------------------
def kept_by_a_reader(video_id: InstrumentedAttribute[str]) -> ColumnElement[bool]:
"""A video someone holds: watched, in progress or hidden (a `VideoState`), or in one of their
playlists. The catalog's cleanup jobs keep such a video; as a clause, so their DELETE stays
one statement."""
return or_(
select(VideoState.id).where(VideoState.video_id == video_id).exists(),
select(PlaylistItem.id).where(PlaylistItem.video_id == video_id).exists(),
)
def playlisted_by(db: Session, video_ids: list[str]) -> list[tuple[int, str]]:
"""(user id, video id) for every user who has one of these videos in a playlist."""
return list(
db.execute(
select(Playlist.user_id, PlaylistItem.video_id)
.join(PlaylistItem, PlaylistItem.playlist_id == Playlist.id)
.where(PlaylistItem.video_id.in_(video_ids))
).tuples()
)
def has_auto_tags(channel_id: InstrumentedAttribute[str]) -> ColumnElement[bool]:
"""A channel the auto-tagger has already labelled, as a clause for its "only the missing" scan."""
return (
select(ChannelTag.id)
.where(
ChannelTag.channel_id == channel_id,
ChannelTag.source == "auto",
ChannelTag.user_id.is_(None),
)
.exists()
)
def view_tag_ids(filters: object) -> list[int]:
"""The tag ids a stored view's filters name; anything else in that slot names none."""
tags = filters.get("tags") if isinstance(filters, dict) else None
if not isinstance(tags, list):
return []
return [t for t in tags if type(t) is int] # `type`, not isinstance: True is not a tag id
def tags_in_feed_views(db: Session) -> set[int]:
"""Every tag id any user's saved feed view filters on. Read defensively (`view_tag_ids`): a view
stored before its route checked the shape must not raise for everyone's job."""
return {
t
for (filters,) in db.execute(select(SavedView.filters).where(SavedView.source == "feed"))
for t in view_tag_ids(filters)
}
+466
View File
@@ -0,0 +1,466 @@
"""Reader's writes: a user's labels and saved views, where they are in a video, their playlists'
items and order — and the system tags the catalog's auto-tagger hands over.
Every function here flushes and leaves the commit to its caller (the handler, or the job that owns
its session). A write that must not double up keeps its guard where it is written: an upsert on the
unique key, a row lock taken before a presence check. The comfort shelf's writes are Home's
(`discover.pins`); a playlist's YouTube sync is `sync.playlists`.
"""
from datetime import datetime, timezone
from sqlalchemy import exists, func, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.models import User
from app.reader import queries
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
from app.sync.playlists import lock_playlists, recompute_dirty
from app.userscope import owned_by
TAG_CATEGORIES = ("language", "topic", "other")
VIEW_NAME_MAX = 80
# --- tags ------------------------------------------------------------------------------------
def create_tag(db: Session, user: User, name: str, category: str, color: str | None) -> Tag:
"""A tag of the user's own. An unknown category falls back to "other" rather than refusing. A
name the user already has raises `IntegrityError` at the flush."""
tag = Tag(
user_id=user.id,
name=name,
color=color,
category=category if category in TAG_CATEGORIES else "other",
)
db.add(tag)
db.flush()
return tag
def update_tag(tag: Tag, *, sent: set[str], name: str | None, color: str | None) -> None:
"""Change the fields the caller sent: `color` sent as None clears it."""
if "name" in sent and name is not None:
tag.name = name
if "color" in sent:
tag.color = color
def delete_tag(db: Session, tag: Tag) -> None:
db.delete(tag) # its channel links cascade
db.flush()
def ensure_system_tag(db: Session, name: str, category: str, color: str | None) -> Tag:
"""The shared system tag of this name, made the first time a channel earns it."""
tag = db.execute(
select(Tag).where(Tag.user_id.is_(None), Tag.name == name)
).scalar_one_or_none()
if tag is None:
tag = Tag(user_id=None, name=name, category=category, color=color)
db.add(tag)
db.flush()
return tag
def set_auto_tags(db: Session, channel_id: str, desired: dict[int, float | None]) -> None:
"""Make a channel's automatic tag links exactly `desired` (tag id → confidence). Only the
auto-tagger's own links move; a user's links on the channel are never touched."""
existing = {
ct.tag_id: ct
for ct in db.execute(
select(ChannelTag).where(
ChannelTag.channel_id == channel_id,
ChannelTag.user_id.is_(None),
ChannelTag.source == "auto",
)
).scalars()
}
for tag_id, ct in existing.items():
if tag_id not in desired:
db.delete(ct)
for tag_id, conf in desired.items():
ct = existing.get(tag_id)
if ct is None:
db.add(
ChannelTag(
channel_id=channel_id,
tag_id=tag_id,
user_id=None,
source="auto",
confidence=conf,
)
)
else:
ct.confidence = conf
db.flush()
def prune_orphan_system_tags(db: Session) -> int:
"""Delete the system tags no channel links to any more (after a mapping change) — but not one a
saved feed view filters on. Views hold tag IDS, and a deleted tag comes back under a new id when
a channel earns it again, so the view's filter would point at nothing; the feed drops an unknown
id, and the view showed the whole feed under its old name. The kept ids are filtered here rather
than in SQL: a `NOT IN` over every view's ids could pass the driver's bind-parameter limit."""
in_views = queries.tags_in_feed_views(db)
orphans = [
tag
for tag in db.execute(
select(Tag).where(Tag.user_id.is_(None), ~exists().where(ChannelTag.tag_id == Tag.id))
).scalars()
if tag.id not in in_views
]
for tag in orphans:
db.delete(tag)
db.flush()
return len(orphans)
# --- saved views -----------------------------------------------------------------------------
def create_view(db: Session, user: User, source: str, name: str, filters: dict) -> SavedView:
"""A view appended after the user's others of the same module (each module numbers its own
0..N). A name the user already has there raises `IntegrityError` at the flush."""
maxpos = db.execute(
select(func.coalesce(func.max(SavedView.position), -1)).where(
owned_by(SavedView.user_id, user), SavedView.source == source
)
).scalar_one()
view = SavedView(
user_id=user.id,
source=source,
name=name[:VIEW_NAME_MAX],
filters=filters,
position=maxpos + 1,
)
db.add(view)
db.flush()
return view
def update_view(
db: Session,
view: SavedView,
*,
sent: set[str],
name: str | None = None,
filters: dict | None = None,
is_default: bool | None = None,
) -> None:
"""Change the fields the caller sent. Making a view the default clears the flag on the user's
other views of the same module first: at most one default per (user, source)."""
if "name" in sent and name is not None:
view.name = name[:VIEW_NAME_MAX]
if "filters" in sent and filters is not None:
view.filters = filters
if "is_default" in sent:
make_default = bool(is_default)
if make_default:
db.execute(
update(SavedView)
.where(
owned_by(SavedView.user_id, view.user_id),
SavedView.source == view.source,
SavedView.id != view.id,
)
.values(is_default=False)
)
view.is_default = make_default
db.flush()
def delete_view(db: Session, view: SavedView) -> None:
db.delete(view)
db.flush()
def reorder_views(db: Session, user: User, ids: list[int]) -> None:
"""Number the listed views 0..n in the listed order; ids that are not the user's are skipped."""
by_id = {v.id: v for v in queries.all_views(db, user)}
pos = 0
for vid in ids:
v = by_id.get(vid)
if v is not None:
v.position = pos
pos += 1
db.flush()
# --- watch state -----------------------------------------------------------------------------
def unmark(db: Session, user: User, video_id: str) -> None:
"""Back to "new". A row that still holds a resume point keeps it (un-marking "watched" restores
the in-progress state, not where the user left off); otherwise the row goes. The player fires
this beside a checkpoint that may delete the same row: the caller's commit then raises
`StaleDataError`, which is that lost race and nothing worse."""
row = queries.video_state(db, user, video_id)
if row is None:
return
if row.position_seconds:
row.status = "new"
row.watched_at = None
else:
db.delete(row)
db.flush()
def mark(db: Session, user: User, video_id: str, status: str) -> None:
""" "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at
the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then
UPDATE raised `StaleDataError` on that race, and an INSERT … ON CONFLICT is immune to it.
"hidden" keeps any earlier `watched_at`."""
now = datetime.now(timezone.utc)
set_: dict = {"status": status, "updated_at": now}
if status == "watched":
set_["watched_at"] = now
db.execute(
pg_insert(VideoState)
.values(
user_id=user.id,
video_id=video_id,
status=status,
watched_at=now if status == "watched" else None,
)
.on_conflict_do_update(constraint="uq_user_video", set_=set_)
)
def clear_state(db: Session, user: User, video_id: str) -> None:
"""The video back to pristine for this user: status, resume point and `watched_at` all go.
Playlist membership lives elsewhere and stays."""
row = queries.video_state(db, user, video_id)
if row is not None:
db.delete(row)
db.flush()
def set_progress(db: Session, user: User, video_id: str, position: int) -> None:
"""The player's resume point. 0 clears it: a row holding nothing else goes, one with a status
(watched, hidden) keeps it with the position zeroed. The status itself is never changed here."""
row = queries.video_state(db, user, video_id)
now = datetime.now(timezone.utc)
if not position:
if row is None:
return
if row.status == "new":
db.delete(row)
else:
row.position_seconds = 0
row.progress_updated_at = now
db.flush()
return
if row is None:
row = VideoState(user_id=user.id, video_id=video_id)
db.add(row)
row.position_seconds = position
row.progress_updated_at = now
db.flush()
# --- playlists -------------------------------------------------------------------------------
class MoveRefused(Exception):
"""A move between playlists that cannot happen: `reason` is the code the UI translates."""
def __init__(self, reason: str):
super().__init__(reason)
self.reason = reason
def touch(pl: Playlist) -> None:
"""Bump the playlist's `updated_at` for an edit that only writes `playlist_items`.
`UpdatedAtMixin`'s `onupdate` fires on an UPDATE of the PLAYLIST row, and add / remove /
reorder / move write the item rows instead — so without this the rail's "recently changed"
sort would only ever see renames. The assignment is what makes the column mean "last time
this list changed", which is what the rail promises the reader.
The DATABASE's clock, like the mixin's own `onupdate`: one sort key written from two clocks
(this process's and Postgres's) could order a rename and an item edit the wrong way round."""
pl.updated_at = func.now()
def _next_playlist_position(db: Session, user_id: int) -> int:
"""Append position for a new playlist in this user's rail."""
return (
db.execute(
select(func.coalesce(func.max(Playlist.position), -1)).where(
owned_by(Playlist.user_id, user_id)
)
).scalar_one()
+ 1
)
def next_item_position(db: Session, playlist_id: int) -> int:
"""Append position for a new item at the end of a playlist. Locks the playlist row first: two
adds (two tabs, the rail and the page) otherwise both read the same max and land on one
position, and the list's order between them is then arbitrary."""
lock_playlists(db, playlist_id)
return (
db.execute(
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
PlaylistItem.playlist_id == playlist_id
)
).scalar_one()
+ 1
)
def create_playlist(db: Session, user: User, name: str) -> Playlist:
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
db.add(pl)
db.flush()
return pl
def watch_later(db: Session, user: User) -> Playlist:
"""The user's built-in Watch later playlist, created on first use."""
pl = queries.watch_later(db, user)
if pl is None:
pl = Playlist(
user_id=user.id,
name="Watch later",
kind="watch_later",
position=_next_playlist_position(db, user.id),
)
db.add(pl)
db.flush() # the id, for the item the caller adds next
return pl
def add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
"""Append a video to a playlist if not already present (idempotent). Returns whether it
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
insert hit uq_playlist_video as a 500."""
lock_playlists(db, pl.id)
if queries.item(db, pl.id, video_id) is not None:
return False
db.add(
PlaylistItem(playlist_id=pl.id, video_id=video_id, position=next_item_position(db, pl.id))
)
if mark_dirty:
recompute_dirty(db, pl)
touch(pl)
db.flush()
return True
def remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
lock_playlists(db, pl.id)
found = queries.item(db, pl.id, video_id)
if found is None:
return False
db.delete(found)
if mark_dirty:
recompute_dirty(db, pl)
touch(pl)
db.flush()
return True
def place_item(db: Session, pl: Playlist, video_id: str, position: int) -> None:
"""Put a video already in the list at `position` (clamped to the end), renumbering the whole
list: positions can have gaps after removals, so "index N" is only meaningful over the
ordered rows."""
rows = queries.items(db, pl.id)
new = next(r for r in rows if r.video_id == video_id)
rows.remove(new)
rows.insert(min(position, len(rows)), new)
for pos, row in enumerate(rows):
row.position = pos
db.flush()
def add_to_playlist(db: Session, pl: Playlist, video_id: str, position: int | None) -> None:
"""Add a video to one of the user's lists, at `position` when given (the list page's "Undo" of
a removal puts the row back where it was in ONE request), else at the end. A video already in
the list stays where it is."""
added = add_item(db, pl, video_id, mark_dirty=False)
if added and position is not None:
place_item(db, pl, video_id, position)
recompute_dirty(db, pl)
db.flush()
def rename_playlist(db: Session, pl: Playlist, name: str) -> None:
"""A new name moves `dirty`; the caller holds the playlist's lock (the scheduled mirror refresh
reads `dirty` under it)."""
if name != pl.name:
pl.name = name
recompute_dirty(db, pl)
db.flush()
def delete_playlist(db: Session, pl: Playlist) -> None:
"""The playlist first, as every item write: the cascade deletes the items after the list row
is locked."""
lock_playlists(db, pl.id)
db.delete(pl) # items cascade
db.flush()
def move_item(db: Session, src: Playlist, dst: Playlist, video_id: str) -> None:
"""Move one video from `src` into `dst`, in the caller's one transaction.
The client-side shape — add to the target, then remove from the source — leaves the video in
BOTH lists whenever the second call fails. Idempotent: a video already in the target is simply
dropped from the source (moving into a list that has it is a removal, not a duplicate). Raises
`MoveRefused` for the same list twice, or a video a concurrent edit already took out."""
if src.id == dst.id:
raise MoveRefused("same_playlist")
# Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
lock_playlists(db, src.id, dst.id)
row = queries.item(db, src.id, video_id)
if row is None:
raise MoveRefused("not_in_playlist")
dst_changed = False
if queries.item(db, dst.id, video_id) is None:
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
# survives the move, and there is no window where the video belongs to neither list. The
# check above holds: both lists are locked, and an insert into the target (another tab's
# add, a YouTube merge) waits for the lock — its foreign key needs the target row.
row.playlist_id = dst.id
row.position = next_item_position(db, dst.id)
dst_changed = True
if not dst_changed:
db.delete(row)
# The source always changed; the target only when the row landed in it. Touching a target
# that already had the video promoted an untouched list in the rail's "recently changed" sort.
recompute_dirty(db, src)
touch(src)
if dst_changed:
recompute_dirty(db, dst)
touch(dst)
db.flush()
def reorder_items(db: Session, pl: Playlist, order: list[str]) -> int:
"""Number the listed videos 0..n in the listed order. Items the list left out (added in another
tab meanwhile) keep their relative order after them, so no two items share a position. The
caller holds the playlist's lock. Returns how many items the list has."""
by_video = {it.video_id: it for it in queries.items(db, pl.id)}
pos = 0
seen: set[str] = set()
for vid in order:
it = by_video.get(vid)
if it is not None:
it.position = pos
pos += 1
seen.add(vid)
for it in sorted(
(it for vid, it in by_video.items() if vid not in seen), key=lambda i: i.position
):
it.position = pos
pos += 1
recompute_dirty(db, pl)
touch(pl)
db.flush()
return pos
+3 -2
View File
@@ -16,6 +16,7 @@ from app.db import Base
from app.models import POSITION_MAX_S, User
from app.issues import gitea
from app.refusal import refused
from app.userscope import owned_row
from app.youtube.client import YouTubeError, refusal_reason
M = TypeVar("M", bound=Base)
@@ -90,8 +91,8 @@ def owned_or_404(
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
obj = get_or_404(db, model, id_, what=what, lock=lock)
if getattr(obj, owner_attr) != user.id:
obj = owned_row(db, model, id_, user, owner_attr=owner_attr, lock=lock)
if obj is None:
raise not_found(what)
return obj
+1 -4
View File
@@ -19,15 +19,12 @@ from app.models import (
LIVE_OR_UPCOMING,
BlockedChannel,
Channel,
ChannelTag,
ExploredChannel,
Playlist,
PlaylistItem,
Subscription,
Tag,
User,
Video,
)
from app.reader.models import ChannelTag, Playlist, PlaylistItem, Tag
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
from app.sync.explore import explore_ingest_page
from app.sync.runner import backfill_channel_now, run_recent_backfill
+9 -19
View File
@@ -13,7 +13,6 @@ from typing import Literal
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -22,9 +21,9 @@ from app.db import get_db
from app.discover import hero as hero_mod
from app.discover import pins, rails
from app.discover.budget import Budget, DEFAULT_BUDGET
from app.models import ComfortPin, User
from app.models import User
from app.plex.cards import norm_lang
from app.routes._common import owned_or_404, not_found
from app.routes._common import not_found
router = APIRouter(prefix="/api/discover", tags=["discover"])
@@ -140,30 +139,19 @@ def add_pin(
raise not_found("item")
column, value = resolved
existing = db.execute(
select(ComfortPin).where(
ComfortPin.user_id == user.id,
getattr(ComfortPin, column) == value,
)
).scalar_one_or_none()
existing = pins.pin_of(db, user, column, value)
if existing is not None:
return {"ok": True, "id": existing.id, "created": False}
pin = ComfortPin(user_id=user.id, position=pins.next_position(db, user), **{column: value})
db.add(pin)
try:
pin = pins.add(db, user, column, value)
db.commit()
except IntegrityError:
# A concurrent pin of the same thing from a second tab. The unique constraint did its job;
# the user's intent ("be on my shelf") is satisfied either way, so report the winner rather
# than a 500 the user can do nothing with.
db.rollback()
winner = db.execute(
select(ComfortPin).where(
ComfortPin.user_id == user.id,
getattr(ComfortPin, column) == value,
)
).scalar_one_or_none()
winner = pins.pin_of(db, user, column, value)
if winner is None:
# Not the unique constraint, then: the only other way in is the target row being pruned
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
@@ -180,8 +168,10 @@ def remove_pin(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
pin = owned_or_404(db, ComfortPin, pin_id, user, what="pin")
db.delete(pin)
pin = pins.own_pin(db, user, pin_id)
if pin is None:
raise not_found("pin")
pins.remove(db, pin)
db.commit()
return {"deleted": pin_id}
+106 -337
View File
@@ -3,23 +3,22 @@
All routes are behind `current_user`; admin routes add `admin_user`. Live job state is polled by the
frontend (the worker is a separate process); this module never downloads — it enqueues, manages job
state, and shares finished files. The library itself — listing, renaming, moving, deleting and
serving a done file, and uploading one — is `routes.files`.
serving a done file, and uploading one — is `routes.files`. Reads come from `downloads.queries`,
writes go through Capture's service (`service`, `links`, `quota`); this module validates, answers in
HTTP and commits.
"""
import re
from typing import Literal
from typing import TYPE_CHECKING, Literal
from datetime import datetime, timedelta, timezone
from urllib.parse import parse_qs, urlparse
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from sqlalchemy import delete, func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from app.mailtext import lang_for_user
from app.collation import collated
from app import audit, sysconfig
from app.auth import admin_user, current_user
from app.config import settings
@@ -29,45 +28,32 @@ from app.downloads import credentials as dlcreds
from app.downloads import edit as editmod
from app.downloads import folders
from app.downloads import links as linksmod
from app.downloads import quota, service, storage
from app.downloads import queries, quota, service, storage
from app.downloads import urlguard
from app.downloads.folders import FolderKind
from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus
from app.models import (
DownloadJob,
DownloadLink,
FileFolder,
DownloadProfile,
DownloadQuota,
DownloadShare,
LibraryState,
MediaAsset,
User,
display_text,
)
from app.downloads.status import NOT_REFETCHABLE, JobStatus
from app.models import User, display_text
from app.routes._common import (
ProgressIn,
get_or_404,
kept_position,
not_found,
owned_or_404,
)
from app.routes.files import (
MAX_LIST,
accessible_job,
assets_for,
own_folder,
own_job,
own_job_for_update,
positions_for,
quota_detail,
serialize,
serialize_job,
source_keys,
)
from app.security import hash_password
from app.userscope import is_messageable_user, messageable_clauses
from app.utils import iso
from app.userscope import emails_of, is_messageable_user, messageable_others, user_by_email
from app.refusal import refused
if TYPE_CHECKING:
from app.downloads.models import DownloadJob, DownloadLink, DownloadProfile, FileFolder
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"])
@@ -119,7 +105,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
# --- profiles ------------------------------------------------------------------------------
def _serialize_profile(p: DownloadProfile) -> dict:
def _serialize_profile(p: "DownloadProfile") -> dict:
return {
"id": p.id,
"name": p.name,
@@ -131,18 +117,7 @@ def _serialize_profile(p: DownloadProfile) -> dict:
@router.get("/profiles")
def list_profiles(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
rows = (
db.execute(
select(DownloadProfile)
.where((DownloadProfile.is_builtin.is_(True)) | (DownloadProfile.user_id == user.id))
.order_by(
DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id
)
)
.scalars()
.all()
)
return [_serialize_profile(p) for p in rows]
return [_serialize_profile(p) for p in queries.profiles_for(db, user)]
class CreateProfileIn(BaseModel):
@@ -158,15 +133,7 @@ def create_profile(
spec = body.spec
if not name:
raise refused(400, "profile_name_required")
maxpos = db.execute(
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
DownloadProfile.user_id == user.id
)
).scalar_one()
p = DownloadProfile(
user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1
)
db.add(p)
p = service.create_profile(db, user, name, spec)
db.commit()
return _serialize_profile(p)
@@ -183,31 +150,32 @@ def update_profile(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
p = db.get(DownloadProfile, profile_id)
if p is None or p.is_builtin or p.user_id != user.id:
raise not_found("profile")
p = _own_profile(db, user, profile_id)
sent = body.model_fields_set
name = None
if "name" in sent:
name = (body.name or "").strip()
if not name:
raise refused(400, "profile_name_required")
p.name = name[:80]
if "spec" in sent:
if body.spec is None:
raise refused(400, "spec_not_object")
p.spec = body.spec
if "spec" in sent and body.spec is None:
raise refused(400, "spec_not_object")
service.update_profile(p, name=name, spec=body.spec if "spec" in sent else None)
db.commit()
return _serialize_profile(p)
def _own_profile(db: Session, user: User, profile_id: int) -> "DownloadProfile":
p = queries.own_profile(db, user, profile_id)
if p is None:
raise not_found("profile")
return p
@router.delete("/profiles/{profile_id}")
def delete_profile(
profile_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
p = db.get(DownloadProfile, profile_id)
if p is None or p.is_builtin or p.user_id != user.id:
raise not_found("profile")
db.delete(p)
service.delete_profile(db, _own_profile(db, user, profile_id))
db.commit()
return {"deleted": profile_id}
@@ -277,18 +245,13 @@ def _resolve_spec(
"""Pick the format spec for an enqueue: an explicit profile_id (builtin or the user's own),
an inline spec, or the first builtin as a fallback."""
if profile_id is not None:
p = db.get(DownloadProfile, profile_id)
if p is None or (not p.is_builtin and p.user_id != user.id):
p = queries.usable_profile(db, user, profile_id)
if p is None:
raise not_found("profile")
return p.spec, p.id
if spec is not None:
return spec, None
fallback = db.execute(
select(DownloadProfile)
.where(DownloadProfile.is_builtin.is_(True))
.order_by(DownloadProfile.sort_order)
.limit(1)
).scalar_one_or_none()
fallback = queries.default_profile(db)
if fallback is None:
raise refused(400, "no_profile")
return fallback.spec, fallback.id
@@ -326,6 +289,7 @@ def enqueue_download(
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
db.commit()
return serialize_job(db, job)
@@ -355,6 +319,7 @@ def enqueue_edit(
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
except service.EditError as e:
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e
db.commit()
return serialize_job(db, job)
@@ -398,38 +363,22 @@ def enqueue_clip(
raise HTTPException(status_code=status, detail=detail) from e
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
db.commit()
return serialize_job(db, job)
# How long a finished job stays in the queue: the tray polls it from every page and must see a job
# turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's.
# A done job is timed from when it became done, never from its last write: a rename, a move or the
# GC's expiry warning touches an old file's row, and the tray would announce it as just finished.
QUEUE_RECENT = timedelta(minutes=10)
@router.get("")
def list_downloads(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
"""The queue: what is waiting, running, paused or failed, and what finished a moment ago."""
recent = datetime.now(timezone.utc) - QUEUE_RECENT
jobs = (
db.execute(
select(DownloadJob)
.where(
DownloadJob.user_id == user.id,
DownloadJob.status.not_in(FINISHED)
| ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent))
| ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)),
)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
.limit(MAX_LIST)
)
.scalars()
.all()
)
assets = assets_for(db, jobs)
positions = positions_for(db, user, jobs)
sources = source_keys(db, jobs)
jobs = queries.queue(db, user, datetime.now(timezone.utc) - QUEUE_RECENT)
assets = queries.assets_for(db, jobs)
positions = queries.positions_for(db, user, jobs)
sources = queries.source_keys(db, jobs)
return [
{
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
@@ -453,14 +402,8 @@ _INDEX_RANK: dict[str, int] = {
def my_download_index(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
"""source_ref -> best active status, for the feed's per-card "downloaded / queued" badge.
Small + cheap so the feed can poll it without loading the full job list."""
rows = db.execute(
select(DownloadJob.source_ref, DownloadJob.status).where(
DownloadJob.user_id == user.id,
DownloadJob.status.in_(tuple(_INDEX_RANK)),
)
).all()
out: dict[str, str] = {}
for ref, status in rows:
for ref, status in queries.statuses_by_source(db, user, tuple(_INDEX_RANK)):
if _INDEX_RANK.get(status, 0) > _INDEX_RANK.get(out.get(ref, ""), 0):
out[ref] = status
return out
@@ -472,6 +415,7 @@ def pause_download(
) -> dict:
job = own_job_for_update(db, user, job_id)
service.pause(db, job)
db.commit()
return serialize_job(db, job)
@@ -481,6 +425,7 @@ def resume_download(
) -> dict:
job = own_job_for_update(db, user, job_id)
service.resume(db, job)
db.commit()
return serialize_job(db, job)
@@ -508,6 +453,7 @@ def redownload_download(
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED):
raise refused(409, "still_downloading")
service.redownload(db, job)
db.commit()
return serialize_job(db, job)
@@ -517,6 +463,7 @@ def cancel_download(
) -> dict:
job = own_job_for_update(db, user, job_id)
service.cancel(db, job)
db.commit()
return serialize_job(db, job)
@@ -534,24 +481,7 @@ def set_library_progress(
one shared with them). A position too early or too close to the end clears it instead."""
accessible_job(db, user, job_id)
position = kept_position(body)
mine = (LibraryState.user_id == user.id) & (LibraryState.job_id == job_id)
if not position:
# A statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon of
# one stop may both clear it, and the second then deletes nothing rather than failing.
db.execute(delete(LibraryState).where(mine))
db.commit()
return {"ok": True, "job_id": job_id, "position_seconds": 0}
now = datetime.now(timezone.utc)
# One row per (user, job) even when two checkpoints race (the same pair): an upsert, not a
# read-then-insert that the second would fail on the unique key.
db.execute(
pg_insert(LibraryState)
.values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now)
.on_conflict_do_update(
constraint="uq_library_user_job",
set_={"position_seconds": position, "progress_updated_at": now},
)
)
service.set_position(db, user, job_id, position)
db.commit()
return {"ok": True, "job_id": job_id, "position_seconds": position}
@@ -564,8 +494,7 @@ def storyboard_meta(
Best-effort: for a very long/high-res source the sprite may not generate in time — then
`available` is False and the editor shows a plain timeline."""
job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
asset = queries.asset_of(db, accessible_job(db, user, job_id))
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
meta = editmod.ensure_storyboard(asset, settings.files_root)
@@ -585,8 +514,7 @@ def storyboard_meta(
@router.get("/{job_id}/poster.jpg")
def poster_image(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
"""The locally-generated poster frame for a download whose source had no thumbnail."""
job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
asset = queries.asset_of(db, accessible_job(db, user, job_id))
if asset is None or not asset.poster_path:
raise not_found("poster")
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
@@ -599,8 +527,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session =
def storyboard_image(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
):
job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
asset = queries.asset_of(db, accessible_job(db, user, job_id))
if asset is None or not asset.storyboard_path:
raise not_found("filmstrip")
path = storage.safe_abs_path(settings.files_root, asset.storyboard_path)
@@ -623,45 +550,20 @@ def share_download(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
job = own_job(db, user, job_id)
if job.status != JobStatus.DONE or job.asset_id is None:
raise refused(409, "share_unfinished")
target = body.email.strip().lower()
if not target:
raise refused(400, "recipient_required")
recipient = db.execute(
select(User).where(func.lower(User.email) == target)
).scalar_one_or_none()
recipient = user_by_email(db, target)
# Uniform "no user" for a missing OR non-messageable (suspended/deactivated) target, so
# sharing can't be used to probe which addresses exist or to reach a suspended account.
if not is_messageable_user(recipient):
if recipient is None or not is_messageable_user(recipient):
raise not_found("recipient")
if recipient.id == user.id:
raise refused(400, "share_own")
exists = db.execute(
select(DownloadShare.id).where(
DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id
)
).first()
if exists is None:
db.add(DownloadShare(job_id=job_id, from_user_id=user.id, to_user_id=recipient.id))
from app.notifications import create_notification
create_notification(
db,
user_id=recipient.id,
type="download_shared",
title=(job.display_name or "A download") + " was shared with you",
# The client names the notice in the reader's language from `data`; the title is the
# English fallback only.
data={
"kind": "download_shared",
"from": user.email,
"job_id": job_id,
"title": job.display_name,
},
commit=False,
)
if service.share(db, job, user, recipient):
db.commit()
return {"ok": True, "shared_with": recipient.email}
@@ -673,22 +575,10 @@ def unshare_download(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
owned_or_404(
db, DownloadJob, job_id, user, what="download"
) # ownership guard (404s if not the caller's job)
recipient = db.execute(
select(User).where(func.lower(User.email) == email.strip().lower())
).scalar_one_or_none()
if recipient is not None:
row = db.execute(
select(DownloadShare).where(
DownloadShare.job_id == job_id,
DownloadShare.to_user_id == recipient.id,
)
).scalar_one_or_none()
if row is not None:
db.delete(row)
db.commit()
own_job(db, user, job_id) # ownership guard (404s if not the caller's job)
recipient = user_by_email(db, email)
if recipient is not None and service.unshare(db, job_id, recipient):
db.commit()
return {"ok": True}
@@ -701,54 +591,26 @@ def share_recipients(
) -> list[dict]:
"""Registered users this user can share a download with (excludes self + any non-messageable
account — suspended or deactivated — so the picker never leaks those addresses)."""
rows = (
db.execute(
select(User)
.where(User.id != user.id, *messageable_clauses())
.order_by(collated(User.email, lang_for_user(user)))
)
.scalars()
.all()
)
rows = messageable_others(db, user, lang_for_user(user))
return [{"id": u.id, "email": u.email, "display_name": u.display_name} for u in rows]
# --- public watch links (share by link) ----------------------------------------------------
def _own_link(db: Session, user: User, link_id: int) -> DownloadLink:
link = db.get(DownloadLink, link_id)
def _own_link(db: Session, user: User, link_id: int) -> "DownloadLink":
link = queries.own_link(db, user, link_id)
if link is None:
raise not_found("link")
target = (
db.get(DownloadJob, link.job_id)
if link.job_id is not None
else db.get(FileFolder, link.folder_id)
)
if target is None or target.user_id != user.id:
raise not_found("link")
return link
def _link_expiry(days: int | None) -> datetime | None:
# None/0/negative → no expiry; otherwise N days out, capped at ~10 years. The Pydantic model
# already coerced/validated the type, so a non-number is a 422 before we get here.
if not days or days <= 0:
return None
return datetime.now(timezone.utc) + timedelta(days=min(days, 3650))
def _links_of(db: Session, *where) -> list[dict]:
rows = db.execute(select(DownloadLink).where(*where).order_by(DownloadLink.id.desc()))
return [linksmod.owner_view(row) for row in rows.scalars()]
@router.get("/{job_id}/links")
def list_links(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> list[dict]:
owned_or_404(db, DownloadJob, job_id, user, what="download")
return _links_of(db, DownloadLink.job_id == job_id)
own_job(db, user, job_id)
return [linksmod.owner_view(link) for link in queries.links_to(db, job_id=job_id)]
class CreateLinkIn(BaseModel):
@@ -764,30 +626,18 @@ class CreateLinkIn(BaseModel):
def _new_link(
db: Session, user: User, body: CreateLinkIn, **target: int
) -> tuple[DownloadLink, str]:
"""Make a link to `target` (`job_id=` or `folder_id=`, already checked to be the user's), and
the token it is opened by — which is in hand only now."""
pw = body.password.strip()
token = linksmod.new_token()
link = DownloadLink(
**target,
created_by=user.id,
allow_download=body.allow_download,
password_hash=hash_password(pw) if pw else None,
expires_at=_link_expiry(body.expires_days),
)
linksmod.seal(link, token)
db.add(link)
db.flush()
audit.record(
) -> tuple["DownloadLink", str]:
"""A link to `target` (`job_id=` or `folder_id=`, already checked to be the user's) and its
token (`links.create`). The Pydantic model already coerced the expiry, so a non-number is a 422
before this runs."""
return linksmod.create(
db,
user.id,
audit.AuditAction.DOWNLOAD_LINK_CREATE,
target_type="download_link",
target_id=link.id,
after={**target, **_link_audit(link)},
user,
password=body.password.strip(),
allow_download=body.allow_download,
expires_days=body.expires_days,
**target,
)
return link, token
@router.post("/{job_id}/links")
@@ -805,28 +655,20 @@ def create_link(
return linksmod.owner_view(link, token=token)
def _shareable_job(db: Session, user: User, job_id: int) -> DownloadJob:
"""One of the user's files, held (FOR SHARE) for the rest of the transaction: a delete of it at
the same moment (the user's, or the retention sweep's) waits and takes the new link with it,
rather than failing the link's foreign key — or, done first, leaves a 404."""
owned_or_404(db, DownloadJob, job_id, user, what="download")
job = db.execute(
select(DownloadJob)
.where(DownloadJob.id == job_id)
.with_for_update(read=True)
.execution_options(populate_existing=True)
).scalar_one_or_none()
def _shareable_job(db: Session, user: User, job_id: int) -> "DownloadJob":
"""One of the user's files, held for the link's transaction (`service.hold_for_share`)."""
job = service.hold_for_share(db, user, job_id)
if job is None:
raise not_found("download")
return job
def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder:
def _shareable_folder(db: Session, user: User, folder_id: int) -> "FileFolder":
"""One of the user's own folders. A source folder lets its files go on its own clock, and the Hub
is the user's channel to a machine of theirs: neither is a shelf to hand out. Held (FOR SHARE)
for the rest of the transaction, so a delete of it at the same moment waits and takes the new
link with it, rather than failing the link's foreign key."""
owned_or_404(db, FileFolder, folder_id, user, what="folder")
own_folder(db, user, folder_id)
try:
folder = folders.own_folder(db, user.id, folder_id)
except folders.FolderError as e:
@@ -840,8 +682,8 @@ def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder:
def list_folder_links(
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> list[dict]:
owned_or_404(db, FileFolder, folder_id, user, what="folder")
return _links_of(db, DownloadLink.folder_id == folder_id)
own_folder(db, user, folder_id)
return [linksmod.owner_view(link) for link in queries.links_to(db, folder_id=folder_id)]
@router.post("/folders/{folder_id}/links")
@@ -857,15 +699,6 @@ def create_folder_link(
return linksmod.owner_view(link, token=token)
def _link_audit(link: DownloadLink) -> dict:
"""A link's settings as the audit records them: whether it has a password, never which."""
return {
"allow_download": link.allow_download,
"has_password": link.password_hash is not None,
"expires_at": iso(link.expires_at),
}
class UpdateLinkIn(BaseModel):
allow_download: bool | None = None
expires_days: int | None = None
@@ -880,29 +713,15 @@ def update_link(
db: Session = Depends(get_db),
) -> dict:
link = _own_link(db, user, link_id)
before = _link_audit(link)
sent = body.model_fields_set
if "allow_download" in sent:
link.allow_download = bool(body.allow_download)
if "expires_days" in sent:
link.expires_at = _link_expiry(body.expires_days)
if "password" in sent:
pw = (body.password or "").strip()
link.password_hash = hash_password(pw) if pw else None
# Bump the version so any grant issued under the previous password stops working now, rather
# than living out its 6h TTL (setting/clearing/changing the password all count as a rotation).
link.password_version += 1
after = _link_audit(link)
if after != before or "password" in sent:
audit.record(
db,
user.id,
audit.AuditAction.DOWNLOAD_LINK_UPDATE,
target_type="download_link",
target_id=link_id,
before=before,
after=after,
)
linksmod.update(
db,
user,
link,
allow_download=body.allow_download,
expires_days=body.expires_days,
password=body.password,
sent=frozenset(body.model_fields_set),
)
db.commit()
return linksmod.owner_view(link)
@@ -911,16 +730,7 @@ def update_link(
def revoke_link(
link_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
link = _own_link(db, user, link_id)
db.delete(link)
audit.record(
db,
user.id,
audit.AuditAction.DOWNLOAD_LINK_REVOKE,
target_type="download_link",
target_id=link_id,
after={"job_id": link.job_id} if link.job_id is not None else {"folder_id": link.folder_id},
)
linksmod.revoke(db, user, _own_link(db, user, link_id))
db.commit()
return {"ok": True, "deleted": link_id}
@@ -932,20 +742,9 @@ def revoke_link(
def admin_list_downloads(
admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> list[dict]:
jobs = (
db.execute(
select(DownloadJob)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
.limit(MAX_LIST)
)
.scalars()
.all()
)
assets = assets_for(db, jobs)
emails = {
u.id: u.email
for u in db.execute(select(User).where(User.id.in_({j.user_id for j in jobs}))).scalars()
}
jobs = queries.recent_jobs(db)
assets = queries.assets_for(db, jobs)
emails = emails_of(db, {j.user_id for j in jobs})
out = []
for j in jobs:
data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None)
@@ -957,27 +756,11 @@ def admin_list_downloads(
@admin_router.get("/storage")
def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
# Every file on disk (`quota.ON_DISK`): ready ones, and uploads waiting for or failed in their
# conversion — the same disk the footprint and the GC cap count.
on_disk = db.execute(
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
).one()
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar()
files_on_disk, total_bytes, total_assets = queries.storage_totals(db)
# EVERY user, not only those holding a file: a quota is set before anyone downloads.
emails: dict[int, str] = {
uid: email
for uid, email in db.execute(select(User.id, User.email).order_by(User.email)).tuples()
}
emails = emails_of(db)
uids = list(emails)
# What each one's Files lists: their finished files.
files: dict[int, int] = {
uid: n
for uid, n in db.execute(
select(DownloadJob.user_id, func.count())
.where(DownloadJob.status == JobStatus.DONE)
.group_by(DownloadJob.user_id)
).tuples()
}
files = queries.done_files_per_user(db)
def row(uid: int) -> dict:
lim = quota.resolve(db, uid)
@@ -991,8 +774,8 @@ def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_d
}
return {
"files_on_disk": on_disk[0],
"total_bytes": int(on_disk[1]),
"files_on_disk": files_on_disk,
"total_bytes": total_bytes,
"total_assets": total_assets,
"total_cap_bytes": sysconfig.get_int(db, "download_total_max_bytes"),
"per_user": [row(uid) for uid in uids],
@@ -1004,10 +787,9 @@ def admin_get_quota(
user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
lim = quota.resolve(db, user_id)
row = db.get(DownloadQuota, user_id)
return {
"user_id": user_id,
"custom": row is not None,
"custom": queries.custom_quota(db, user_id) is not None,
"max_bytes": lim.max_bytes,
"max_concurrent": lim.max_concurrent,
"max_jobs": lim.max_jobs,
@@ -1032,25 +814,14 @@ def admin_set_quota(
db: Session = Depends(get_db),
) -> dict:
get_or_404(db, User, user_id, what="user")
cur = quota.resolve(db, user_id)
row = db.get(DownloadQuota, user_id)
if row is None:
row = DownloadQuota(
user_id=user_id,
max_bytes=cur.max_bytes,
max_concurrent=cur.max_concurrent,
max_jobs=cur.max_jobs,
unlimited=cur.unlimited,
)
db.add(row)
if body.max_bytes is not None:
row.max_bytes = max(0, body.max_bytes)
if body.max_concurrent is not None:
row.max_concurrent = max(1, body.max_concurrent)
if body.max_jobs is not None:
row.max_jobs = max(1, body.max_jobs)
if body.unlimited is not None:
row.unlimited = body.unlimited
quota.set_custom(
db,
user_id,
max_bytes=body.max_bytes,
max_concurrent=body.max_concurrent,
max_jobs=body.max_jobs,
unlimited=body.unlimited,
)
db.commit()
return admin_get_quota(user_id, admin, db)
@@ -1059,8 +830,6 @@ def admin_set_quota(
def admin_reset_quota(
user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db)
) -> dict:
row = db.get(DownloadQuota, user_id)
if row is not None:
db.delete(row)
if quota.reset_custom(db, user_id):
db.commit()
return admin_get_quota(user_id, admin, db)
+51 -183
View File
@@ -1,9 +1,7 @@
from datetime import date, datetime, timezone
from datetime import date
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel
from sqlalchemy import and_, func, or_, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import StaleDataError
@@ -11,34 +9,10 @@ from app import quota
from app.auth import current_user
from app.db import get_db
from app.mailtext import lang_for_user
from app.feedquery import (
decode_cursor,
encode_cursor,
feed_summary,
feed_years,
filtered_query,
keyset_predicate,
order_by,
rank_window_start,
serialize,
sort_keys,
windowed_rank,
windowed_relevance_rows,
)
from app.models import (
ChannelTag,
Tag,
User,
Video,
VideoState,
)
from app.routes._common import (
ProgressIn,
get_or_404,
kept_position,
not_found,
youtube_refusal,
)
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
from app.models import User
from app.reader import queries, service
from app.routes._common import ProgressIn, kept_position, not_found, youtube_refusal
from app.sync.videos import parse_iso8601_duration
from app.utils import iso
from app.youtube.client import YouTubeClient, YouTubeError
@@ -108,49 +82,20 @@ def get_feed(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
query, rank_expr = filtered_query(db, user, **params)
lang = lang_for_user(user)
keys = sort_keys(sort, seed, rank_expr, lang)
vals, cursor_id, window = None, None, None
if cursor:
# The read-model raises a plain ValueError on a malformed cursor; translate it to the
# HTTP 400 here so feedquery stays web-framework-free.
try:
vals, cursor_id, window = decode_cursor(keys, cursor)
except ValueError:
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
raise refused(400, "invalid_cursor") from None
ranked = sort == "relevance" and rank_expr is not None
if ranked and not cursor:
window = rank_window_start(db, query)
if ranked and window is not None:
keys = sort_keys(sort, seed, windowed_rank(rank_expr, window), lang)
# Expose each key column on the row so we can build the next cursor from the last item.
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
if vals is not None:
for k, v in zip(keys, vals, strict=True):
k["value"] = v
if ranked and window is not None:
rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1)
else:
query = query.order_by(*order_by(keys))
if cursor_id is not None:
query = query.where(keyset_predicate(keys, cursor_id))
rows = db.execute(query.limit(limit + 1)).all()
has_more = len(rows) > limit
page = rows[:limit]
return {
"items": [serialize(r) for r in page],
"next_cursor": (
encode_cursor(keys, page[-1], window if ranked else None)
if (has_more and page)
else None
),
"has_more": has_more,
"limit": limit,
}
try:
return feed_page(
db,
user,
params,
sort=sort,
seed=seed,
limit=limit,
cursor=cursor,
lang=lang_for_user(user),
)
except BadCursor:
# `from None`: a malformed cursor is an expected bad-input signal, not an error to chain.
raise refused(400, "invalid_cursor") from None
@router.get("/feed/count")
@@ -162,9 +107,7 @@ def get_feed_count(
# SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference
# oracle that test_feed_summary_matches_count_plus_facets asserts /feed/summary's `count` against;
# remove both together with that test if this endpoint is ever retired.
query, _rank = filtered_query(db, user, **params)
total = db.scalar(select(func.count()).select_from(query.subquery()))
return {"count": total or 0}
return {"count": feed_count(db, user, **params)}
@router.get("/facets")
@@ -186,31 +129,7 @@ def get_facets(
SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference
oracle for test_feed_summary_matches_count_plus_facets (it asserts /feed/summary's `facets`
against these `counts`); retire the two together."""
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
counts: dict[int, int] = {}
for category in ("language", "topic", "other"):
# Disjunctive (OR) facets drop the category's own selections so its chips keep
# independent counts (you can OR more of them in). Conjunctive (AND, topics only)
# keeps them applied, so each remaining chip narrows to channels that ALSO have all
# already-selected topics — and tags that can't co-occur drop to zero (hidden).
conjunctive = params.get("tag_modes", {}).get(category) == "and"
base, _rank = filtered_query(
db,
user,
**{**params, "exclude_tag_category": None if conjunctive else category},
)
channels = base.with_only_columns(Video.channel_id).distinct().subquery()
rows = db.execute(
select(ChannelTag.tag_id, func.count(func.distinct(ChannelTag.channel_id)))
.select_from(channels)
.join(ChannelTag, ChannelTag.channel_id == channels.c.channel_id)
.join(Tag, and_(Tag.id == ChannelTag.tag_id, Tag.category == category))
.where(visible)
.group_by(ChannelTag.tag_id)
).all()
for tag_id, count in rows:
counts[tag_id] = count
return {"counts": counts}
return {"counts": facet_counts(db, user, **params)}
@router.get("/feed/summary")
@@ -239,6 +158,11 @@ def get_feed_years(
return {"years": feed_years(db, user, **params)}
def _video_or_404(db: Session, video_id: str) -> None:
if queries.video(db, video_id) is None:
raise not_found("video")
class VideoStateIn(BaseModel):
status: str
@@ -253,50 +177,17 @@ def set_video_state(
status = body.status
if status not in VALID_STATES:
raise refused(400, "invalid_status")
get_or_404(db, Video, video_id, what="video")
now = datetime.now(timezone.utc)
_video_or_404(db, video_id)
if status == "new":
# Un-marking: keep the row if it still holds a resume position (un-marking
# "watched" should restore the in-progress state, not wipe where the user left
# off); otherwise drop it. The in-app player fires this alongside a progress
# checkpoint that may DELETE the same row, so tolerate it vanishing under us.
row = db.execute(
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
).scalar_one_or_none()
if row is not None:
if row.position_seconds:
row.status = "new"
row.watched_at = None
else:
db.delete(row)
try:
db.commit()
# silent-ok: the documented lost race — a concurrent player checkpoint already removed
# or moved this row. The state the caller asked for is the state that holds either way.
except StaleDataError:
db.rollback()
try:
service.unmark(db, user, video_id)
db.commit()
# silent-ok: the documented lost race — a concurrent player checkpoint already removed or
# moved this row (`service.unmark`). The state the caller asked for holds either way.
except StaleDataError:
db.rollback()
return {"ok": True, "video_id": video_id, "status": "new"}
# watched / hidden: atomic upsert. The player marks "watched" at end-of-playback at
# the same instant a progress checkpoint may DELETE the in-progress row; a plain
# SELECT-then-UPDATE then raises StaleDataError ("0 rows matched") on that race. An
# INSERT … ON CONFLICT DO UPDATE keyed on uq_user_video is immune to it.
set_: dict = {"status": status, "updated_at": now}
if status == "watched":
set_["watched_at"] = now # hidden keeps any existing watched_at
stmt = (
pg_insert(VideoState)
.values(
user_id=user.id,
video_id=video_id,
status=status,
watched_at=now if status == "watched" else None,
)
.on_conflict_do_update(constraint="uq_user_video", set_=set_)
)
db.execute(stmt)
service.mark(db, user, video_id, status)
db.commit()
return {"ok": True, "video_id": video_id, "status": status}
@@ -310,16 +201,12 @@ def clear_video_state(
"""Reset a video to pristine for this user — drop the whole VideoState row (status,
watch position and watched_at), as if it had never been opened. Saved/playlist membership
lives elsewhere and is untouched."""
row = db.execute(
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
).scalar_one_or_none()
if row is not None:
db.delete(row)
try:
db.commit()
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
except StaleDataError:
db.rollback()
try:
service.clear_state(db, user, video_id)
db.commit()
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
except StaleDataError:
db.rollback()
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
@@ -335,36 +222,17 @@ def set_video_progress(
partially-watched video can render a progress bar and match the "in progress" filter.
Trivially-early and near-finished positions clear the position rather than store it."""
position = kept_position(body)
get_or_404(db, Video, video_id, what="video")
row = db.execute(
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
).scalar_one_or_none()
_video_or_404(db, video_id)
if not position:
# Nothing meaningful to store: drop a status-less row entirely, otherwise just
# zero the position (a saved/hidden video keeps its status).
if row is not None:
if row.status == "new":
db.delete(row)
else:
row.position_seconds = 0
row.progress_updated_at = datetime.now(timezone.utc)
# A concurrent state change (e.g. the end-of-playback "watched" mark) may
# have removed/updated this row already — tolerate the lost race.
try:
db.commit()
# silent-ok: exactly the race the comment above describes; the response stays true.
except StaleDataError:
db.rollback()
try:
service.set_progress(db, user, video_id, 0)
db.commit()
# silent-ok: a concurrent state change (the end-of-playback "watched" mark) removed or
# updated this row already — the lost race; the response stays true.
except StaleDataError:
db.rollback()
return {"ok": True, "video_id": video_id, "position_seconds": 0}
if row is None:
row = VideoState(user_id=user.id, video_id=video_id)
db.add(row)
row.position_seconds = position
row.progress_updated_at = datetime.now(timezone.utc)
service.set_progress(db, user, video_id, position)
db.commit()
return {"ok": True, "video_id": video_id, "position_seconds": position}
@@ -382,7 +250,7 @@ def get_video_detail(
our DB (e.g. a YouTube link inside another video's description that the in-app
player navigated to) is resolved via the YouTube API (videos.list, 1 unit,
attributed to the requesting user)."""
v = db.get(Video, video_id)
v = queries.video(db, video_id)
if v is not None:
return {
"id": v.id,
+91 -244
View File
@@ -5,8 +5,10 @@ the per-user layer (name, folder, kept, expiry, shares and links all hang off it
bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk.
In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above
the routes are shared with that module — serialization, the ownership locks, the upload spool —
because a download and a library file are one row at two moments.
the routes are shared with that module — serialization, the ownership checks, the upload spool —
because a download and a library file are one row at two moments. What is read comes from
`downloads.queries`, what is written goes through Capture's service (`service`, `folders`); this
module validates, answers in HTTP and commits.
A machine with an API token (`app.apitokens`) may use the routes that take `files_caller`: list the
folders and files, fetch a file and its hash, upload, delete — each only in a folder its token was
@@ -19,51 +21,37 @@ import hashlib
import os
import tempfile
import threading
from collections.abc import AsyncIterator, Sequence
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import BinaryIO, Literal
from typing import TYPE_CHECKING, BinaryIO, Literal
from urllib.parse import unquote, urlparse
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import FileResponse, Response
from pydantic import BaseModel, Field
from sqlalchemy import func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app import audit, sysconfig
from app.apitokens import FilesCaller, Right, files_caller
from app.audit import AuditAction
from app.auth import current_user
from app.collation import collated
from app.config import settings
from app.db import get_db, release_read_connection
from app.downloads import folders, preview, quota, service, storage
from app.downloads import folders, preview, queries, quota, service, storage
from app.downloads import upload as uploadmod
from app.downloads.status import JobStatus
from app.mailtext import lang_for_user
from app.models import (
DISPLAY_TEXT_MAX,
DownloadJob,
DownloadLink,
DownloadShare,
FileFolder,
LibraryState,
MediaAsset,
User,
display_text,
)
from app.models import DISPLAY_TEXT_MAX, User, display_text
from app.refusal import refused
from app.routes._common import capped_stream_to, not_found, owned_or_404
from app.utils import iso, like_escape
from app.routes._common import capped_stream_to, not_found
from app.utils import iso
if TYPE_CHECKING:
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
router = APIRouter(prefix="/api/files", tags=["files"])
# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's
# own cap; far beyond any real library).
MAX_LIST = 500
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
_MAX_RETENTION_DAYS = 3650
_HASH_CHUNK = 1 << 20
@@ -75,7 +63,7 @@ _HASHING = threading.BoundedSemaphore(_HASHERS)
# --- shared with routes.downloads -------------------------------------------------------
def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
def serialize(job: "DownloadJob", asset: "MediaAsset | None") -> dict:
ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
return {
"id": job.id,
@@ -135,58 +123,18 @@ def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
}
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
ids = {j.asset_id for j in jobs if j.asset_id}
if not ids:
return {}
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
return {a.id: a for a in rows}
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
ids = [j.id for j in jobs]
if not ids:
return {}
rows = db.execute(
select(LibraryState.job_id, LibraryState.position_seconds).where(
LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
)
).all()
return {job_id: pos for job_id, pos in rows}
def serialize_job(db: Session, job: DownloadJob) -> dict:
def serialize_job(db: Session, job: "DownloadJob") -> dict:
"""Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
every single-job handler shares."""
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
return serialize(job, asset)
return serialize(job, queries.asset_of(db, job))
def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
or writes the status those decide on (pause). The job is re-read under a row lock, so two
overlapping requests for one job run one after the other and the second decides on the first's
committed result — never both releasing the same hold, and never a pause turning a job a cancel
just released back into a holder; either can free a file another job still holds.
The asset is locked BEFORE the job: the order a finishing worker child takes them in
(`service.finalize_if_running`), so the two can never deadlock."""
while True:
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
if asset_id is not None:
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
job = db.execute(
select(DownloadJob)
.where(DownloadJob.id == job_id)
.with_for_update()
.execution_options(populate_existing=True)
).scalar_one_or_none()
if job is None:
raise not_found("download")
if job.asset_id == asset_id:
return job
db.rollback() # another request moved the job to another asset meanwhile: lock that one
def own_job_for_update(db: Session, user: User, job_id: int) -> "DownloadJob":
"""One of the user's jobs under its row lock (`service.lock_own_job`), or a 404."""
job = service.lock_own_job(db, user, job_id)
if job is None:
raise not_found("download")
return job
def quota_detail(e: quota.QuotaExceeded) -> dict:
@@ -322,30 +270,32 @@ async def spool_upload(
tmp_path.unlink(missing_ok=True)
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
"""A file the caller may read: their own, or one shared with them."""
job = db.get(DownloadJob, job_id)
def accessible_job(db: Session, user: User, job_id: int) -> "DownloadJob":
"""A file the caller may read (`queries.accessible_job`: their own, or shared with them)."""
job = queries.accessible_job(db, user, job_id)
if job is None:
raise not_found("download")
if job.user_id == user.id:
return job
shared = db.execute(
select(DownloadShare.id).where(
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
)
).first()
if shared is None:
raise not_found("download")
return job
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob:
"""A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's
own in a folder it may read."""
if caller.grants is None:
return accessible_job(db, caller.user, job_id)
job = db.get(DownloadJob, job_id)
if job is None or job.user_id != caller.user.id or not caller.may(job.folder_id, Right.READ):
def own_job(db: Session, user: User, job_id: int) -> "DownloadJob":
job = queries.own_job(db, user, job_id)
if job is None:
raise not_found("download")
return job
def own_folder(db: Session, user: User, folder_id: int) -> "FileFolder":
folder = queries.own_folder(db, user, folder_id)
if folder is None:
raise not_found("folder")
return folder
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> "DownloadJob":
"""A file `caller` may read (`queries.readable_job`), or a 404."""
job = queries.readable_job(db, caller, job_id)
if job is None:
raise not_found("download")
return job
@@ -353,7 +303,7 @@ def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob:
# --- folders -------------------------------------------------------------------------------
def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool = False) -> dict:
def _serialize_folder(folder: "FileFolder", count: int, size: int, linked: bool = False) -> dict:
return {
"id": folder.id,
"kind": folder.kind,
@@ -369,18 +319,6 @@ def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool =
}
def _linked(db: Session, folder_ids: list[int]) -> set[int]:
"""Which of these folders a live public link opens (one not past its expiry)."""
now = datetime.now(timezone.utc)
rows = db.execute(
select(DownloadLink.folder_id).where(
DownloadLink.folder_id.in_(folder_ids),
or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now),
)
).scalars()
return {folder_id for folder_id in rows if folder_id is not None}
@router.get("/folders")
def list_folders(
caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
@@ -388,23 +326,9 @@ def list_folders(
"""The caller's folders with how many files each holds and their size (a token: the ones it may
list). A source folder its files have all left stays (count 0): the client decides whether to
show it."""
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
stmt = select(
FileFolder,
func.count(DownloadJob.id),
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
).where(FileFolder.user_id == caller.user.id)
listable = caller.folders_with(Right.LIST)
if listable is not None:
stmt = stmt.where(FileFolder.id.in_(listable))
rows = db.execute(
stmt.outerjoin(DownloadJob, done)
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.group_by(FileFolder.id)
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
).all()
linked = _linked(db, [f.id for f, _, _ in rows])
return [_serialize_folder(f, count, int(size), f.id in linked) for f, count, size in rows]
rows = queries.folders_with_totals(db, caller.user, caller.folders_with(Right.LIST))
linked = queries.linked_folders(db, [f.id for f, _, _ in rows])
return [_serialize_folder(f, count, size, f.id in linked) for f, count, size in rows]
class FolderIn(BaseModel):
@@ -422,13 +346,10 @@ def _folder_name(raw: str) -> str:
def create_folder(
body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name))
try:
with db.begin_nested():
db.add(folder)
db.flush()
except IntegrityError as e: # the unique (user, lower(name)): one folder per name
raise refused(409, "file_folder_name_taken") from e
folder = folders.create_own(db, user.id, _folder_name(body.name))
except folders.FolderError as e:
raise refused(409, e.reason) from e
audit.record(
db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id
)
@@ -451,17 +372,13 @@ def update_folder(
) -> dict:
"""Rename one of the caller's own folders (a source folder's name is its origin's), and/or set
how long any of their folders keeps a file (`folders.set_retention`)."""
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
folder = own_folder(db, user, folder_id)
sent = body.model_fields_set
if "name" in sent:
if folder.kind != folders.FolderKind.OWN:
raise refused(409, "file_folder_not_own")
folder.name = _folder_name(body.name or "")
try:
with db.begin_nested():
db.flush()
except IntegrityError as e:
raise refused(409, "file_folder_name_taken") from e
folders.rename(db, folder, _folder_name(body.name or ""))
except folders.FolderError as e:
raise refused(409, e.reason) from e
audit.record(
db,
user.id,
@@ -486,18 +403,9 @@ def update_folder(
after={"retention_days": body.retention_days},
)
db.commit()
count, size = _folder_totals(db, folder.id)
return _serialize_folder(folder, count, size, folder.id in _linked(db, [folder.id]))
def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
count, size = db.execute(
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
.select_from(DownloadJob)
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
).one()
return count, int(size)
count, size = queries.folder_totals(db, folder.id)
linked = folder.id in queries.linked_folders(db, [folder.id])
return _serialize_folder(folder, count, size, linked)
@router.delete("/folders/{folder_id}")
@@ -505,7 +413,7 @@ def delete_folder(
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
"""Delete one of the caller's own folders; its files go back to their source folders."""
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
folder = own_folder(db, user, folder_id)
try:
moved = folders.delete_own(db, folder)
except folders.FolderError as e:
@@ -547,13 +455,7 @@ def remove_shared_with_me(
"""Remove a file that was shared WITH this user from their 'Shared with me' list. Deletes
only the recipient's share grant — the owner's job and the physical file are untouched (this is
a per-user dismissal, not a delete)."""
row = db.execute(
select(DownloadShare).where(
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
)
).scalar_one_or_none()
if row is not None:
db.delete(row)
if service.unshare(db, job_id, user):
db.commit()
return {"ok": True, "removed": job_id}
@@ -582,46 +484,22 @@ def list_files(
raise refused(403, "api_token_scope")
if folder_id is not None and folder_id not in listable:
raise not_found("folder")
stmt = (
select(DownloadJob)
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(DownloadJob.status == JobStatus.DONE)
)
shared = scope == "shared"
if shared:
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
DownloadShare.to_user_id == user.id
)
else:
stmt = stmt.where(DownloadJob.user_id == user.id)
if folder_id is not None:
stmt = stmt.where(DownloadJob.folder_id == folder_id)
elif listable is not None:
stmt = stmt.where(DownloadJob.folder_id.in_(listable))
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
term = q.strip()
if term:
like = f"%{like_escape(term)}%"
stmt = stmt.where(
or_(
name.ilike(like),
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
)
)
# A file is added to its owner's list when it is done, not when it was asked for (a download that
# failed for days and then went through is new); one shared with the caller, when it was shared.
key = {
"added": DownloadShare.created_at if shared else DownloadJob.ready_at,
"name": collated(name, lang_for_user(user)),
"size": MediaAsset.size_bytes,
}[sort]
first = key.asc() if order == "asc" else key.desc()
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all()
assets = assets_for(db, jobs)
positions = positions_for(db, user, jobs)
shared_at = _shared_at(db, user, jobs) if shared else {}
sources = source_keys(db, jobs)
jobs = queries.library(
db,
user,
folder_id=folder_id,
listable=listable,
shared=shared,
term=q.strip(),
sort=sort,
order=order,
lang=lang_for_user(user),
)
assets = queries.assets_for(db, jobs)
positions = queries.positions_for(db, user, jobs)
shared_at = queries.shared_at(db, user, jobs) if shared else {}
sources = queries.source_keys(db, jobs)
return [
{
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
@@ -634,36 +512,6 @@ def list_files(
]
def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
"""Where each file came from (its source folder's key), for the list's Source column, the
"back to its source folder" move and the download tray's site: a file in a source folder has its folder's key; one moved into
an own folder (or someone else's, shared) is traced back by `folders.source_key_for`."""
ids = {j.folder_id for j in jobs if j.folder_id is not None}
by_id = (
{f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
if ids
else {}
)
out: dict[int, str] = {}
for j in jobs:
folder = by_id.get(j.folder_id) if j.folder_id is not None else None
if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
out[j.id] = folder.source_key
else:
out[j.id] = folders.source_key_for(db, j)
return out
def _shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
rows = db.execute(
select(DownloadShare.job_id, DownloadShare.created_at).where(
DownloadShare.to_user_id == user.id,
DownloadShare.job_id.in_([j.id for j in jobs]),
)
).all()
return {job_id: at for job_id, at in rows}
_MAX_EXTRA_LINKS = 8
@@ -711,18 +559,20 @@ def update_file(
"""Rename a file and its channel, set its reference links (all display-only: the file on disk
keeps its own name), move it to another folder (`folders.move`), or keep it until deleted /
hand it back to its folder's retention."""
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
job = own_job(db, user, job_id)
sent = body.model_fields_set
display: dict[str, object] = {}
if "name" in sent:
job.display_name = display_text(body.name)
display["display_name"] = display_text(body.name)
if "uploader" in sent:
job.display_uploader = display_text(body.uploader)
display["display_uploader"] = display_text(body.uploader)
if "uploader_url" in sent:
job.display_uploader_url = _clean_url(body.uploader_url)
display["display_uploader_url"] = _clean_url(body.uploader_url)
if "extra_links" in sent:
raw = body.extra_links or []
cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
job.extra_links = cleaned or None
display["extra_links"] = cleaned or None
service.set_display(job, **display)
if "folder_id" in sent:
if body.folder_id is None:
raise refused(400, "file_folder_gone")
@@ -768,7 +618,8 @@ def delete_file(
target_id=job_id,
summary=caller.via,
)
service.delete(db, job) # commits, the audit row with it
service.delete(db, job)
db.commit() # the audit row with it
return {"ok": True, "deleted": job_id}
@@ -787,7 +638,7 @@ def upload_file(
return serialize_job(db, ingest_spooled(db, upload))
def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
def ingest_spooled(db: Session, upload: SpooledUpload) -> "DownloadJob":
"""The spooled upload into the library (`upload.ingest_upload`, which commits its own rows),
its refusals answered as HTTP."""
if upload.size == 0:
@@ -811,8 +662,8 @@ def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
return job
def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
def _ready_file(db: Session, job: "DownloadJob") -> tuple["MediaAsset", Path]:
asset = queries.asset_of(db, job)
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
@@ -821,7 +672,7 @@ def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
return asset, path
def serve_file(db: Session, job: DownloadJob) -> FileResponse:
def serve_file(db: Session, job: "DownloadJob") -> FileResponse:
"""The file of `job` (the caller may read it), range-aware, named by the user's display name.
The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
@@ -839,7 +690,7 @@ def serve_file(db: Session, job: DownloadJob) -> FileResponse:
if asset.sha256:
digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode()
headers["Repr-Digest"] = f"sha-256=:{digest}:"
asset.last_access_at = datetime.now(timezone.utc) # the handler commits it
service.touch(asset) # the handler commits it
media_type = storage.served_media_type(asset.media_kind, asset.container)
if media_type is None:
return FileResponse(
@@ -894,13 +745,13 @@ def preview_file(
text = preview.text_utf8(head)
if text is None:
raise refused(415, "file_no_preview")
asset.last_access_at = datetime.now(timezone.utc)
service.touch(asset)
db.commit()
return Response(text, media_type=preview.media_type(kind, asset.container), headers=headers)
filename = storage.download_filename(
job.display_name or asset.title or asset.source_ref, asset.container, path
)
asset.last_access_at = datetime.now(timezone.utc)
service.touch(asset)
db.commit()
return FileResponse(
path,
@@ -936,10 +787,6 @@ def get_sha256(
finally:
_HASHING.release()
digest = h.hexdigest()
db.execute(
update(MediaAsset)
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
.values(sha256=digest)
)
service.store_sha256(db, asset_id, digest)
db.commit()
return {"sha256": digest}
+61 -304
View File
@@ -3,31 +3,34 @@ CRUD (create / rename / delete, add / remove / reorder items) plus YouTube two-w
mirror the user's own YouTube playlists in (`/sync-youtube`) and push/revert local edits back
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
from typing import TYPE_CHECKING
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import func, select, and_
from sqlalchemy.orm import Session, aliased
from sqlalchemy.orm import Session
from app import quota
from app.auth import current_user, has_read_scope, has_write_scope, require_write_scope
from app.db import get_db
from app.feedquery import feed_columns, serialize
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
from app.feedquery import serialize
from app.models import User
from app.reader import queries, service
from app.sync.playlists import (
PlaylistGone,
lock_playlists,
merge_playlist,
plan_merge,
recompute_dirty,
pull_user_playlists,
repull_playlist,
sync_claim,
user_sync_claim,
)
from app.ratelimit import RateLimiter
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
from app.routes._common import not_found, youtube_refusal
from app.youtube.client import YouTubeClient, YouTubeError
if TYPE_CHECKING:
from app.reader.models import Playlist
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
# SEC-14, same reasoning as the sync module's limiter: a playlist pull lists every playlist the
@@ -56,90 +59,20 @@ def _refused(status: int, reason: str) -> HTTPException:
return HTTPException(status_code=status, detail={"code": "playlist", "reason": reason})
def _touch(pl: Playlist) -> None:
"""Bump the playlist's `updated_at` for an edit that only writes `playlist_items`.
`UpdatedAtMixin`'s `onupdate` fires on an UPDATE of the PLAYLIST row, and add / remove /
reorder / move write the item rows instead — so without this the rail's "recently changed"
sort would only ever see renames. The assignment is what makes the column mean "last time
this list changed", which is what the rail promises the reader.
The DATABASE's clock, like the mixin's own `onupdate`: one sort key written from two clocks
(this process's and Postgres's) could order a rename and an item edit the wrong way round."""
pl.updated_at = func.now()
def _own_playlist(db: Session, user: User, playlist_id: int, *, lock: bool = False) -> "Playlist":
pl = queries.own_playlist(db, user, playlist_id, lock=lock)
if pl is None:
raise not_found("playlist")
return pl
def _next_playlist_position(db: Session, user_id: int) -> int:
"""Append position for a new playlist in this user's rail."""
return (
db.scalar(
select(func.coalesce(func.max(Playlist.position), -1)).where(
Playlist.user_id == user_id
)
)
+ 1
)
def _next_item_position(db: Session, playlist_id: int) -> int:
"""Append position for a new item at the end of a playlist. Locks the playlist row first: two
adds (two tabs, the rail and the page) otherwise both read the same max and land on one
position, and the list's order between them is then arbitrary."""
lock_playlists(db, playlist_id)
return (
db.scalar(
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
PlaylistItem.playlist_id == playlist_id
)
)
+ 1
)
def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
"""Append a video to a playlist if not already present (idempotent). Returns whether it
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
insert hit uq_playlist_video as a 500."""
lock_playlists(db, pl.id)
exists = db.scalar(
select(PlaylistItem).where(
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
)
)
if exists is not None:
return False
db.add(
PlaylistItem(playlist_id=pl.id, video_id=video_id, position=_next_item_position(db, pl.id))
)
if mark_dirty:
recompute_dirty(db, pl)
_touch(pl)
db.flush() # the handler commits (CLAUDE.md: helpers flush)
return True
def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
lock_playlists(db, pl.id)
item = db.scalar(
select(PlaylistItem).where(
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
)
)
if item is None:
return False
db.delete(item)
if mark_dirty:
recompute_dirty(db, pl)
_touch(pl)
db.flush() # the handler commits (CLAUDE.md: helpers flush)
return True
def _video_or_404(db: Session, video_id: str) -> None:
if queries.video(db, video_id) is None:
raise not_found("video")
def _summary(
pl: Playlist,
pl: "Playlist",
count: int,
total_duration: int = 0,
has_video: bool | None = None,
@@ -156,7 +89,7 @@ def _summary(
"total_duration_seconds": total_duration,
"cover_thumbnail": cover,
# When this list last CHANGED (rename, add/remove/move/reorder) — the rail's
# "recently changed first" sort reads it. See _touch for why it isn't automatic.
# "recently changed first" sort reads it. See `service.touch` for why it isn't automatic.
"updated_at": pl.updated_at,
}
if has_video is not None:
@@ -172,73 +105,14 @@ def list_playlists(
) -> list[dict]:
"""The user's playlists (newest custom order). With `contains=<video_id>`, each entry
also reports whether that video is already in it — used by the add-to-playlist popover."""
pls = (
db.execute(
select(Playlist)
.where(Playlist.user_id == user.id)
.order_by(Playlist.position, Playlist.created_at)
)
.scalars()
.all()
)
pls = queries.playlists_for(db, user)
if not pls:
return []
ids = [p.id for p in pls]
counts = dict(
db.execute(
select(PlaylistItem.playlist_id, func.count())
.where(PlaylistItem.playlist_id.in_(ids))
.group_by(PlaylistItem.playlist_id)
).all()
)
durations = dict(
db.execute(
select(
PlaylistItem.playlist_id,
func.coalesce(func.sum(Video.duration_seconds), 0),
)
.join(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id.in_(ids))
.group_by(PlaylistItem.playlist_id)
).all()
)
member: set[int] = set()
if contains:
member = set(
db.execute(
select(PlaylistItem.playlist_id).where(
PlaylistItem.playlist_id.in_(ids),
PlaylistItem.video_id == contains,
)
)
.scalars()
.all()
)
# Batch the cover thumbnails (first item per playlist by position) in ONE window query
# instead of a per-playlist SELECT inside _summary (the old N+1).
rn = (
func.row_number()
.over(
partition_by=PlaylistItem.playlist_id,
order_by=PlaylistItem.position,
)
.label("rn")
)
first_items = (
select(
PlaylistItem.playlist_id.label("pid"),
Video.thumbnail_url.label("thumb"),
rn,
)
.join(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id.in_(ids))
.subquery()
)
covers = dict(
db.execute(
select(first_items.c.pid, first_items.c.thumb).where(first_items.c.rn == 1)
).all()
)
counts = queries.item_counts(db, ids)
durations = queries.durations(db, ids)
member = queries.containing(db, ids, contains) if contains else set()
covers = queries.covers(db, ids)
return [
_summary(
p,
@@ -269,29 +143,11 @@ def create_playlist(
name = body.name.strip()
if not name:
raise _refused(400, "name_required")
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
db.add(pl)
pl = service.create_playlist(db, user, name)
db.commit()
return _summary(pl, 0)
def _watch_later(db: Session, user: User) -> Playlist:
"""The user's built-in Watch later playlist, created on first use."""
pl = db.scalar(
select(Playlist).where(Playlist.user_id == user.id, Playlist.kind == "watch_later")
)
if pl is None:
pl = Playlist(
user_id=user.id,
name="Watch later",
kind="watch_later",
position=_next_playlist_position(db, user.id),
)
db.add(pl)
db.flush() # the id, for the item the caller adds next; the handler commits
return pl
@router.post("/watch-later")
def add_watch_later(
body: VideoRefIn,
@@ -300,9 +156,9 @@ def add_watch_later(
) -> dict:
"""Bookmark shortcut: add a video to the built-in Watch later playlist."""
video_id = body.video_id
get_or_404(db, Video, video_id, what="video")
pl = _watch_later(db, user)
_add_item(db, pl, video_id, mark_dirty=False)
_video_or_404(db, video_id)
pl = service.watch_later(db, user)
service.add_item(db, pl, video_id, mark_dirty=False)
db.commit()
return {"ok": True, "saved": True, "playlist_id": pl.id}
@@ -313,11 +169,9 @@ def remove_watch_later(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
pl = db.scalar(
select(Playlist).where(Playlist.user_id == user.id, Playlist.kind == "watch_later")
)
pl = queries.watch_later(db, user)
if pl is not None:
_remove_item(db, pl, video_id, mark_dirty=False)
service.remove_item(db, pl, video_id, mark_dirty=False)
db.commit()
return {"ok": True, "saved": False}
@@ -354,7 +208,7 @@ def revert_youtube(
"""Reset one YouTube-linked playlist to its current YouTube state, discarding local
edits (and clearing the dirty flag). The user-facing 'undo all my changes' for a list
once the in-session undo history is gone."""
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
pl = _own_playlist(db, user, playlist_id)
if not pl.yt_playlist_id:
raise _refused(400, "not_on_youtube")
if not has_read_scope(user):
@@ -375,10 +229,10 @@ def revert_youtube(
raise youtube_refusal(exc, playlist=True) from exc
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
def _pushable(db: Session, user: User, playlist_id: int) -> "Playlist":
"""A local playlist the user may push to YouTube (create or update). The built-in
Watch later and read-only YouTube mirrors are never pushable."""
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
pl = _own_playlist(db, user, playlist_id)
if pl.kind == "watch_later":
raise _refused(400, "watch_later_not_pushable")
return pl
@@ -448,17 +302,8 @@ def get_playlist(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
state = aliased(VideoState)
rows = db.execute(
select(*feed_columns(user, state))
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
.join(Channel, Channel.id == Video.channel_id)
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
.where(PlaylistItem.playlist_id == pl.id)
# `id` breaks a position tie the same way on every read (the renumbering add does too).
.order_by(PlaylistItem.position, PlaylistItem.id)
).all()
pl = _own_playlist(db, user, playlist_id)
rows = queries.playlist_rows(db, user, pl.id)
return {
"id": pl.id,
"name": pl.name,
@@ -482,34 +327,15 @@ def rename_playlist(
db: Session = Depends(get_db),
) -> dict:
# Locked: the new name moves `dirty`, which the scheduled mirror refresh reads under this lock.
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
pl = _own_playlist(db, user, playlist_id, lock=True)
if "name" in body.model_fields_set:
name = (body.name or "").strip()
if not name:
raise _refused(400, "name_required")
if name != pl.name:
pl.name = name
recompute_dirty(db, pl)
service.rename_playlist(db, pl, name)
db.commit()
# One aggregate for count + duration (outer join so an empty playlist still returns 0),
# plus a single cover lookup — was 3 separate round-trips.
count, total = db.execute(
select(
func.count(PlaylistItem.id),
func.coalesce(func.sum(Video.duration_seconds), 0),
)
.select_from(PlaylistItem)
.outerjoin(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id == pl.id)
).one()
cover = db.scalar(
select(Video.thumbnail_url)
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
.where(PlaylistItem.playlist_id == pl.id)
.order_by(PlaylistItem.position)
.limit(1)
)
return _summary(pl, count or 0, total or 0, cover=cover)
count, total, cover = queries.playlist_totals(db, pl.id)
return _summary(pl, count, total, cover=cover)
@router.delete("/{playlist_id}")
@@ -519,7 +345,7 @@ def delete_playlist(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
pl = _own_playlist(db, user, playlist_id)
if pl.kind == "watch_later":
raise _refused(400, "watch_later_undeletable")
# Not in the middle of a sync of it (`sync_claim`): a push that read the list before the delete
@@ -540,9 +366,7 @@ def delete_playlist(
yt.delete_playlist(pl.yt_playlist_id)
except YouTubeError as exc:
raise youtube_refusal(exc, playlist=True) from exc
# The playlist first, as every item write: the cascade deletes the items before the list row.
lock_playlists(db, pl.id)
db.delete(pl) # items cascade
service.delete_playlist(db, pl)
db.commit()
return {"deleted": playlist_id}
@@ -561,26 +385,10 @@ def add_item(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
pl = _own_playlist(db, user, playlist_id)
video_id = body.video_id
get_or_404(db, Video, video_id, what="video")
added = _add_item(db, pl, video_id, mark_dirty=False)
if added and body.position is not None:
# Renumber the whole list with the new row at `position` (clamped to the end): positions
# can have gaps after removals, so "index N" is only meaningful over the ordered rows.
rows = list(
db.execute(
select(PlaylistItem)
.where(PlaylistItem.playlist_id == pl.id)
.order_by(PlaylistItem.position, PlaylistItem.id)
).scalars()
)
new = next(r for r in rows if r.video_id == video_id)
rows.remove(new)
rows.insert(min(body.position, len(rows)), new)
for pos, row in enumerate(rows):
row.position = pos
recompute_dirty(db, pl)
_video_or_404(db, video_id)
service.add_to_playlist(db, pl, video_id, body.position)
db.commit()
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
@@ -592,8 +400,8 @@ def remove_item(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
_remove_item(db, pl, video_id, mark_dirty=True)
pl = _own_playlist(db, user, playlist_id)
service.remove_item(db, pl, video_id, mark_dirty=True)
db.commit()
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
@@ -617,49 +425,21 @@ def move_item(
exactly the kind of half-applied state this codebase treats as a bug rather than an edge
case. Idempotent: if the video is already in the target, the source row is simply dropped
(moving into a list that already has it is a removal, not a duplicate)."""
src = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
src = _own_playlist(db, user, playlist_id)
# Its own reason: "this playlist doesn't exist any more" about a TARGET that is gone was read as
# the list on screen — which is still there.
dst = owned_or_404(db, Playlist, body.target_playlist_id, user, what="move_target")
if src.id == dst.id:
dst = queries.own_playlist(db, user, body.target_playlist_id)
if dst is None:
raise not_found("move_target")
try:
service.move_item(db, src, dst, video_id)
except service.MoveRefused as e:
# A same-list move is a bad request; a video a concurrent edit (another tab, a YouTube
# re-pull) already took out is gone.
status = 400 if e.reason == "same_playlist" else 404
raise HTTPException(
status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
)
# Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
lock_playlists(db, src.id, dst.id)
item = db.scalar(
select(PlaylistItem).where(
PlaylistItem.playlist_id == src.id, PlaylistItem.video_id == video_id
)
)
if item is None:
# A concurrent edit (another tab, or a YouTube re-pull) already took it out.
raise HTTPException(
status_code=404, detail={"code": "playlist_move", "reason": "not_in_playlist"}
)
already = db.scalar(
select(PlaylistItem).where(
PlaylistItem.playlist_id == dst.id, PlaylistItem.video_id == video_id
)
)
dst_changed = False
if already is None:
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
# survives the move, and there is no window where the video belongs to neither list. The
# check above holds: both lists are locked, and an insert into the target (another tab's
# add, a YouTube merge) waits for the lock — its foreign key needs the target row.
item.playlist_id = dst.id
item.position = _next_item_position(db, dst.id)
dst_changed = True
if not dst_changed:
db.delete(item)
# The source always changed; the target only when the row landed in it. `_touch` on a target
# that already had the video promoted an untouched list in the rail's "recently changed" sort.
recompute_dirty(db, src)
_touch(src)
if dst_changed:
recompute_dirty(db, dst)
_touch(dst)
status_code=status, detail={"code": "playlist_move", "reason": e.reason}
) from None
db.commit()
return {"ok": True, "from_playlist_id": src.id, "to_playlist_id": dst.id, "video_id": video_id}
@@ -676,30 +456,7 @@ def reorder_items(
db: Session = Depends(get_db),
) -> dict:
# Locked: a concurrent add between reading the items and writing the order would collide.
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
order = body.video_ids
items = {
it.video_id: it
for it in db.execute(
select(PlaylistItem).where(PlaylistItem.playlist_id == pl.id)
).scalars()
}
pos = 0
seen: set[str] = set()
for vid in order:
it = items.get(vid)
if it is not None:
it.position = pos
pos += 1
seen.add(vid)
# Any items the payload omitted (e.g. added concurrently in another tab) keep their relative
# order but get fresh trailing positions, so no two items collide on the same position.
for it in sorted(
(it for vid, it in items.items() if vid not in seen), key=lambda i: i.position
):
it.position = pos
pos += 1
recompute_dirty(db, pl)
_touch(pl)
pl = _own_playlist(db, user, playlist_id, lock=True)
pos = service.reorder_items(db, pl, body.video_ids)
db.commit()
return {"ok": True, "playlist_id": pl.id, "count": pos}
+27 -32
View File
@@ -6,23 +6,26 @@ the app or any account data. Password-protected links exchange the password once
a short-lived signed grant that the media URL carries (a `<video src>` can't send a header).
"""
from typing import TYPE_CHECKING
from fastapi import APIRouter, Depends, Request
from fastapi.responses import FileResponse
from pydantic import BaseModel
from sqlalchemy import select, update
from sqlalchemy.orm import Session
from app.config import settings
from app.db import get_db
from app.downloads import links as linksmod
from app.downloads import storage
from app.downloads import queries, storage
from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset, User
from app.ratelimit import RateLimiter, client_key
from app.security import verify_password
from app.refusal import refused
from app.routes._common import not_found
if TYPE_CHECKING:
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
router = APIRouter(prefix="/api/public", tags=["public"])
# Slow password guessing on a link (the token itself is 192-bit, so it needs no throttle). Counted per
@@ -37,26 +40,25 @@ _unlock_limit = RateLimiter(max_events=10, window_seconds=300)
_unlock_failures = RateLimiter(max_events=50, window_seconds=300)
def _link_or_404(db: Session, token: str) -> DownloadLink:
def _link_or_404(db: Session, token: str) -> "DownloadLink":
link = linksmod.find(db, token)
if link is None or linksmod.is_expired(link):
raise refused(404, "link_gone")
return link
def _being_made(db: Session, job: DownloadJob) -> bool:
def _being_made(db: Session, job: "DownloadJob") -> bool:
"""The job is running, or queued for a worker that will take it: the worker leaves a suspended
owner's queued jobs where they are (`worker._claim_job`), as long as the suspension lasts."""
if job.status == JobStatus.RUNNING:
return True
if job.status != JobStatus.QUEUED:
return False
owner = db.get(User, job.user_id)
return owner is not None and not owner.is_suspended
return not queries.owner_suspended(db, job)
def _ready_job(db: Session, job: DownloadJob | None) -> tuple[DownloadJob, MediaAsset]:
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
def _ready_job(db: Session, job: "DownloadJob | None") -> tuple["DownloadJob", "MediaAsset"]:
asset = queries.asset_of(db, job)
if (
job is not None
and asset is not None
@@ -78,25 +80,24 @@ def _ready_job(db: Session, job: DownloadJob | None) -> tuple[DownloadJob, Media
return job, asset
def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAsset]:
def _ready_target(db: Session, link: "DownloadLink") -> tuple["DownloadJob", "MediaAsset"]:
"""A file link's file. A folder link names no one file: its files are asked for by id."""
if link.job_id is None:
raise refused(404, "video_gone")
return _ready_job(db, db.get(DownloadJob, link.job_id))
return _ready_job(db, queries.job(db, link.job_id))
def _link_folder(db: Session, link: DownloadLink) -> FileFolder:
folder = db.get(FileFolder, link.folder_id) if link.folder_id is not None else None
def _link_folder(db: Session, link: "DownloadLink") -> "FileFolder":
folder = queries.link_folder(db, link)
if folder is None:
raise refused(404, "link_gone")
return folder
def _folder_job(db: Session, link: DownloadLink, job_id: int) -> tuple[DownloadJob, MediaAsset]:
folder = _link_folder(db, link)
job = db.execute(
select(DownloadJob).where(DownloadJob.id == job_id, *linksmod.in_folder(folder))
).scalar_one_or_none()
def _folder_job(
db: Session, link: "DownloadLink", job_id: int
) -> tuple["DownloadJob", "MediaAsset"]:
job = queries.folder_job(db, _link_folder(db, link), job_id)
if job is None:
raise refused(404, "video_gone")
return _ready_job(db, job)
@@ -113,7 +114,7 @@ def _file_url(token: str, grant: str | None = None, job_id: int | None = None) -
def _poster_url(
token: str, asset: MediaAsset, grant: str | None = None, job_id: int | None = None
token: str, asset: "MediaAsset", grant: str | None = None, job_id: int | None = None
) -> str | None:
if not asset.poster_path:
return None
@@ -121,7 +122,7 @@ def _poster_url(
return f"{url}?g={grant}" if grant else url
def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
def _folder_meta(db: Session, link: "DownloadLink", token: str, grant: str | None) -> dict:
"""A folder link's page: the folder's name and each file's own watch metadata
(`linksmod.folder_files`: ready, on disk, newest first)."""
folder = _link_folder(db, link)
@@ -147,7 +148,7 @@ def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None)
}
def _file_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
def _file_meta(db: Session, link: "DownloadLink", token: str, grant: str | None) -> dict:
job, asset = _ready_target(db, link)
return {
"needs_password": False,
@@ -158,20 +159,14 @@ def _file_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -
}
def _opened(db: Session, link: DownloadLink, token: str, grant: str | None = None) -> dict:
def _opened(db: Session, link: "DownloadLink", token: str, grant: str | None = None) -> dict:
"""The page an open link shows, counted as one view (the handler commits the count)."""
meta = (
_folder_meta(db, link, token, grant)
if link.folder_id is not None
else _file_meta(db, link, token, grant)
)
# In SQL, not `+= 1` on the loaded row: two opens at once would both write the same count.
db.execute(
update(DownloadLink)
.where(DownloadLink.id == link.id)
.values(view_count=DownloadLink.view_count + 1)
.execution_options(synchronize_session=False)
)
linksmod.count_view(db, link)
return meta
@@ -217,13 +212,13 @@ def watch_unlock(
return {"ok": True, **meta}
def _granted(link: DownloadLink, token: str, g: str | None) -> None:
def _granted(link: "DownloadLink", token: str, g: str | None) -> None:
"""A password link's files want the grant its unlock handed out."""
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
raise refused(403, "share_password_required")
def _serve(link: DownloadLink, asset: MediaAsset) -> FileResponse:
def _serve(link: "DownloadLink", asset: "MediaAsset") -> FileResponse:
"""Range-aware stream of a shared file. Inline by default (plays in the page); an
`allow_download` link serves it as an attachment."""
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
@@ -252,7 +247,7 @@ def _serve(link: DownloadLink, asset: MediaAsset) -> FileResponse:
)
def _poster(asset: MediaAsset) -> FileResponse:
def _poster(asset: "MediaAsset") -> FileResponse:
if not asset.poster_path:
raise not_found("poster")
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
+29 -65
View File
@@ -1,15 +1,20 @@
from typing import TYPE_CHECKING
from fastapi import APIRouter, Depends
from pydantic import BaseModel, field_validator
from sqlalchemy import func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.auth import current_user
from app.db import get_db
from app.models import SavedView, User
from app.routes._common import owned_or_404
from app.models import User
from app.reader import queries, service
from app.routes._common import not_found
from app.refusal import refused
if TYPE_CHECKING:
from app.reader.models import SavedView
# Saved "smart views": per-user named snapshots of a module's filters (feed or plex, keyed by
# `source`).
router = APIRouter(prefix="/api/saved-views", tags=["saved-views"])
@@ -23,7 +28,7 @@ def _norm_source(source: str) -> str:
return source if source in _SOURCES else "feed"
def _serialize(v: SavedView) -> dict:
def _serialize(v: "SavedView") -> dict:
return {
"id": v.id,
"name": v.name,
@@ -33,23 +38,20 @@ def _serialize(v: SavedView) -> dict:
}
def _own_view(db: Session, user: User, view_id: int) -> "SavedView":
view = queries.own_view(db, user, view_id)
if view is None:
raise not_found("view")
return view
@router.get("")
def list_views(
source: str = "feed",
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> list[dict]:
source = _norm_source(source)
rows = (
db.execute(
select(SavedView)
.where(SavedView.user_id == user.id, SavedView.source == source)
.order_by(SavedView.position, SavedView.id)
)
.scalars()
.all()
)
return [_serialize(v) for v in rows]
return [_serialize(v) for v in queries.views(db, user, _norm_source(source))]
# A feed view's tag pick is read by more than the feed: the autotag job keeps every tag a view names
@@ -86,22 +88,8 @@ def create_view(
name = body.name.strip()
if not name:
raise refused(400, "name_required")
source = _norm_source(body.source)
# Position is per (user, source), so each module's views number 0..N independently.
maxpos = db.execute(
select(func.coalesce(func.max(SavedView.position), -1)).where(
SavedView.user_id == user.id, SavedView.source == source
)
).scalar_one()
view = SavedView(
user_id=user.id,
source=source,
name=name[:80],
filters=body.filters,
position=maxpos + 1,
)
db.add(view)
try:
view = service.create_view(db, user, _norm_source(body.source), name, body.filters)
db.commit()
except IntegrityError:
db.rollback()
@@ -126,32 +114,17 @@ def update_view(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
view = owned_or_404(db, SavedView, view_id, user, what="view")
view = _own_view(db, user, view_id)
sent = body.model_fields_set
if "name" in sent:
name = (body.name or "").strip()
if not name:
raise refused(400, "name_required")
view.name = name[:80]
if "filters" in sent:
if body.filters is None:
raise refused(400, "filters_not_object")
view.filters = body.filters
if "is_default" in sent:
make_default = bool(body.is_default)
if make_default:
# At most one default per (user, source): clear the flag on this module's others first.
db.execute(
update(SavedView)
.where(
SavedView.user_id == user.id,
SavedView.source == view.source,
SavedView.id != view.id,
)
.values(is_default=False)
)
view.is_default = make_default
name = (body.name or "").strip()
if "name" in sent and not name:
raise refused(400, "name_required")
if "filters" in sent and body.filters is None:
raise refused(400, "filters_not_object")
try:
service.update_view(
db, view, sent=sent, name=name, filters=body.filters, is_default=body.is_default
)
db.commit()
except IntegrityError:
db.rollback()
@@ -165,8 +138,7 @@ def delete_view(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
view = owned_or_404(db, SavedView, view_id, user, what="view")
db.delete(view)
service.delete_view(db, _own_view(db, user, view_id))
db.commit()
return {"deleted": view_id}
@@ -181,14 +153,6 @@ def reorder_views(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
ids = body.ids
rows = db.execute(select(SavedView).where(SavedView.user_id == user.id)).scalars().all()
by_id = {v.id: v for v in rows}
pos = 0
for vid in ids:
v = by_id.get(vid)
if v is not None:
v.position = pos
pos += 1
service.reorder_views(db, user, body.ids)
db.commit()
return {"ok": True}
+1 -3
View File
@@ -29,14 +29,12 @@ from app.models import (
LIVE_OR_UPCOMING,
BlockedChannel,
Channel,
Playlist,
PlaylistItem,
SearchFind,
Subscription,
User,
Video,
VideoState,
)
from app.reader.models import Playlist, PlaylistItem, VideoState
from app.feedquery import feed_columns, serialize
from app.utils import like_escape
from app.sync.subscriptions import apply_channel_details
+24 -32
View File
@@ -1,19 +1,32 @@
from typing import TYPE_CHECKING
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy import func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.auth import current_user
from app.db import get_db
from app.models import ChannelTag, Tag, User
from app.routes._common import owned_or_404
from app.models import User
from app.reader import queries, service
from app.routes._common import not_found
from app.sync.tagsuggest import compute_tag_suggestions
from app.refusal import refused
if TYPE_CHECKING:
from app.reader.models import Tag
router = APIRouter(prefix="/api/tags", tags=["tags"])
def _own_tag(db: Session, user: User, tag_id: int) -> "Tag":
"""One of the user's own tags, or a 404 — a system tag (no owner) is not theirs to edit."""
tag = queries.own_tag(db, user, tag_id)
if tag is None:
raise not_found("tag")
return tag
@router.get("/suggestions")
def tag_suggestions(
user: User = Depends(current_user), db: Session = Depends(get_db)
@@ -27,13 +40,6 @@ def tag_suggestions(
@router.get("")
def list_tags(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
rows = db.execute(
select(Tag, func.count(ChannelTag.id))
.outerjoin(ChannelTag, ChannelTag.tag_id == Tag.id)
.where(or_(Tag.user_id.is_(None), Tag.user_id == user.id))
.group_by(Tag.id)
.order_by(Tag.category, Tag.name)
).all()
return [
{
"id": tag.id,
@@ -43,7 +49,7 @@ def list_tags(user: User = Depends(current_user), db: Session = Depends(get_db))
"system": tag.user_id is None,
"channel_count": count,
}
for tag, count in rows
for tag, count in queries.tags_with_counts(db, user)
]
@@ -62,16 +68,8 @@ def create_tag(
name = body.name.strip()
if not name:
raise refused(400, "name_required")
# An unknown category silently falls back to "other" (unchanged behaviour) rather than 422.
category = body.category if body.category in ("language", "topic", "other") else "other"
tag = Tag(
user_id=user.id,
name=name,
color=body.color,
category=category,
)
db.add(tag)
try:
tag = service.create_tag(db, user, name, body.category, body.color)
db.commit()
except IntegrityError:
db.rollback()
@@ -99,17 +97,13 @@ def update_tag(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
# System tags (user_id NULL) are not user-editable: owned_or_404 404s them too.
tag = owned_or_404(db, Tag, tag_id, user, what="tag")
tag = _own_tag(db, user, tag_id)
sent = body.model_fields_set
if "name" in sent:
name = (body.name or "").strip()
if not name:
raise refused(400, "name_required")
tag.name = name
if "color" in sent:
tag.color = body.color
name = (body.name or "").strip()
if "name" in sent and not name:
raise refused(400, "name_required")
try:
service.update_tag(tag, sent=sent, name=name, color=body.color)
db.commit()
except IntegrityError:
db.rollback()
@@ -129,8 +123,6 @@ def delete_tag(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
# System tags (user_id NULL) are not user-editable: owned_or_404 404s them too.
tag = owned_or_404(db, Tag, tag_id, user, what="tag")
db.delete(tag) # ChannelTag rows cascade on tag delete.
service.delete_tag(db, _own_tag(db, user, tag_id))
db.commit()
return {"deleted": tag_id}
+12 -8
View File
@@ -21,8 +21,8 @@ from app.apitokens import GRANT_COLUMNS, Right
from app.audit import AuditAction
from app.auth import current_user
from app.db import get_db
from app.downloads import folders
from app.models import ApiToken, ApiTokenGrant, FileFolder, User
from app.downloads import folders, queries
from app.models import ApiToken, ApiTokenGrant, User
from app.refusal import refused
from app.routes._common import owned_or_404
from app.security import hash_token
@@ -50,12 +50,16 @@ class TokenIn(BaseModel):
def _serialize(db: Session, token: ApiToken) -> dict:
rows = db.execute(
select(ApiTokenGrant, FileFolder)
.join(FileFolder, FileFolder.id == ApiTokenGrant.folder_id)
.where(ApiTokenGrant.token_id == token.id)
.order_by(FileFolder.id)
).all()
grants = list(
db.execute(
select(ApiTokenGrant)
.where(ApiTokenGrant.token_id == token.id)
.order_by(ApiTokenGrant.folder_id)
).scalars()
)
# The folders are Capture's: read through its queries, not its table.
by_id = queries.folders_by_id(db, {g.folder_id for g in grants})
rows = [(g, by_id[g.folder_id]) for g in grants if g.folder_id in by_id]
now = datetime.now(timezone.utc)
return {
"id": token.id,
+14 -87
View File
@@ -8,11 +8,13 @@ regenerated freely; user tags are never touched here.
import re
from sqlalchemy import and_, exists, func, select
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app import logs, progress, sysconfig
from app.models import Channel, ChannelTag, SavedView, Tag, Video
from app.models import Channel, Video
from app.reader import queries as reader_queries
from app.reader import service as reader_service
log = logs.get_logger(__name__)
@@ -209,71 +211,30 @@ def compute_channel_topics(db: Session, channel: Channel) -> set[str]:
return topics
def ensure_system_tag(db: Session, name: str, category: str, color: str | None) -> Tag:
tag = db.execute(
select(Tag).where(Tag.user_id.is_(None), Tag.name == name)
).scalar_one_or_none()
if tag is None:
tag = Tag(user_id=None, name=name, category=category, color=color)
db.add(tag)
db.flush()
return tag
def apply_channel_autotags(db: Session, channel: Channel) -> None:
desired: dict[int, float | None] = {}
lang, confidence = detect_channel_language(db, channel)
if lang:
tag = ensure_system_tag(db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR)
tag = reader_service.ensure_system_tag(
db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR
)
desired[tag.id] = confidence
for topic in compute_channel_topics(db, channel):
tag = ensure_system_tag(db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR))
tag = reader_service.ensure_system_tag(
db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR)
)
desired.setdefault(tag.id, None)
existing = {
ct.tag_id: ct
for ct in db.execute(
select(ChannelTag).where(
ChannelTag.channel_id == channel.id,
ChannelTag.user_id.is_(None),
ChannelTag.source == "auto",
)
).scalars()
}
for tag_id, ct in existing.items():
if tag_id not in desired:
db.delete(ct)
for tag_id, conf in desired.items():
ct = existing.get(tag_id)
if ct is None:
db.add(
ChannelTag(
channel_id=channel.id,
tag_id=tag_id,
user_id=None,
source="auto",
confidence=conf,
)
)
else:
ct.confidence = conf
reader_service.set_auto_tags(db, channel.id, desired)
db.commit()
def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
query = select(Channel)
if only_missing:
query = query.where(
~exists().where(
and_(
ChannelTag.channel_id == Channel.id,
ChannelTag.source == "auto",
ChannelTag.user_id.is_(None),
)
)
)
query = query.where(~reader_queries.has_auto_tags(Channel.id))
channels = db.execute(query).scalars().all()
tagged = 0
for i, channel in enumerate(channels, 1):
@@ -284,40 +245,6 @@ def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
db.rollback()
log.exception("Auto-tagging failed for channel %s", channel.id)
progress.report(i, len(channels), "autotag")
removed = _cleanup_orphan_system_tags(db)
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
def _view_tag_ids(filters: object) -> list[int]:
"""The tag ids a stored view's filters name; anything else in that slot names none."""
tags = filters.get("tags") if isinstance(filters, dict) else None
if not isinstance(tags, list):
return []
return [t for t in tags if type(t) is int] # `type`, not isinstance: True is not a tag id
def _cleanup_orphan_system_tags(db: Session) -> int:
"""Delete system tags that no longer link to any channel (e.g. after a mapping change) — but not
one a saved feed view filters on. Views hold tag IDS, and a deleted tag comes back under a new id
when a channel earns it again, so the view's filter would point at nothing; the feed drops an
unknown id, and the view showed the whole feed under its old name.
The views are read defensively, and the kept ids are filtered here rather than in SQL: a view
stored before its route checked the shape (or written around it) must not raise for everyone's
job, nor push a `NOT IN` past the driver's bind-parameter limit."""
in_views = {
t
for (filters,) in db.execute(select(SavedView.filters).where(SavedView.source == "feed"))
for t in _view_tag_ids(filters)
}
orphans = [
tag
for tag in db.execute(
select(Tag).where(Tag.user_id.is_(None), ~exists().where(ChannelTag.tag_id == Tag.id))
).scalars()
if tag.id not in in_views
]
for tag in orphans:
db.delete(tag)
removed = reader_service.prune_orphan_system_tags(db)
db.commit()
return len(orphans)
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
+4 -17
View File
@@ -14,15 +14,8 @@ from sqlalchemy import delete, or_, select
from sqlalchemy.orm import Session
from app import logs, sysconfig
from app.models import (
Channel,
ExploredChannel,
PlaylistItem,
SearchFind,
Subscription,
Video,
VideoState,
)
from app.models import Channel, ExploredChannel, SearchFind, Subscription, Video
from app.reader import queries as reader_queries
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
@@ -129,15 +122,12 @@ def purge_unkept_explored(db: Session) -> dict:
if not orphan_ids:
return {"expired": expired, "videos_deleted": 0, "channels_deleted": 0}
state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
find_exists = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists()
videos_deleted = db.execute(
delete(Video).where(
Video.channel_id.in_(orphan_ids),
Video.via_explore.is_(True),
~state_exists,
~pl_exists,
~reader_queries.kept_by_a_reader(Video.id),
~find_exists,
)
).rowcount
@@ -166,15 +156,12 @@ def purge_unkept_search(db: Session, grace_days: int | None = None) -> dict:
0 quota."""
grace = sysconfig.get_int(db, "search_grace_days") if grace_days is None else grace_days
cutoff = datetime.now(timezone.utc) - timedelta(days=grace)
state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
sub_exists = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
deleted = db.execute(
delete(Video).where(
Video.via_search.is_(True),
Video.created_at < cutoff,
~state_exists,
~pl_exists,
~reader_queries.kept_by_a_reader(Video.id),
~sub_exists,
)
).rowcount
+3 -6
View File
@@ -27,7 +27,8 @@ from sqlalchemy.orm import Session
from app import logs, progress, quota, sysconfig
from app.config import settings
from app.models import Playlist, PlaylistItem, Video
from app.models import Video
from app.reader import queries as reader_queries
from app.notifications import create_notification
from app.sync.runner import get_service_user
from app.sync.videos import apply_video_details, parse_dt
@@ -88,11 +89,7 @@ def _collect_user_impact(db: Session, videos: list[Video]) -> dict[int, list[dic
# Anyone who had it in a playlist — which now also covers "saved": Watch later became a
# built-in playlist, so "saved" is no longer a VideoState status (the old status=="saved"
# scan matched nothing and under-counted impacted users).
for user_id, video_id in db.execute(
select(Playlist.user_id, PlaylistItem.video_id)
.join(PlaylistItem, PlaylistItem.playlist_id == Playlist.id)
.where(PlaylistItem.video_id.in_(ids))
):
for user_id, video_id in reader_queries.playlisted_by(db, ids):
impact.setdefault(user_id, set()).add(video_id)
return {
user_id: [
+13 -54
View File
@@ -12,7 +12,6 @@ import json
from collections.abc import Callable, Iterator
from contextlib import AbstractContextManager, contextmanager
from datetime import datetime, timezone
from typing import TypeVar
from sqlalchemy import Engine, create_engine, delete, func, select
@@ -21,12 +20,14 @@ from sqlalchemy.pool import NullPool
from app import logs, progress, quota
from app.auth import has_read_scope
from app.models import Channel, Playlist, PlaylistItem, User, Video
from app.models import User, Video
from app.reader.models import Playlist, PlaylistItem
from app.sync.playlist_merge import merge_plan
from app.sync.runner import token_users
from app.sync.subscriptions import blank_listing, refuse_blank_listing, vanished
from app.sync.videos import apply_video_details, parse_dt
from app.sync.videos import ensure_videos
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
from app.userscope import owned_by
log = logs.get_logger(__name__)
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
@@ -227,55 +228,11 @@ def recompute_dirty(db: Session, pl: Playlist) -> None:
pl.dirty = pl.synced_fingerprint != current_fingerprint(db, pl)
def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
"""Make sure every given video id exists in the catalog, fetching + ingesting the
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
the API won't return are simply left out."""
if not video_ids:
return
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
if not missing:
return
items = yt.get_videos(missing)
chan_ids = {
it.get("snippet", {}).get("channelId")
for it in items
if it.get("snippet", {}).get("channelId")
}
existing_ch = set(
db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
)
now = datetime.now(timezone.utc)
for it in items:
sn = it.get("snippet", {})
cid = sn.get("channelId")
if cid and cid not in existing_ch:
db.add(Channel(id=cid, title=sn.get("channelTitle")))
existing_ch.add(cid)
db.flush()
seen: set[str] = set()
for it in items:
vid = it.get("id")
sn = it.get("snippet", {})
cid = sn.get("channelId")
if not vid or not cid or vid in seen:
continue
seen.add(vid)
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
# fallback here too so the constructor never carries an explicit NULL.
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
apply_video_details(v, it)
v.enriched_at = now
db.add(v)
db.commit()
def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
(`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
(`sync.videos.ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
present = (
@@ -346,7 +303,9 @@ def pull_user_playlists(db: Session, user: User) -> dict:
existing = {
pl.yt_playlist_id: pl
for pl in db.execute(
select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube")
select(Playlist).where(
owned_by(Playlist.user_id, user.id), Playlist.source == "youtube"
)
).scalars()
}
# YouTube playlists that we already own as local (exported) playlists must not be
@@ -354,7 +313,7 @@ def pull_user_playlists(db: Session, user: User) -> dict:
owned_yt_ids = set(
db.execute(
select(Playlist.yt_playlist_id).where(
Playlist.user_id == user.id,
owned_by(Playlist.user_id, user.id),
Playlist.source == "local",
Playlist.yt_playlist_id.is_not(None),
)
@@ -401,7 +360,7 @@ def pull_user_playlists(db: Session, user: User) -> dict:
continue
# Ingest first — it commits — so the row lock below is held from the dirty check
# to the commit that replaces the items.
_ensure_videos(db, yt, ids)
ensure_videos(db, yt, ids)
# `dirty` read above is a snapshot from before the YouTube calls (sessions keep
# loaded rows past a commit): a remove or reorder made meanwhile must not be
# replaced by YouTube's order. Item writes lock the playlist row, so this waits for
@@ -612,7 +571,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
prev_unavailable = parse_unavailable(pl)
still_listed = [v for v in yt_ids if v in prev_unavailable]
if still_listed:
_ensure_videos(db, yt, still_listed)
ensure_videos(db, yt, still_listed)
healed = set(
db.execute(select(Video.id).where(Video.id.in_(still_listed))).scalars().all()
)
@@ -624,7 +583,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
# --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
# merged order authoritatively. Do this first so local reflects the merge even if a later
# YouTube write trips the breaker.
_ensure_videos(db, yt, mp.add_to_local)
ensure_videos(db, yt, mp.add_to_local)
# `local` was read before the YouTube calls above: an add or remove committed meanwhile
# (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
# behind us — and plan again from what is there now. Nothing new needs ingesting: a video
@@ -749,7 +708,7 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
)
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
refuse_blank_listing(ids, snippet.get("item_count"))
_ensure_videos(db, yt, ids)
ensure_videos(db, yt, ids)
_lock_or_gone(db, pl)
pl.name = snippet.get("title") or pl.name
ordered = _replace_items_from_youtube(db, pl, ids)
+7 -3
View File
@@ -23,7 +23,9 @@ from collections import Counter
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.models import ChannelTag, Subscription, Tag, User
from app.models import Subscription, User
from app.reader.models import ChannelTag, Tag
from app.userscope import owned_by
# A user tag needs at least this many manually-tagged example channels before we trust its
# profile — fewer is noise (one gaming-cooking channel shouldn't make "cooking" mean "gaming").
@@ -57,7 +59,9 @@ def compute_tag_suggestions(db: Session, user: User) -> dict[str, list[dict]]:
# Channel Manager shows ghost chips only on non-hidden rows, so suggesting for hidden channels would
# inflate the header count past what's shown and let bulk-accept tag channels the user can't see).
subs = db.execute(
select(Subscription.channel_id, Subscription.hidden).where(Subscription.user_id == user.id)
select(Subscription.channel_id, Subscription.hidden).where(
owned_by(Subscription.user_id, user.id)
)
).all()
subscribed: set[str] = {cid for cid, _ in subs}
hidden: set[str] = {cid for cid, h in subs if h}
@@ -74,7 +78,7 @@ def compute_tag_suggestions(db: Session, user: User) -> dict[str, list[dict]]:
user_tagged: set[str] = set()
for tag_id, cid in db.execute(
select(ChannelTag.tag_id, ChannelTag.channel_id).where(
ChannelTag.user_id == user.id, ChannelTag.source == "user"
owned_by(ChannelTag.user_id, user.id), ChannelTag.source == "user"
)
).all():
if cid not in subscribed:
+45
View File
@@ -418,3 +418,48 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
progress.report(probed, len(candidates), "probe")
db.commit()
return {"probed": probed, "shorts": shorts}
def ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
"""Make sure every given video id exists in the catalog, fetching + ingesting the
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
the API won't return are simply left out. The catalog's write for a playlist sync that names
videos it has never seen; commits."""
if not video_ids:
return
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
if not missing:
return
items = yt.get_videos(missing)
chan_ids = {
it.get("snippet", {}).get("channelId")
for it in items
if it.get("snippet", {}).get("channelId")
}
existing_ch = set(
db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
)
now = datetime.now(timezone.utc)
for it in items:
sn = it.get("snippet", {})
cid = sn.get("channelId")
if cid and cid not in existing_ch:
db.add(Channel(id=cid, title=sn.get("channelTitle")))
existing_ch.add(cid)
db.flush()
seen: set[str] = set()
for it in items:
vid = it.get("id")
sn = it.get("snippet", {})
cid = sn.get("channelId")
if not vid or not cid or vid in seen:
continue
seen.add(vid)
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
# fallback here too so the constructor never carries an explicit NULL.
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
apply_video_details(v, it)
v.enriched_at = now
db.add(v)
db.commit()
+10
View File
@@ -0,0 +1,10 @@
"""Every table, registered on `Base.metadata`.
Each unit declares its tables in its own `models` module, so importing `app.models` alone no longer
names the whole schema. Whatever needs all of it — the migrations, the test database, a check over
every mapper — imports this module.
"""
import app.downloads.models # Capture's tables
import app.models # the core tables
import app.reader.models # noqa: F401 — Reader's tables
+73 -5
View File
@@ -1,13 +1,54 @@
"""Shared "who is an active, non-suspended account" predicate.
"""Whose a row is, and who an account is to the others — one place for both.
Used wherever a user must be filtered out of a recipient list or rejected as a target: the download
share-by-email and recipient-picker endpoints. Kept in one place so the two forms (SQL clauses for a
query, Python check for a loaded row) can't drift apart, and so a route module doesn't have to import
another route module just to reuse the rule.
**Ownership.** A user-owned table says whose a row is in one column (`user_id`, or `to_user_id` for
something handed to them). A unit's reads and writes name that column through `owned_by` and fetch a
row by id through `owned_row`, never with an inline `Model.user_id == …`: the rule "a row owned by
someone else is answered like a missing one" then lives here, and a contract test
(`test_unit_fences.py`) holds a switched-on unit to it.
**Recipients.** "An active, non-suspended account" — the download share-by-email and the recipient
picker filter on it. Kept here so the two forms (SQL clauses for a query, a Python check for a loaded
row) can't drift apart.
"""
from typing import Any, TypeVar
from sqlalchemy import ColumnElement, func, select
from sqlalchemy.orm import InstrumentedAttribute, Session
from app.collation import collated
from app.db import Base
from app.models import User
M = TypeVar("M", bound=Base)
def _owner_id(owner: User | int) -> int:
return owner if isinstance(owner, int) else owner.id
def owned_by(column: InstrumentedAttribute, owner: User | int) -> ColumnElement[bool]:
"""`column` — a table's owner column — names `owner`."""
return column == _owner_id(owner)
def owned_row(
db: Session,
model: type[M],
id_: Any,
owner: User | int,
*,
owner_attr: str = "user_id",
lock: bool = False,
) -> M | None:
"""The row of `model` with this primary key when `owner` owns it, else None — a row of someone
else's is the same answer as a missing one, so a caller can never tell the two apart. `lock`
takes it FOR UPDATE (past the identity map)."""
row = db.get(model, id_, with_for_update=True if lock else None)
if row is None or getattr(row, owner_attr) != _owner_id(owner):
return None
return row
def messageable_clauses() -> list:
"""SQL clauses for "an active, non-suspended account" — splat into a recipient query."""
@@ -20,3 +61,30 @@ def messageable_clauses() -> list:
def is_messageable_user(u: User | None) -> bool:
"""Python mirror of `messageable_clauses()` for an already-loaded row (a share recipient)."""
return bool(u and u.is_active and not u.is_suspended)
def user_by_email(db: Session, email: str) -> User | None:
"""The account with this address, matched without regard to case."""
return db.execute(
select(User).where(func.lower(User.email) == email.strip().lower())
).scalar_one_or_none()
def messageable_others(db: Session, user: User, lang: str) -> list[User]:
"""Every account `user` may hand something to: active, not suspended, not themselves; in the
reader's alphabet by address."""
return list(
db.execute(
select(User)
.where(User.id != user.id, *messageable_clauses())
.order_by(collated(User.email, lang))
).scalars()
)
def emails_of(db: Session, ids: set[int] | None = None) -> dict[int, str]:
"""Each account's address by id, in address order: these ids, or every account."""
stmt = select(User.id, User.email).order_by(User.email)
if ids is not None:
stmt = stmt.where(User.id.in_(ids))
return dict(db.execute(stmt).tuples().all())
+2 -1
View File
@@ -46,7 +46,8 @@ from app.downloads.jobdb import (
_staging_dir,
)
from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
from app.models import DISPLAY_TEXT_MAX
from app.downloads.models import DownloadJob, MediaAsset
log = logs.get_logger(__name__)
_stop = threading.Event()
+1 -1
View File
@@ -25,7 +25,7 @@ from fastapi.testclient import TestClient
from sqlalchemy import create_engine, event, text
from starlette.requests import HTTPConnection
import app.models # register every model on Base.metadata
import app.tables # every table on Base.metadata
import app.state as app_state
from app import apitokens, security
from app.auth import current_user
+1 -1
View File
@@ -106,7 +106,7 @@ def test_the_single_playlist_repull_shares_the_pull_cooldown(db, as_user, make_u
# SEC-14, the sweep half: `revert-youtube` spends under the SAME QuotaAction.PLAYLISTS_PULL key
# as the bulk `sync-youtube`, so limiting only one of them left the identical drain open one
# route over. Both must draw on the one allowance.
from app.models import Playlist
from app.reader.models import Playlist
from app.routes import playlists as plroutes
user = make_user()
+2 -10
View File
@@ -25,16 +25,8 @@ from app.config import settings
from app.db import SessionLocal, get_db
from app.main import app
from app.notifications import create_notification
from app.models import (
ApiToken,
ApiTokenGrant,
AuditLog,
DownloadJob,
FileFolder,
MediaAsset,
Notification,
User,
)
from app.models import ApiToken, ApiTokenGrant, AuditLog, Notification, User
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
from app.ratelimit import RateLimiter
from app.routes import tokens as tokens_routes
from app.routes.auth import bump_session_epoch
+10 -6
View File
@@ -1,4 +1,4 @@
"""sync/autotag's orphan cleanup (the module was 32% covered).
"""The auto-tag job's orphan cleanup (`reader.service.prune_orphan_system_tags`).
A system tag no channel carries any more is deleted — unless a saved feed view filters on it. The
views hold tag ids; a deleted tag returns under a NEW id when a channel earns it again, and the feed
@@ -7,8 +7,9 @@ drops an unknown id, so the view silently showed the whole feed under its old na
from sqlalchemy import select
from app.models import Channel, ChannelTag, SavedView, Tag
from app.sync import autotag
from app.models import Channel
from app.reader.models import ChannelTag, SavedView, Tag
from app.reader import service as reader_service
def test_an_orphan_system_tag_goes_but_one_a_saved_view_filters_on_stays(db, make_user):
@@ -30,7 +31,8 @@ def test_an_orphan_system_tag_goes_but_one_a_saved_view_filters_on_stays(db, mak
db.add(SavedView(user_id=user.id, source="plex", name="P", filters={"genres": ["Drama"]}))
db.commit()
assert autotag._cleanup_orphan_system_tags(db) == 1
assert reader_service.prune_orphan_system_tags(db) == 1
db.commit()
db.expire_all()
names = set(db.execute(select(Tag.name)).scalars())
assert names == {"Gaming", "Music", "Mine"} # only the unreferenced system orphan went
@@ -48,7 +50,8 @@ def test_a_malformed_saved_view_cannot_break_the_cleanup(db, make_user):
):
db.add(SavedView(user_id=user.id, source="feed", name=f"bad{i}", filters=filters))
db.commit()
assert autotag._cleanup_orphan_system_tags(db) == 1
assert reader_service.prune_orphan_system_tags(db) == 1
db.commit()
db.expire_all()
assert db.execute(select(Tag).where(Tag.name == "Cooking")).first() is None
@@ -61,4 +64,5 @@ def test_a_view_kept_tag_list_larger_than_a_bind_parameter_limit_still_cleans_up
huge = list(range(10_000_000, 10_070_000)) # past psycopg's 65535 bind parameters
db.add(SavedView(user_id=user.id, source="feed", name="huge", filters={"tags": huge}))
db.commit()
assert autotag._cleanup_orphan_system_tags(db) == 1
assert reader_service.prune_orphan_system_tags(db) == 1
db.commit()
+295
View File
@@ -0,0 +1,295 @@
"""Capture's service and queries, one use case each, below the HTTP layer.
The routes are thin callers now, and their own tests stay the contract a client sees. These hold
what each write does and leaves on its own: that it flushes and leaves the commit to its caller,
that a row of someone else's reads as a missing one, and the rule each write carries (a hash kept
once, a password change that retires the old grants, a quota floored). DB-backed (real Postgres).
"""
import pytest
from sqlalchemy import select
from app.audit import AuditAction
from app.downloads import folders, links, queries, quota, service
from app.downloads.models import (
DownloadJob,
DownloadLink,
DownloadQuota,
DownloadShare,
FileFolder,
LibraryState,
MediaAsset,
)
from app.models import AuditLog, Notification
from app.userscope import owned_by, owned_row
def _file(db, user, *, sha256=None) -> DownloadJob:
asset = MediaAsset(
source_kind="url",
source_ref=f"https://example.com/{user.id}.mp4",
format_sig=f"sig-{user.id}",
status="ready",
rel_path=f"f-{user.id}.mp4",
ref_count=1,
media_kind="video",
container="mp4",
sha256=sha256,
)
db.add(asset)
db.flush()
job = DownloadJob(
user_id=user.id,
asset_id=asset.id,
source_kind="url",
source_ref=asset.source_ref,
status="done",
profile_snapshot={},
display_name="Clip",
)
db.add(job)
db.commit()
return job
def _uncommitted(db):
"""What the caller still has to commit: the session holds pending work."""
return bool(db.new or db.dirty or db.deleted) or db.in_transaction()
# --- whose a row is ------------------------------------------------------------------------
def test_a_row_of_someone_elses_reads_as_a_missing_one(db, make_user):
owner, stranger = make_user(), make_user()
job = _file(db, owner)
assert owned_row(db, DownloadJob, job.id, owner) is job
assert owned_row(db, DownloadJob, job.id, stranger) is None
assert owned_row(db, DownloadJob, job.id + 1000, owner) is None
mine = db.execute(select(DownloadJob.id).where(owned_by(DownloadJob.user_id, owner.id)))
assert mine.scalars().all() == [job.id]
def test_a_file_is_readable_by_its_owner_and_whoever_it_was_shared_with(db, make_user):
owner, friend, stranger = make_user(), make_user(), make_user()
job = _file(db, owner)
assert service.share(db, job, owner, friend) is True
db.commit()
assert queries.accessible_job(db, owner, job.id) is job
assert queries.accessible_job(db, friend, job.id) is job
assert queries.accessible_job(db, stranger, job.id) is None
assert queries.own_job(db, friend, job.id) is None # shared is not owned
def test_the_tray_and_the_library_list_only_the_callers_own_files(db, make_user):
from datetime import datetime, timedelta, timezone
mine, theirs = make_user(), make_user()
own, other = _file(db, mine), _file(db, theirs)
since = datetime.now(timezone.utc) - timedelta(days=1)
for job in (own, other):
job.ready_at = datetime.now(timezone.utc)
db.commit()
assert [j.id for j in queries.queue(db, mine, since)] == [own.id]
library = queries.library(
db,
mine,
folder_id=None,
listable=None,
shared=False,
term="",
sort="added",
order="desc",
lang="en",
)
assert [j.id for j in library] == [own.id]
assert [r for r, _ in queries.statuses_by_source(db, mine, ["done"])] == [own.source_ref]
# --- locks -----------------------------------------------------------------------------------
def test_the_job_lock_answers_none_for_a_job_not_the_callers(db, make_user):
owner, stranger = make_user(), make_user()
job = _file(db, owner)
assert service.lock_own_job(db, stranger, job.id) is None
assert service.lock_own_job(db, owner, job.id) is job
assert service.hold_for_share(db, stranger, job.id) is None
assert service.hold_for_share(db, owner, job.id) is job
# --- profiles --------------------------------------------------------------------------------
def test_a_profile_is_made_after_the_users_others_and_only_theirs_to_change(db, make_user):
user, other = make_user(), make_user()
first = service.create_profile(db, user, "x" * 100, {"kind": "video"})
second = service.create_profile(db, user, "Audio", {"kind": "audio"})
assert _uncommitted(db)
db.commit()
assert first.name == "x" * 80 # names are capped
assert second.sort_order == first.sort_order + 1
assert queries.own_profile(db, other, first.id) is None
service.update_profile(first, name="Video", spec=None)
db.commit()
assert (first.name, first.spec) == ("Video", {"kind": "video"})
service.delete_profile(db, first)
db.commit()
assert [p.id for p in queries.profiles_for(db, user)] == [second.id]
# --- resume points ---------------------------------------------------------------------------
def test_a_resume_point_is_one_row_per_user_and_file_and_a_zero_clears_it(db, make_user):
user = make_user()
job = _file(db, user)
service.set_position(db, user, job.id, 40)
service.set_position(db, user, job.id, 90)
db.commit()
rows = db.execute(select(LibraryState.position_seconds)).scalars().all()
assert rows == [90]
assert queries.positions_for(db, user, [job]) == {job.id: 90}
service.set_position(db, user, job.id, 0)
service.set_position(db, user, job.id, 0) # the beacon's second clear deletes nothing
db.commit()
assert db.execute(select(LibraryState)).first() is None
# --- shares ----------------------------------------------------------------------------------
def test_a_share_is_made_once_with_one_notice_and_undone_by_either_side(db, make_user):
owner, friend = make_user(), make_user()
job = _file(db, owner)
assert service.share(db, job, owner, friend) is True
assert service.share(db, job, owner, friend) is False
assert _uncommitted(db)
db.commit()
notices = db.execute(select(Notification).where(Notification.user_id == friend.id)).scalars()
assert [n.data["job_id"] for n in notices] == [job.id]
assert service.unshare(db, job.id, friend) is True
assert service.unshare(db, job.id, friend) is False
db.commit()
assert db.execute(select(DownloadShare)).first() is None
# --- a file's display fields, its access clock, its hash ------------------------------------
def test_only_display_fields_can_be_set_on_a_file(db, make_user):
job = _file(db, make_user())
service.set_display(job, display_name="New", extra_links=None)
assert job.display_name == "New"
with pytest.raises(ValueError):
service.set_display(job, status="error")
assert job.status == "done"
def test_a_hash_is_kept_once_and_never_overwritten(db, make_user):
job = _file(db, make_user())
service.store_sha256(db, job.asset_id, "a" * 64)
service.store_sha256(db, job.asset_id, "b" * 64)
db.commit()
assert db.get(MediaAsset, job.asset_id).sha256 == "a" * 64
def test_serving_a_file_restarts_its_idle_clock(db, make_user):
job = _file(db, make_user())
asset = queries.asset_of(db, job)
before = asset.last_access_at
service.touch(asset)
assert asset.last_access_at is not None and asset.last_access_at != before
# --- links -----------------------------------------------------------------------------------
def _actions(db) -> list[str]:
return db.execute(select(AuditLog.action).order_by(AuditLog.id)).scalars().all()
def test_a_link_is_made_changed_and_revoked_with_an_audit_row_each(db, make_user):
user = make_user()
job = _file(db, user)
link, token = links.create(
db, user, password="pw", allow_download=False, expires_days=3, job_id=job.id
)
db.commit()
assert links.find(db, token) is link
assert link.expires_at is not None and link.password_hash is not None
version = link.password_version
links.update(db, user, link, password="other", sent=frozenset({"password"}))
db.commit()
assert link.password_version == version + 1 # the old grants stop working
links.update(db, user, link, allow_download=True, sent=frozenset()) # nothing was sent
assert link.allow_download is False
assert [link.id for link in queries.links_to(db, job_id=job.id)] == [link.id]
links.revoke(db, user, link)
db.commit()
assert db.execute(select(DownloadLink)).first() is None
assert _actions(db) == [
AuditAction.DOWNLOAD_LINK_CREATE,
AuditAction.DOWNLOAD_LINK_UPDATE,
AuditAction.DOWNLOAD_LINK_REVOKE,
]
def test_a_link_is_the_callers_only_through_a_file_or_folder_of_theirs(db, make_user):
owner, stranger = make_user(), make_user()
job = _file(db, owner)
link, _ = links.create(
db, owner, password="", allow_download=True, expires_days=None, job_id=job.id
)
db.commit()
assert queries.own_link(db, owner, link.id) is link
assert queries.own_link(db, stranger, link.id) is None
def test_a_view_is_counted_in_sql(db, make_user):
user = make_user()
job = _file(db, user)
link, _ = links.create(
db, user, password="", allow_download=True, expires_days=None, job_id=job.id
)
db.commit()
links.count_view(db, link)
links.count_view(db, link)
db.commit()
db.expire_all()
assert db.get(DownloadLink, link.id).view_count == 2
# --- folders ---------------------------------------------------------------------------------
def test_a_folder_name_is_one_per_user_and_a_source_folder_keeps_its_own(db, make_user):
user, other = make_user(), make_user()
mine = folders.create_own(db, user.id, "Trips")
with pytest.raises(folders.FolderError, match="file_folder_name_taken"):
folders.create_own(db, user.id, "trips")
folders.create_own(db, other.id, "Trips") # another user's namesake is theirs
db.commit()
source = folders.source_folder(db, user.id, folders.SourceKey.UPLOAD)
with pytest.raises(folders.FolderError, match="file_folder_not_own"):
folders.rename(db, source, "Mine now")
folders.rename(db, mine, "Holidays")
db.commit()
assert db.get(FileFolder, mine.id).name == "Holidays"
assert queries.own_folder(db, other, mine.id) is None
# --- quota -----------------------------------------------------------------------------------
def test_a_custom_quota_starts_from_what_applies_and_floors_each_value(db, make_user):
user = make_user()
defaults = quota.resolve(db, user.id)
quota.set_custom(db, user.id, max_concurrent=0, max_bytes=-5)
db.commit()
row = queries.custom_quota(db, user.id)
assert (row.max_concurrent, row.max_bytes, row.max_jobs) == (1, 0, defaults.max_jobs)
assert quota.reset_custom(db, user.id) is True
assert quota.reset_custom(db, user.id) is False
db.commit()
assert db.get(DownloadQuota, user.id) is None
+2 -1
View File
@@ -14,7 +14,8 @@ import app.state as state
from app.auth import current_user
from app.db import get_db
from app.main import app
from app.models import Channel, ChannelTag, Tag, User
from app.models import Channel, User
from app.reader.models import ChannelTag, Tag
def test_channel_detail_returns_only_this_users_tag_ids(db, monkeypatch, override_dep):
+2 -1
View File
@@ -18,7 +18,8 @@ from app.downloads import errors as dlerrors
from app.downloads import formats, pipeline, quota, runner, service
from app.downloads import probe as probemod
from app.downloads.edit import INPUT_GUARD
from app.models import Channel, DownloadJob, MediaAsset, PlexItem, PlexLibrary, PlexShow, Video
from app.models import Channel, PlexItem, PlexLibrary, PlexShow, Video
from app.downloads.models import DownloadJob, MediaAsset
from app.plex import paths as plex_paths
# --- the spec -----------------------------------------------------------------------------------
+2 -11
View File
@@ -9,17 +9,8 @@ from app.auth import current_user
from app.collation import sorted_words
from app.db import get_db
from app.main import app
from app.models import (
Channel,
Playlist,
PlaylistItem,
PlexCollection,
PlexItem,
PlexLibrary,
Subscription,
User,
Video,
)
from app.models import Channel, PlexCollection, PlexItem, PlexLibrary, Subscription, User, Video
from app.reader.models import Playlist, PlaylistItem
from datetime import datetime, timezone
WORDS = ["Zorró", "Életrajzi", "Ötödik", "oroszlán", "Csillag", "Cukor"]
+1 -4
View File
@@ -20,18 +20,15 @@ from app.plex.cards import leaf_card
from app.models import (
BlockedChannel,
Channel,
ComfortPin,
PlexItem,
PlexLibrary,
PlexShow,
PlexState,
Playlist,
PlaylistItem,
Subscription,
User,
Video,
VideoState,
)
from app.reader.models import ComfortPin, Playlist, PlaylistItem, VideoState
NOW = datetime(2026, 6, 1, tzinfo=timezone.utc)
+2 -1
View File
@@ -18,7 +18,8 @@ from cryptography.fernet import Fernet
from app import security
from app.downloads import credentials, formats, service
from app.models import DownloadCredential, DownloadJob, MediaAsset, User
from app.models import User
from app.downloads.models import DownloadCredential, DownloadJob, MediaAsset
@pytest.fixture(autouse=True)
@@ -24,7 +24,8 @@ from app import security
from app.config import settings
from app.db import engine
from app.downloads import credentials, formats, pipeline, service
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
from app.models import User
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
from app.worker import _process_job
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
+3 -1
View File
@@ -18,7 +18,8 @@ from sqlalchemy import select
from app.apitokens import FilesCaller
from app.db import SessionLocal
from app.downloads import service
from app.models import DownloadJob, MediaAsset, User
from app.models import User
from app.downloads.models import DownloadJob, MediaAsset
from app.routes.downloads import cancel_download, pause_download
from app.routes.files import delete_file, own_job_for_update
@@ -88,6 +89,7 @@ def test_overlapping_requests_for_one_job_release_its_hold_once(
# The second request waits for the first one's locks instead of deciding on the same row.
assert not finished.wait(0.5), "the second request did not wait for the first"
transition(first, locked)
first.commit() # the handler's commit: the service only flushes
other.join(10)
assert finished.is_set()
finally:
+4 -2
View File
@@ -13,7 +13,8 @@ its OWN session, so every test COMMITS its setup before calling it.
from app.config import settings
from app.downloads import errors as dlerrors
from app.downloads import formats, pipeline, quota
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
from app.models import User
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
from app.worker import _claim_job
@@ -179,6 +180,7 @@ def test_a_suspended_users_queued_jobs_wait(db):
def test_list_downloads_is_capped(db, monkeypatch):
from app.downloads import queries
from app.routes import downloads as dl
u = User(email="heavy@example.com")
@@ -188,7 +190,7 @@ def test_list_downloads_is_capped(db, monkeypatch):
_job(db, u.id, queue_pos=i, status="done")
db.commit()
monkeypatch.setattr(dl, "MAX_LIST", 2)
monkeypatch.setattr(queries, "MAX_LIST", 2)
rows = dl.list_downloads(user=u, db=db)
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
+2 -1
View File
@@ -13,7 +13,8 @@ DB lane (`db` fixture) — skipped without Postgres.
from sqlalchemy import select, text
from app.downloads import service
from app.models import DownloadJob, MediaAsset, User
from app.models import User
from app.downloads.models import DownloadJob, MediaAsset
def _user(db, email="ref@example.com"):
+2 -1
View File
@@ -4,7 +4,8 @@ you") — the only place the title lived, so a Hungarian reader got the English.
from sqlalchemy import select
from app.models import DownloadJob, MediaAsset, Notification
from app.models import Notification
from app.downloads.models import DownloadJob, MediaAsset
def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user):
+2 -1
View File
@@ -8,7 +8,8 @@ behaviour-equivalently. DB lane (`db` fixture) — skipped without Postgres.
from sqlalchemy import text
from app.models import DownloadJob, MediaAsset, User
from app.models import User
from app.downloads.models import DownloadJob, MediaAsset
def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None):
+2 -1
View File
@@ -11,7 +11,8 @@ from sqlalchemy import text
from app.downloads import service
from app.downloads.status import JobStatus
from app.models import DownloadJob, User
from app.models import User
from app.downloads.models import DownloadJob
def _running_job(db) -> DownloadJob:
+1 -1
View File
@@ -13,7 +13,7 @@ from app.config import settings
from app.downloads import errors as dlerrors
from app.downloads import pipeline, quota, service
from app.downloads import upload as uploadmod
from app.models import DownloadJob, DownloadQuota, MediaAsset
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
+2 -9
View File
@@ -12,15 +12,8 @@ contradicts the remote's own count is not believed either.
import pytest
from sqlalchemy import select
from app.models import (
Channel,
Playlist,
PlaylistItem,
PlexCollection,
PlexItem,
PlexLibrary,
Video,
)
from app.models import Channel, PlexCollection, PlexItem, PlexLibrary, Video
from app.reader.models import Playlist, PlaylistItem
from app.plex import sync as plex_sync
from app.sync import playlists as plsync
from app.youtube.client import YouTubeError
+1 -1
View File
@@ -20,7 +20,7 @@ import yt_dlp
from app import logs, sysconfig, worker
from app.downloads import pipeline
from app.downloads.stated import StatedFailure
from app.models import DownloadJob, MediaAsset
from app.downloads.models import DownloadJob, MediaAsset
from app.plex import stream as plex_stream
+2 -2
View File
@@ -1,7 +1,7 @@
"""Feed cursor endpoint: a malformed ?cursor= must be a clean HTTP 400, not a 500.
Locks the refactor that split cursor handling across modules — feedquery.decode_cursor raises a
plain ValueError and routes/feed.get_feed translates it to 400 — so a future regression (a dropped
Cursor handling spans modules — feedquery.decode_cursor raises a plain ValueError, feed_page turns
it into BadCursor, and routes/feed.get_feed translates that to 400 — so a regression (a dropped
try/except, or a new decode_cursor caller without its own catch) turns the 400 back into a 500 and
fails right here. DB lane (`db` fixture) — skipped without Postgres.
"""
+2 -1
View File
@@ -14,7 +14,8 @@ import pytest
from app import feedquery
from app.models import Channel, ChannelTag, Subscription, Tag, Video, VideoState
from app.models import Channel, Subscription, Video
from app.reader.models import ChannelTag, Tag, VideoState
def _dt(day: int) -> datetime:
+2 -1
View File
@@ -17,7 +17,8 @@ from app.downloads import folders, formats, service
from app.downloads import gc as gcmod
from app.db import SessionLocal
from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset, Notification
from app.models import Notification
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
NOW = datetime.now(timezone.utc)
+2 -1
View File
@@ -18,7 +18,8 @@ from app.config import settings
from app.downloads import folders, service
from app.downloads import upload as uploadmod
from app.downloads.status import JobStatus
from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset
from app.models import AuditLog
from app.downloads.models import DownloadJob, DownloadShare, FileFolder, MediaAsset
NOW = datetime.now(timezone.utc)
+1 -1
View File
@@ -11,7 +11,7 @@ from sqlalchemy import select
from app.config import settings
from app.downloads import folders, preview, service
from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadShare, MediaAsset
from app.downloads.models import DownloadJob, DownloadShare, MediaAsset
from app.routes.files import TRUNCATED_HEADER
+4 -3
View File
@@ -18,7 +18,8 @@ from app.config import settings
from app.db import engine
from app.downloads import folders, og
from app.downloads import probe as probemod
from app.models import AuditLog, DownloadJob, DownloadLink, FileFolder, MediaAsset
from app.models import AuditLog
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
_VIDEO = {
"streams": [
@@ -193,7 +194,7 @@ def test_every_open_counts_as_a_view(shelf, client):
def test_the_folder_list_is_capped_like_the_owners_files_list():
from app.downloads import links as linksmod
from app.routes.files import MAX_LIST
from app.downloads.queries import MAX_LIST
assert linksmod.FOLDER_MAX == MAX_LIST
@@ -317,7 +318,7 @@ def test_a_password_file_link_says_it_opens_a_file(shelf, client):
def test_a_file_whose_bytes_are_gone_is_not_offered(shelf, client, db):
from pathlib import Path
from app.models import MediaAsset
from app.downloads.models import MediaAsset
token = _link(shelf)["token"]
gone = shelf["inside"][0]
+1 -1
View File
@@ -9,7 +9,7 @@ cannot reach, is ever read or written. DB-backed (real Postgres).
import pytest
from sqlalchemy import select
from app.models import DownloadJob, DownloadShare, LibraryState, MediaAsset
from app.downloads.models import DownloadJob, DownloadShare, LibraryState, MediaAsset
from app.routes._common import ProgressIn, kept_position
+2 -1
View File
@@ -18,7 +18,8 @@ from app import security
from app.config import settings
from app.db import engine
from app.downloads import probe as probemod
from app.models import AuditLog, DownloadLink
from app.models import AuditLog
from app.downloads.models import DownloadLink
_VIDEO = {
"streams": [
+2 -1
View File
@@ -12,7 +12,8 @@ from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import select
from app.models import Channel, Notification, Playlist, PlaylistItem, Video
from app.models import Channel, Notification, Video
from app.reader.models import Playlist, PlaylistItem
from app.sync import maintenance
+1 -1
View File
@@ -7,7 +7,7 @@ the shared one, so every other owner check kept the English. Needs the DB.
"""
from app.auth import require_write_scope
from app.models import DownloadJob, MediaAsset
from app.downloads.models import DownloadJob, MediaAsset
def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user):
+1 -1
View File
@@ -17,7 +17,7 @@ import pytest
from app.config import settings
from app.downloads import pipeline, playable, probe, service, storage
from app.downloads import upload as uploadmod
from app.models import DownloadJob, MediaAsset
from app.downloads.models import DownloadJob, MediaAsset
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
+1 -1
View File
@@ -17,7 +17,7 @@ import pytest
from app.config import settings
from app.downloads import pipeline, playable, probe
from app.models import DownloadJob, MediaAsset
from app.downloads.models import DownloadJob, MediaAsset
APP = Path(__file__).resolve().parents[1] / "app"
+2 -1
View File
@@ -8,7 +8,8 @@ the position is an index over the ORDERED rows, and the list is renumbered aroun
from datetime import datetime, timezone
from app.models import Channel, Playlist, PlaylistItem, Video
from app.models import Channel, Video
from app.reader.models import Playlist, PlaylistItem
def _seed(db, user, ids, *, positions=None, **kw):
+4 -3
View File
@@ -10,7 +10,8 @@ from datetime import datetime, timezone
import pytest
from app.models import Channel, Playlist, PlaylistItem, Video
from app.models import Channel, Video
from app.reader.models import Playlist, PlaylistItem
from app.sync import playlists as pl_mod
from app.sync.playlist_merge import merge_plan
from app.sync.playlists import (
@@ -74,7 +75,7 @@ class TestMergePlan:
class FakeYouTube:
"""Enough of YouTubeClient for merge_playlist: an in-memory playlist of (item_id, video_id)
and the write ops that mutate it. Every catalog video is pre-seeded so _ensure_videos never
and the write ops that mutate it. Every catalog video is pre-seeded so ensure_videos never
needs get_videos."""
def __init__(
@@ -88,7 +89,7 @@ class FakeYouTube:
self.title = title
self._seq = 0
self.items = [(self._new_id(), v) for v in items]
# Ids the Data API won't return (private/deleted) — get_videos omits them, so _ensure_videos
# Ids the Data API won't return (private/deleted) — get_videos omits them, so ensure_videos
# can't ingest them. They still live in the playlist (`items`) like a real un-ingestable video.
self.unavailable = set(unavailable or ())
# Ids whose add_playlist_item raises, to exercise the partial-failure baseline-hold path.
+4 -3
View File
@@ -16,7 +16,8 @@ from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy.exc import OperationalError
from app.models import Channel, Playlist, PlaylistItem, Video
from app.models import Channel, Video
from app.reader.models import Playlist, PlaylistItem
def _seed_videos(db, ids):
@@ -279,13 +280,13 @@ def test_no_add_can_land_in_the_target_while_a_move_holds_it(db, make_user):
from sqlalchemy import text
from app.db import SessionLocal
from app.routes import playlists as plroutes
from app.sync.playlists import lock_playlists
user = make_user()
_seed_videos(db, ["v1"])
src = _playlist(db, user, "Source", ["v1"])
dst = _playlist(db, user, "Target", [])
plroutes.lock_playlists(db, src.id, dst.id) # what the move holds until its commit
lock_playlists(db, src.id, dst.id) # what the move holds until its commit
with SessionLocal() as other:
other.execute(text("SET LOCAL lock_timeout = '300ms'"))
other.add(PlaylistItem(playlist_id=dst.id, video_id="v1", position=0))
+2 -10
View File
@@ -19,16 +19,8 @@ from datetime import datetime, timezone
import pytest
from sqlalchemy import event
from app.models import (
Channel,
Playlist,
PlaylistItem,
PlexItem,
PlexLibrary,
PlexPlaylist,
PlexPlaylistItem,
Video,
)
from app.models import Channel, PlexItem, PlexLibrary, PlexPlaylist, PlexPlaylistItem, Video
from app.reader.models import Playlist, PlaylistItem
@pytest.fixture
+1 -1
View File
@@ -10,7 +10,7 @@ from sqlalchemy import text
from app.auth import require_write_scope
from app.db import SessionLocal
from app.models import Playlist
from app.reader.models import Playlist
from app.routes import playlists as plroutes
from app.sync import playlists as plsync
@@ -18,7 +18,8 @@ import pytest
from sqlalchemy import delete, event, text
from app.db import SessionLocal
from app.models import Channel, Playlist, PlaylistItem, Video
from app.models import Channel, Video
from app.reader.models import Playlist, PlaylistItem
from app.sync import playlists as pl_mod
ITEMS = PlaylistItem.__table__.name
+310
View File
@@ -0,0 +1,310 @@
"""Reader's service and queries, one use case each, below the HTTP layer.
The routes are thin callers now, and their own tests stay the contract a client sees. These hold
what each write does and leaves on its own: that it flushes and leaves the commit to its caller,
that a row of someone else's reads as a missing one, and the rule each write carries (one default
view per module, a resume point a status keeps, a move that never leaves a video in two lists, the
auto-tagger touching only its own links). DB-backed (real Postgres).
"""
from datetime import datetime, timezone
import pytest
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from app.discover import pins
from app.models import Channel, Video
from app.reader import queries, service
from app.reader.models import ChannelTag, ComfortPin, Playlist, PlaylistItem, Tag, VideoState
def _videos(db, *ids):
if db.get(Channel, "UCr") is None:
db.add(Channel(id="UCr", title="R"))
now = datetime.now(timezone.utc)
for v in ids:
db.add(Video(id=v, channel_id="UCr", published_at=now, duration_seconds=60))
db.commit()
def _uncommitted(db):
"""What the caller still has to commit: the session holds pending work."""
return bool(db.new or db.dirty or db.deleted) or db.in_transaction()
def _ids(db, pl):
return [it.video_id for it in queries.items(db, pl.id)]
# --- whose a row is ------------------------------------------------------------------------
def test_a_row_of_someone_elses_reads_as_a_missing_one(db, make_user):
owner, stranger = make_user(), make_user()
tag = service.create_tag(db, owner, "Mine", "other", None)
view = service.create_view(db, owner, "feed", "V", {})
pl = service.create_playlist(db, owner, "L")
db.commit()
assert queries.own_tag(db, owner, tag.id) is tag
assert queries.own_tag(db, stranger, tag.id) is None
assert queries.own_view(db, stranger, view.id) is None
assert queries.own_playlist(db, stranger, pl.id) is None
assert queries.own_playlist(db, owner, pl.id) is pl
assert queries.playlists_for(db, stranger) == []
def test_a_system_tag_is_nobodys_to_edit(db, make_user):
user, other = make_user(), make_user()
system = service.ensure_system_tag(db, "Music", "topic", None)
service.create_tag(db, other, "Theirs", "other", None) # another user's tag stays theirs
db.commit()
assert queries.own_tag(db, user, system.id) is None
assert [t.id for t, _ in queries.tags_with_counts(db, user)] == [system.id]
# --- tags ------------------------------------------------------------------------------------
def test_a_tag_is_made_once_per_name_and_an_unknown_category_is_other(db, make_user):
user = make_user()
tag = service.create_tag(db, user, "Later", "nonsense", "#fff")
assert _uncommitted(db)
db.commit()
assert tag.category == "other"
with pytest.raises(IntegrityError):
service.create_tag(db, user, "Later", "topic", None)
db.rollback()
def test_a_tag_changes_only_what_was_sent_and_a_sent_null_clears_its_color(db, make_user):
user = make_user()
tag = service.create_tag(db, user, "A", "other", "#fff")
service.update_tag(tag, sent={"name"}, name="B", color=None)
assert (tag.name, tag.color) == ("B", "#fff")
service.update_tag(tag, sent={"color"}, name=None, color=None)
assert (tag.name, tag.color) == ("B", None)
service.delete_tag(db, tag)
db.commit()
assert db.get(Tag, tag.id) is None
def test_the_auto_tagger_moves_only_its_own_links(db, make_user):
user = make_user()
_videos(db)
gaming = service.ensure_system_tag(db, "Gaming", "topic", None)
music = service.ensure_system_tag(db, "Music", "topic", None)
assert service.ensure_system_tag(db, "Gaming", "topic", None) is gaming # made once
mine = service.create_tag(db, user, "Fav", "other", None)
db.add(ChannelTag(channel_id="UCr", tag_id=mine.id, user_id=user.id, source="user"))
db.commit() # the user's link is in the table the auto-tagger reads
service.set_auto_tags(db, "UCr", {gaming.id: 0.5})
db.commit()
service.set_auto_tags(db, "UCr", {music.id: None})
db.commit()
links = db.execute(select(ChannelTag.tag_id, ChannelTag.source)).tuples().all()
assert sorted(links) == sorted([(music.id, "auto"), (mine.id, "user")])
assert db.execute(
select(Channel.id).where(queries.has_auto_tags(Channel.id))
).scalars().all() == ["UCr"]
# --- saved views -----------------------------------------------------------------------------
def test_views_number_per_module_and_one_default_per_module(db, make_user):
user = make_user()
a = service.create_view(db, user, "feed", "A", {})
b = service.create_view(db, user, "feed", "B", {})
plex = service.create_view(db, user, "plex", "P", {})
assert (a.position, b.position, plex.position) == (0, 1, 0)
service.update_view(db, plex, sent={"is_default"}, is_default=True)
service.update_view(db, a, sent={"is_default"}, is_default=True)
service.update_view(db, b, sent={"is_default"}, is_default=True)
db.commit()
db.expire_all()
assert [(v.name, v.is_default) for v in queries.views(db, user, "feed")] == [
("A", False),
("B", True),
]
assert queries.views(db, user, "plex")[0].is_default is True # another module's default stays
def test_reordering_views_skips_ids_that_are_not_the_callers(db, make_user):
user, other = make_user(), make_user()
a = service.create_view(db, user, "feed", "A", {})
b = service.create_view(db, user, "feed", "B", {})
theirs = service.create_view(db, other, "feed", "T", {})
db.commit()
service.reorder_views(db, user, [theirs.id, b.id, a.id])
db.commit()
assert [v.name for v in queries.views(db, user, "feed")] == ["B", "A"]
assert theirs.position == 0
# --- watch state -----------------------------------------------------------------------------
def _state(db, user, vid):
db.expire_all()
return queries.video_state(db, user, vid)
def test_unmarking_keeps_a_resume_point_and_drops_a_row_with_nothing_else(db, make_user):
user = make_user()
_videos(db, "v1", "v2")
service.mark(db, user, "v1", "watched")
service.mark(db, user, "v2", "watched")
service.set_progress(db, user, "v1", 120)
db.commit()
service.unmark(db, user, "v1")
service.unmark(db, user, "v2")
db.commit()
kept = _state(db, user, "v1")
assert (kept.status, kept.watched_at, kept.position_seconds) == ("new", None, 120)
assert _state(db, user, "v2") is None
def test_hiding_a_watched_video_keeps_when_it_was_watched(db, make_user):
user = make_user()
_videos(db, "v1")
service.mark(db, user, "v1", "watched")
db.commit()
watched_at = _state(db, user, "v1").watched_at
service.mark(db, user, "v1", "hidden")
db.commit()
row = _state(db, user, "v1")
assert (row.status, row.watched_at) == ("hidden", watched_at)
assert db.execute(select(VideoState)).scalars().all() == [row] # one row per user and video
def test_a_cleared_resume_point_drops_a_bare_row_and_zeroes_one_with_a_status(db, make_user):
user = make_user()
_videos(db, "v1", "v2")
service.set_progress(db, user, "v1", 30)
service.set_progress(db, user, "v2", 30)
service.mark(db, user, "v2", "hidden")
db.commit()
service.set_progress(db, user, "v1", 0)
service.set_progress(db, user, "v2", 0)
db.commit()
assert _state(db, user, "v1") is None
assert (_state(db, user, "v2").status, _state(db, user, "v2").position_seconds) == ("hidden", 0)
service.clear_state(db, user, "v2")
db.commit()
assert _state(db, user, "v2") is None
# --- playlists -------------------------------------------------------------------------------
def test_watch_later_is_made_once(db, make_user):
user = make_user()
first = service.watch_later(db, user)
db.commit()
assert service.watch_later(db, user) is first
assert first.kind == "watch_later"
assert [p.id for p in queries.playlists_for(db, user)] == [first.id]
def test_an_add_is_idempotent_and_lands_where_asked(db, make_user):
user = make_user()
_videos(db, "a", "b", "c")
pl = service.create_playlist(db, user, "L")
assert service.add_item(db, pl, "a", mark_dirty=False) is True
assert service.add_item(db, pl, "a", mark_dirty=False) is False
service.add_to_playlist(db, pl, "b", None)
service.add_to_playlist(db, pl, "c", 0)
service.add_to_playlist(db, pl, "a", 2) # already there: stays put
assert _uncommitted(db)
db.commit()
assert _ids(db, pl) == ["c", "a", "b"]
assert service.remove_item(db, pl, "a", mark_dirty=True) is True
assert service.remove_item(db, pl, "a", mark_dirty=True) is False
db.commit()
assert _ids(db, pl) == ["c", "b"]
assert queries.item_counts(db, [pl.id]) == {pl.id: 2}
assert queries.durations(db, [pl.id]) == {pl.id: 120}
assert queries.containing(db, [pl.id], "b") == {pl.id}
def test_a_move_never_leaves_a_video_in_two_lists(db, make_user):
user = make_user()
_videos(db, "a", "b")
src = service.create_playlist(db, user, "S")
dst = service.create_playlist(db, user, "D")
service.add_item(db, src, "a", mark_dirty=False)
service.add_item(db, src, "b", mark_dirty=False)
service.add_item(db, dst, "b", mark_dirty=False)
db.commit()
with pytest.raises(service.MoveRefused, match="same_playlist"):
service.move_item(db, src, src, "a")
service.move_item(db, src, dst, "a")
service.move_item(db, src, dst, "b") # already in the target: dropped from the source
db.commit()
assert (_ids(db, src), _ids(db, dst)) == ([], ["b", "a"])
with pytest.raises(service.MoveRefused, match="not_in_playlist"):
service.move_item(db, src, dst, "a")
db.rollback()
def test_a_reorder_keeps_what_it_left_out_after_the_listed(db, make_user):
user = make_user()
_videos(db, "a", "b", "c")
pl = service.create_playlist(db, user, "L")
for v in ("a", "b", "c"):
service.add_item(db, pl, v, mark_dirty=False)
db.commit()
assert service.reorder_items(db, pl, ["c", "zzz", "b"]) == 3
db.commit()
assert _ids(db, pl) == ["c", "b", "a"]
def test_a_rename_and_a_delete(db, make_user):
user = make_user()
_videos(db, "a")
pl = service.create_playlist(db, user, "Old")
service.add_item(db, pl, "a", mark_dirty=False)
service.rename_playlist(db, pl, "New")
db.commit()
assert queries.playlist_totals(db, pl.id) == (1, 60, None)
assert pl.name == "New"
service.delete_playlist(db, pl)
db.commit()
assert db.execute(select(PlaylistItem)).first() is None
assert db.get(Playlist, pl.id) is None
# --- what the other units ask ----------------------------------------------------------------
def test_a_video_someone_holds_is_kept_and_its_playlisters_are_known(db, make_user):
user = make_user()
_videos(db, "watched", "listed", "also", "loose")
service.mark(db, user, "watched", "watched")
pl = service.create_playlist(db, user, "L")
service.add_item(db, pl, "listed", mark_dirty=False)
service.add_item(db, pl, "also", mark_dirty=False)
db.commit()
kept = db.execute(select(Video.id).where(queries.kept_by_a_reader(Video.id))).scalars()
assert sorted(kept) == ["also", "listed", "watched"]
assert queries.playlisted_by(db, ["listed", "loose"]) == [(user.id, "listed")]
# --- the comfort shelf -----------------------------------------------------------------------
def test_a_pin_is_one_per_thing_and_only_its_owners(db, make_user):
user, stranger = make_user(), make_user()
_videos(db, "a")
pin = pins.add(db, user, "video_id", "a")
db.commit()
assert pins.pin_of(db, user, "video_id", "a") is pin
assert pins.pin_of(db, stranger, "video_id", "a") is None
assert pins.own_pin(db, stranger, pin.id) is None
with pytest.raises(IntegrityError):
pins.add(db, user, "video_id", "a")
db.rollback()
pins.remove(db, pins.own_pin(db, user, pin.id))
db.commit()
assert db.execute(select(ComfortPin)).first() is None
+2 -1
View File
@@ -11,7 +11,8 @@ network, pure DB state.
Uses the DB lane (`db` fixture) — skipped when no Postgres is reachable.
"""
from app.models import Channel, ChannelTag, Subscription, Tag, User
from app.models import Channel, Subscription, User
from app.reader.models import ChannelTag, Tag
from app.sync.tagsuggest import (
AUTOAPPLY_THRESHOLD,
CHAR_SUPPORT,
+525
View File
@@ -0,0 +1,525 @@
"""The HTTP layer of a unit does no database work of its own, held in code rather than prose.
A route validates, calls its unit's `queries` (reads) or `service` (writes), serializes and commits.
What it may not do is build SQL, call the session's data methods, change a model's attributes or
import a unit's tables: when a route did those, ownership, the commit and the conditional `WHERE`
lived in a different place in each handler, and a fix in one never reached its siblings.
The rule is switched on unit by unit. `ALLOWED` pins what each HTTP file still does today, and the
pin is exact: a count that rises fails (new database work in a route), and one that falls fails
until its entry is lowered (the fence only tightens, as `baseline.json` does for basedpyright). An
empty `ALLOWED` is the end state — a file listed there is a debt, not an exemption.
A core route (identity, admin, ops) is not a unit's HTTP layer and may use the session; it may
still not reach into a unit's tables. A route may NAME a unit's model as a type (under
`TYPE_CHECKING`), never use one as a value.
Below the HTTP layer, a fenced unit's own code keeps to its edges (`FENCED`, `READS`): see the
second half of this file.
"""
import ast
from pathlib import Path
import app.tables # noqa: F401 — every unit's tables, for the ownership check
from app.db import Base
APP = Path(__file__).parent.parent / "app"
ROUTES = APP / "routes"
# Each unit's tables. A new model is placed here (or in CORE_MODELS) by the test below, so a table
# never joins the schema without an owner.
UNIT_MODELS = {
"capture": {
"MediaAsset",
"DownloadProfile",
"FileFolder",
"DownloadJob",
"DownloadShare",
"DownloadQuota",
"DownloadLink",
"DownloadCredential",
"LibraryState",
},
"reader": {
"VideoState",
"Playlist",
"PlaylistItem",
"SavedView",
"ComfortPin",
"Tag",
"ChannelTag",
},
"catalog": {
"Channel",
"Subscription",
"Video",
"SearchFind",
"ExploredChannel",
"BlockedChannel",
},
"plex": {
"PlexLibrary",
"PlexShow",
"PlexSeason",
"PlexItem",
"PlexPerson",
"PlexCollection",
"PlexState",
"PlexLink",
"PlexPlaylist",
"PlexPlaylistItem",
},
}
CORE_MODELS = {
"User",
"OAuthToken",
"AuthToken",
"Invite",
"ApiQuotaUsage",
"QuotaEvent",
"AuditLog",
"LogEvent",
"DebugSession",
"DebugScreenshot",
"AppState",
"JobRun",
"SystemConfig",
"Notification",
"ApiToken",
"ApiTokenGrant",
}
# Which unit each HTTP file belongs to (path relative to app/routes; a folder covers its files).
# Anything not listed is a core route.
UNIT_ROUTES = {
"downloads.py": "capture",
"files.py": "capture",
"public.py": "capture",
"feed.py": "reader",
"playlists.py": "reader",
"saved_views.py": "reader",
"discover.py": "reader",
"tags.py": "reader",
"channels.py": "catalog",
"search.py": "catalog",
"sync.py": "catalog",
"plex/": "plex",
}
# What each HTTP file still does, by file: the debt a unit's sprint pays off. Exact counts. A core
# route's count is only the unit tables it imports; it is paid by the sprint of the unit it reaches.
ALLOWED = {
"channels.py": 83,
"plex/_shared.py": 1,
"plex/admin.py": 12,
"plex/browse.py": 93,
"plex/collections.py": 33,
"plex/detail.py": 29,
"plex/discover.py": 44,
"plex/people.py": 20,
"plex/playback.py": 42,
"plex/playlists.py": 80,
"runs.py": 1,
"scheduler.py": 3,
"search.py": 46,
"sync.py": 38,
}
# The session methods that read or write data. `commit` and `rollback` stay with the handler.
SESSION_DATA = {
"add",
"add_all",
"delete",
"execute",
"get",
"merge",
"flush",
"refresh",
"query",
"scalar",
"scalars",
"expunge",
}
# What a route may name from SQLAlchemy: the session's type, for its `db: Session` parameter, and
# the errors its commit raises (the commit is the handler's, so is catching what it throws).
SQLALCHEMY_ALLOWED = {"Session"}
SQLALCHEMY_ALLOWED_MODULES = {"sqlalchemy.exc", "sqlalchemy.orm.exc"}
# Objects a route owns and may set attributes on; anything else it writes to is a model.
OWN_OBJECTS = {"self", "cls", "request", "response"}
UNIT_MODEL_NAMES = {name: unit for unit, names in UNIT_MODELS.items() for name in names}
def _unit_of(rel: str) -> str | None:
for key, unit in UNIT_ROUTES.items():
if rel == key or (key.endswith("/") and rel.startswith(key)):
return unit
return None
def _session_names(tree: ast.AST) -> set[str]:
"""The names a module binds to a database session: a parameter typed `Session`, or a name
assigned (or `with`-bound) from `SessionLocal()`."""
names = set()
for node in ast.walk(tree):
if isinstance(node, ast.arg) and isinstance(node.annotation, ast.Name):
if node.annotation.id == "Session":
names.add(node.arg)
made = None
if isinstance(node, ast.Assign) and len(node.targets) == 1:
made, target = node.value, node.targets[0]
elif isinstance(node, ast.withitem):
made, target = node.context_expr, node.optional_vars
if (
isinstance(made, ast.Call)
and isinstance(made.func, ast.Name)
and made.func.id == "SessionLocal"
and isinstance(target, ast.Name)
):
names.add(target.id)
return names
def _type_only_imports(tree: ast.AST) -> set[ast.AST]:
"""The imports under `if TYPE_CHECKING:` — they name a type for an annotation and are never
loaded at run time, so a model imported there cannot be queried or written."""
out: set[ast.AST] = set()
for node in ast.walk(tree):
if isinstance(node, ast.If):
test = node.test
name = test.id if isinstance(test, ast.Name) else getattr(test, "attr", None)
if name == "TYPE_CHECKING":
for child in node.body:
out |= set(ast.walk(child))
return out
def _root(node: ast.expr) -> ast.expr:
while isinstance(node, (ast.Attribute, ast.Subscript)):
node = node.value
return node
def violations(source: str, unit: str | None) -> list[str]:
"""Each piece of database work in an HTTP file, as `line: what`. A core route (`unit` None) is
held only to the last rule: no unit's tables."""
tree = ast.parse(source)
found = []
sql_names = set()
typing_only = _type_only_imports(tree)
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node not in typing_only:
module = node.module or ""
for alias in node.names:
bound = alias.asname or alias.name
if (
module.split(".")[0] == "sqlalchemy"
and module not in SQLALCHEMY_ALLOWED_MODULES
and alias.name not in SQLALCHEMY_ALLOWED
):
sql_names.add(bound)
if (
alias.name in UNIT_MODEL_NAMES
and module.startswith("app.")
and "models" in module
):
found.append(
f"{node.lineno}: imports {UNIT_MODEL_NAMES[alias.name]}'s {alias.name}"
)
elif isinstance(node, ast.Import):
for alias in node.names:
if alias.name.split(".")[0] == "sqlalchemy":
sql_names.add((alias.asname or alias.name).split(".")[0])
if unit is None:
return found
sessions = _session_names(tree)
for node in ast.walk(tree):
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) and node.id in sql_names:
found.append(f"{node.lineno}: uses sqlalchemy's {node.id}")
if isinstance(node, ast.Call):
f = node.func
if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name):
if f.value.id in sessions and f.attr in SESSION_DATA:
found.append(f"{node.lineno}: {f.value.id}.{f.attr}()")
targets = []
if isinstance(node, ast.Assign):
targets = node.targets
elif isinstance(node, (ast.AugAssign, ast.AnnAssign)):
targets = [node.target]
for target in targets:
if isinstance(target, ast.Attribute):
base = _root(target)
if not (isinstance(base, ast.Name) and base.id in OWN_OBJECTS):
found.append(f"{node.lineno}: sets .{target.attr}")
return sorted(found, key=lambda v: int(v.split(":")[0]))
def _http_files():
for path in sorted(ROUTES.rglob("*.py")):
rel = path.relative_to(ROUTES).as_posix()
yield rel, path.read_text(encoding="utf-8")
def test_every_table_has_an_owner():
tables = {mapper.class_.__name__ for mapper in Base.registry.mappers}
owned = set(UNIT_MODEL_NAMES) | CORE_MODELS
assert tables - owned == set(), "a model with no unit: place it in UNIT_MODELS or CORE_MODELS"
assert owned - tables == set(), "UNIT_MODELS/CORE_MODELS name a model that no longer exists"
assert not set(UNIT_MODEL_NAMES) & CORE_MODELS
def test_every_listed_route_exists():
rels = [rel for rel, _ in _http_files()]
for key in UNIT_ROUTES:
assert any(rel == key or (key.endswith("/") and rel.startswith(key)) for rel in rels), key
assert set(ALLOWED) <= set(rels), set(ALLOWED) - set(rels)
def test_no_http_file_does_database_work_beyond_its_pin():
drift = {}
for rel, src in _http_files():
found = violations(src, _unit_of(rel))
pinned = ALLOWED.get(rel, 0)
if len(found) != pinned:
drift[rel] = (pinned, len(found), found if len(found) > pinned else "lower its pin")
assert drift == {}, (
"an HTTP file's database work moved off its pin (pinned, now, what): move new work into"
" the unit's queries/service; when a count fell, lower ALLOWED to it"
)
# The detector itself: each kind of database work it must see, and what it must leave alone.
DETECTOR_CASES = {
"from sqlalchemy import select\nq = select(1)": 1,
"import sqlalchemy as sa\nsa.select(1).where(sa.true())": 2,
"from sqlalchemy import func\nn = func.count()": 1,
"from sqlalchemy.orm import Session\ndef f(db: Session): db.commit(); db.rollback()": 0,
"from sqlalchemy.orm import Session\ndef f(db: Session): db.add(x)": 1,
"def f(conn: Session): conn.execute(q)": 1,
"from app.db import SessionLocal\nwith SessionLocal() as s:\n s.get(X, 1)": 1,
"from app.db import SessionLocal\ns = SessionLocal()\ns.scalars(q)": 1,
"def f(job): job.status = 'x'": 1,
"def f(job): job.meta['k'] = 1": 0,
"def f(job): job.parts.count += 1": 1,
"def f(self, request, response): self.x = 1; request.state.y = 2; response.status_code = 3": 0,
"from app.models import DownloadJob, User": 1,
"from app.capture.models import DownloadJob as J": 1,
"def f(cache): cache.get('k')": 0,
"from sqlalchemy.exc import IntegrityError\ntry:\n pass\nexcept IntegrityError:\n pass": 0,
"from sqlalchemy.orm.exc import StaleDataError\ntry:\n pass\nexcept StaleDataError:\n pass": 0,
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.downloads.models import Job": 0,
"import typing\nif typing.TYPE_CHECKING:\n from app.models import DownloadJob": 0,
"if TYPE_CHECKING:\n pass\nfrom app.downloads.models import DownloadJob": 1,
}
def test_the_detector_sees_each_kind_of_database_work():
for source, expected in DETECTOR_CASES.items():
assert len(violations(source, "capture")) == expected, source
assert violations("from app.models import Video\nfrom sqlalchemy import select", None) == [
"1: imports catalog's Video"
]
# --- inside a unit: its package --------------------------------------------------------------
# A unit whose sprint has landed is fenced beyond its HTTP layer too. Its own code reads another
# unit's tables only along a declared edge (the feed joins Video with VideoState in one query; a
# boundary that split it would cost the speed the app is for), and writes another unit's tables
# never — that goes through the owner's service. Whose a row is, it says through `userscope`, and
# it commits only where a job owns its session.
# Each unit's code outside the HTTP layer: its package, and the entry point that runs its jobs.
UNIT_PACKAGES = {
"capture": ("downloads/", "worker.py"),
# Home (`discover/`) is Reader's landing page; a playlist's YouTube sync and the tag suggestions
# work on Reader's tables alone.
"reader": (
"reader/",
"discover/",
"feedquery.py",
"sync/playlists.py",
"sync/playlist_merge.py",
"sync/tagsuggest.py",
),
}
# The units fenced so far; the rest join with their sprint.
FENCED = {"capture", "reader"}
# Whose tables a unit's own code may READ. Core code reads a fenced unit's tables through its queries.
READS = {"capture": {"catalog", "plex"}, "reader": {"catalog", "plex"}}
# The columns that say whose a row is.
OWNER_COLUMNS = {"user_id", "to_user_id", "from_user_id", "created_by"}
# Where a fenced unit's code may commit: the jobs that own their session (the worker and what it
# runs, the GC, the backfill — none of them inside a request), and the one write that must commit
# where it stands: an upload's rows and the file it placed on disk land together or not at all, so
# a failed commit can remove the file (`upload.ingest_upload`). Everything else flushes; the handler
# commits.
UNIT_COMMITS = {
"capture": {
"worker.py",
"downloads/backfill.py",
"downloads/gc.py",
"downloads/jobdb.py",
"downloads/pipeline.py",
"downloads/upload.py:ingest_upload",
},
# A playlist's YouTube sync runs over many lists and many YouTube calls: it commits what each
# step settled before the next (a list created on YouTube, one list mirrored), so a later
# failure neither forgets a write YouTube already holds nor drops the lists already done. Its
# claim is a lock held on a connection of its own.
"reader": {
"sync/playlists.py:_advisory_claim",
"sync/playlists.py:pull_user_playlists",
"sync/playlists.py:merge_playlist",
"sync/playlists.py:repull_playlist",
},
}
SQL_WRITES = {"insert", "update", "delete", "pg_insert"}
def _package_unit(rel: str) -> str | None:
for unit, prefixes in UNIT_PACKAGES.items():
if any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in prefixes):
return unit
return None
def _app_files():
for path in sorted(APP.rglob("*.py")):
rel = path.relative_to(APP).as_posix()
if not rel.startswith("routes/"):
yield rel, path.read_text(encoding="utf-8")
def _imported_models(tree: ast.AST) -> dict[str, str]:
"""The unit models a module imports at run time (a `TYPE_CHECKING` import names a type only),
by the name it binds them to."""
typing_only = _type_only_imports(tree)
out = {}
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node not in typing_only:
module = node.module or ""
if module.startswith("app.") and "models" in module:
for alias in node.names:
if alias.name in UNIT_MODEL_NAMES:
out[alias.asname or alias.name] = alias.name
return out
def _enclosing_functions(tree: ast.AST) -> dict[ast.AST, str]:
out = {}
for fn in ast.walk(tree):
if isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
for node in ast.walk(fn):
out.setdefault(node, fn.name)
return out
def package_violations(rel: str, source: str) -> list[str]:
"""What a module outside the HTTP layer does against a fenced unit's boundary, as `line: what`."""
tree = ast.parse(source)
mine = _package_unit(rel)
found = []
models = _imported_models(tree)
for name in models.values():
theirs = UNIT_MODEL_NAMES[name]
if theirs == mine or (mine not in FENCED and theirs not in FENCED):
continue
if theirs not in READS.get(mine or "", set()):
found.append(f"0: {mine or 'core'} imports {theirs}'s {name}")
for node in ast.walk(tree):
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name):
continue
written = None
if node.func.id in models:
written = models[node.func.id]
elif node.func.id in SQL_WRITES and node.args and isinstance(node.args[0], ast.Name):
written = models.get(node.args[0].id)
if written is None:
continue
theirs = UNIT_MODEL_NAMES[written]
if theirs != mine and (mine in FENCED or theirs in FENCED):
found.append(f"{node.lineno}: {mine or 'core'} writes {theirs}'s {written}")
if mine in FENCED:
found += _owner_filters(tree)
functions = _enclosing_functions(tree)
allowed = UNIT_COMMITS.get(mine, set())
for node in ast.walk(tree):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "commit"
and rel not in allowed
and f"{rel}:{functions.get(node)}" not in allowed
):
found.append(f"{node.lineno}: commits in {functions.get(node)}")
return sorted(found, key=lambda v: int(v.split(":")[0]))
def _owner_filters(tree: ast.AST) -> list[str]:
"""A comparison on a table's owner column written inline (`Model.user_id == …`), not through
`userscope.owned_by`."""
found = []
for node in ast.walk(tree):
if isinstance(node, ast.Compare):
for side in (node.left, *node.comparators):
if (
isinstance(side, ast.Attribute)
and isinstance(side.value, ast.Name)
and side.value.id in set(UNIT_MODEL_NAMES) | CORE_MODELS
and side.attr in OWNER_COLUMNS
):
found.append(f"{node.lineno}: filters {side.value.id}.{side.attr} inline")
return found
def test_a_fenced_units_code_keeps_to_its_edges():
hits = {rel: found for rel, src in _app_files() if (found := package_violations(rel, src))}
assert hits == {}, (
"read another unit's tables only along READS, write them only through their service, say"
" whose a row is through userscope, commit only where UNIT_COMMITS says"
)
def test_a_fenced_units_routes_filter_owners_through_userscope():
hits = {
rel: found
for rel, src in _http_files()
if _unit_of(rel) in FENCED and (found := _owner_filters(ast.parse(src)))
}
assert hits == {}
def test_every_unit_package_exists():
for prefixes in UNIT_PACKAGES.values():
for p in prefixes:
assert (APP / p).exists(), p
for commits in UNIT_COMMITS.values():
for entry in commits:
assert (APP / entry.split(":")[0]).exists(), entry
PACKAGE_CASES = {
# (module path, source): violations
("downloads/x.py", "from app.models import Video"): 0, # capture reads catalog
("downloads/x.py", "from app.models import Playlist"): 1, # no capture→reader edge
("downloads/x.py", "from app.models import Video\nVideo(id=1)"): 1, # a write across
("downloads/x.py", "from app.models import Video\nupdate(Video)"): 1,
("downloads/x.py", "from app.downloads.models import DownloadJob\nDownloadJob()"): 0,
("search.py", "from app.downloads.models import DownloadJob"): 1, # core reads capture
("search.py", "from app.models import Video\nVideo()"): 0, # neither side fenced yet
("downloads/x.py", "q = DownloadJob.user_id == 1"): 1,
("downloads/x.py", "q = owned_by(DownloadJob.user_id, 1)"): 0,
("downloads/x.py", "def f(db):\n db.commit()"): 1,
("downloads/gc.py", "def f(db):\n db.commit()"): 0,
("downloads/upload.py", "def ingest_upload(db):\n db.commit()"): 0,
("downloads/upload.py", "def other(db):\n db.commit()"): 1,
(
"downloads/x.py",
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.models import Playlist",
): 0,
}
def test_the_package_detector_sees_each_kind():
for (rel, source), expected in PACKAGE_CASES.items():
assert len(package_violations(rel, source)) == expected, (rel, source)
+2 -1
View File
@@ -33,7 +33,8 @@ from app.downloads import mediakind, pipeline, playable, quota, service, storage
from app.downloads import probe as probemod
from app.downloads import upload as uploadmod
from app.downloads.stated import StatedFailure
from app.models import DISPLAY_TEXT_MAX, DownloadJob, DownloadQuota, MediaAsset
from app.models import DISPLAY_TEXT_MAX
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
# --- pure: the kind invariant ---------------------------------------------------------------
+2 -1
View File
@@ -21,7 +21,8 @@ from app.downloads import pipeline, quota, storage
from app.downloads import probe as probemod
from app.downloads import upload as uploadmod
from app.downloads.stated import StatedFailure
from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification
from app.models import Notification
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
from sqlalchemy import select, update
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session
+1 -1
View File
@@ -12,7 +12,7 @@ from app.config import settings
from app.downloads import pipeline, runner, service, storage
from app.downloads import probe as probemod
from app.downloads import upload as uploadmod
from app.models import DownloadJob, MediaAsset
from app.downloads.models import DownloadJob, MediaAsset
@pytest.mark.parametrize(
+1
View File
@@ -113,6 +113,7 @@ project checks needs a row.
| CVE gate | HIGH/CRITICAL with a fix available only | `--ignore-unfixed` + `--severity HIGH,CRITICAL`: a finding with no fix cannot be acted on by an upgrade; it shows in the scan output. A fixable one fails the publish, and the only way past it is `.trivyignore.yaml`. | 0 open (E32 S1) |
| npm audit | `--omit=dev --audit-level=high` | What ships to the browser, at high or above. The dev toolchain's advisories were burned down by hand in E32 S1 (18 → 0). | 0 |
| knip | `entry` — one file added | Not a narrowing: knip's Vite plugin reads entries from index.html only, so the shared link's page (`watch.html` → `src/watchMain.tsx`) is named here, or the lane reports that whole entry unused. It adds files to the lane, never takes any out. | 1 entry |
| unit fence | the database work the HTTP layer still does, pinned | `backend/tests/test_unit_fences.py`: a unit's route file may not build SQL, call the session's data methods, set a model's attributes or import a unit's tables; a core route may not import a unit's tables. Switched on unit by unit: `ALLOWED` pins each file's count exactly, so a rise fails and a fall fails until the pin is lowered — the list may only shrink. End state: empty, and the pin mechanism goes. A unit in `FENCED` is held below its HTTP layer too: its code reads another unit's tables only along `READS`, writes them never, filters owners through `userscope`, and commits only where `UNIT_COMMITS` names a job's own session. | 826 in 23 files (E42 S0) → 667 in 19 (S1: Capture 0, fenced) → 525 in 14 (S2: Reader 0, fenced) |
| React Compiler | the functions it skips, pinned | `components/compilerCoverage.contract.test.ts` — the list may only shrink. End state: only VirtualGrid (TanStack Virtual, an IncompatibleLibrary) remains. | 93 (E32 S1) → 29 (E32 S3) → 1, VirtualGrid (E32 S4) |
## Accepted findings
@@ -29,8 +29,10 @@ describe("display-name bound", () => {
it("is the width the columns are actually declared with", () => {
// The mutation guard for the check above: both constants could agree on a number neither
// column uses. `String(DISPLAY_TEXT_MAX)` is how the three display columns are declared, so a
// literal creeping back into one of them fails here.
const models = src("../../../../backend/app/models.py");
// literal creeping back into one of them fails here. `User.display_name` is a core table; the
// job's two are Capture's.
const models =
src("../../../../backend/app/models.py") + src("../../../../backend/app/downloads/models.py");
const declared = [...models.matchAll(/display_(?:name|uploader): Mapped.*?String\((.*?)\)/g)];
expect(declared).toHaveLength(3);
for (const m of declared) expect(m[1]).toBe("DISPLAY_TEXT_MAX");
+7 -2
View File
@@ -10,6 +10,11 @@ import { FILE_SORTS, FILES_LIST_CAP, listQuery } from "./files";
// matches everything. The route's parameter names are read from its source, as text — the backend
// cannot be imported here.
const routes = readFileSync(resolve(__dirname, "../../../../backend/app/routes/files.py"), "utf-8");
// The cap is the query's (backend downloads/queries.py), which the route lists through.
const queries = readFileSync(
resolve(__dirname, "../../../../backend/app/downloads/queries.py"),
"utf-8",
);
function routeParams(): Map<string, string> {
const sig = routes.match(/^def list_files\(([\s\S]*?)\) -> /m);
@@ -43,8 +48,8 @@ describe("the Files list's request", () => {
it("knows the cap the route lists up to", () => {
// A list as long as the cap may be missing older files; the page then searches on the server.
const cap = routes.match(/^MAX_LIST = (\d+)$/m);
expect(cap, "routes/files.py no longer defines MAX_LIST").toBeTruthy();
const cap = queries.match(/^MAX_LIST = (\d+)$/m);
expect(cap, "downloads/queries.py no longer defines MAX_LIST").toBeTruthy();
expect(FILES_LIST_CAP).toBe(Number(cap![1]));
});
});
+1 -3
View File
@@ -27,13 +27,11 @@ from app.db import SessionLocal
from app.models import (
AppState,
Channel,
Playlist,
PlaylistItem,
Subscription,
User,
Video,
VideoState,
)
from app.reader.models import Playlist, PlaylistItem, VideoState
from app.security import hash_password
EMAIL = os.environ.get("E2E_EMAIL", "e2e@siftlode.local")