Compare commits
2
Commits
d66174ed16
...
e9c60f72cd
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9c60f72cd | ||
|
|
3a688fa0f5 |
@@ -692,48 +692,6 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/downloads.py": [
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 17,
|
||||
"endColumn": 19,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 82,
|
||||
"endColumn": 84,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 87,
|
||||
"endColumn": 89,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 30,
|
||||
"endColumn": 32,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 49,
|
||||
"endColumn": 54,
|
||||
"lineCount": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/playlists.py": [
|
||||
{
|
||||
"code": "reportOptionalOperand",
|
||||
|
||||
@@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config, pool
|
||||
from app.config import settings
|
||||
from app.db import Base
|
||||
from app.logs import configure_logging
|
||||
import app.models # noqa: F401 (import so models register on Base.metadata)
|
||||
import app.tables # noqa: F401 (import so every table registers on Base.metadata)
|
||||
|
||||
config = context.config
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
|
||||
@@ -22,7 +22,7 @@ from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
from app.downloads.models import MediaAsset
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
@@ -33,8 +33,9 @@ from app import logs
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
from app.downloads.models import DownloadCredential, MediaAsset
|
||||
from app.security import decrypt_optional, encrypt
|
||||
from app.userscope import owned_by
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
@@ -157,7 +158,7 @@ def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||
def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None:
|
||||
return db.execute(
|
||||
select(DownloadCredential).where(
|
||||
DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key
|
||||
owned_by(DownloadCredential.user_id, user_id), DownloadCredential.host_key == host_key
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
@@ -214,7 +215,7 @@ def status_list(db: Session, user_id: int) -> list[dict]:
|
||||
have = {
|
||||
c.host_key: c
|
||||
for c in db.execute(
|
||||
select(DownloadCredential).where(DownloadCredential.user_id == user_id)
|
||||
select(DownloadCredential).where(owned_by(DownloadCredential.user_id, user_id))
|
||||
).scalars()
|
||||
}
|
||||
out = []
|
||||
|
||||
@@ -32,7 +32,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, FileFolder, MediaAsset
|
||||
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||
from app.userscope import owned_by
|
||||
|
||||
|
||||
class FolderKind(StrEnum):
|
||||
@@ -106,7 +107,9 @@ def source_key_for(db: Session, job: DownloadJob) -> str:
|
||||
def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
|
||||
"""The user's source folder for `key`, made on first use. A race on the unique (user, key) is
|
||||
settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
|
||||
stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
|
||||
stmt = select(FileFolder).where(
|
||||
owned_by(FileFolder.user_id, user_id), FileFolder.source_key == key
|
||||
)
|
||||
folder = db.execute(stmt).scalar_one_or_none()
|
||||
if folder is not None:
|
||||
return folder
|
||||
@@ -131,7 +134,7 @@ def hub_folder(db: Session, user_id: int) -> FileFolder:
|
||||
"""The user's Hub folder, made on first use; a race on its one-per-user index is settled like
|
||||
`source_folder`'s."""
|
||||
stmt = select(FileFolder).where(
|
||||
FileFolder.user_id == user_id, FileFolder.kind == FolderKind.HUB
|
||||
owned_by(FileFolder.user_id, user_id), FileFolder.kind == FolderKind.HUB
|
||||
)
|
||||
folder = db.execute(stmt).scalar_one_or_none()
|
||||
if folder is not None:
|
||||
@@ -310,3 +313,28 @@ def delete_own(db: Session, folder: FileFolder) -> int:
|
||||
db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete
|
||||
db.delete(folder)
|
||||
return len(jobs)
|
||||
|
||||
|
||||
def create_own(db: Session, user_id: int, name: str) -> FileFolder:
|
||||
"""A new folder of the user's own; FolderError("file_folder_name_taken") when they already have
|
||||
one by that name (the unique (user, lower(name)))."""
|
||||
folder = FileFolder(user_id=user_id, kind=FolderKind.OWN, name=name)
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(folder)
|
||||
db.flush()
|
||||
except IntegrityError as e:
|
||||
raise FolderError("file_folder_name_taken") from e
|
||||
return folder
|
||||
|
||||
|
||||
def rename(db: Session, folder: FileFolder, name: str) -> None:
|
||||
"""Rename one of the user's own folders (a source folder's name is its origin's)."""
|
||||
if folder.kind != FolderKind.OWN:
|
||||
raise FolderError("file_folder_not_own")
|
||||
folder.name = name
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.flush()
|
||||
except IntegrityError as e:
|
||||
raise FolderError("file_folder_name_taken") from e
|
||||
|
||||
@@ -31,7 +31,7 @@ from app import apitokens, logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
@@ -6,7 +6,7 @@ from sqlalchemy import text
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
||||
|
||||
@@ -15,20 +15,24 @@ grant appended to the file URL, so the raw password never rides in the media URL
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import update as sql_update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import security
|
||||
from app import audit, security
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
from app.models import User
|
||||
from app.utils import iso
|
||||
from app.userscope import owned_by
|
||||
|
||||
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
|
||||
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
|
||||
# MAX_LIST`; a test holds the two equal — this layer may not import that one).
|
||||
# The most files a folder link lists: as many as the owner's own Files list shows
|
||||
# (`queries.MAX_LIST`; a test holds the two equal — `queries` imports this module).
|
||||
FOLDER_MAX = 500
|
||||
|
||||
|
||||
@@ -59,7 +63,7 @@ def in_folder(folder: FileFolder) -> tuple:
|
||||
moved out stops opening through the link at its next request; one moved in starts."""
|
||||
return (
|
||||
DownloadJob.folder_id == folder.id,
|
||||
DownloadJob.user_id == folder.user_id,
|
||||
owned_by(DownloadJob.user_id, folder.user_id),
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
)
|
||||
|
||||
@@ -170,3 +174,117 @@ def public_meta(
|
||||
"file_url": file_url,
|
||||
"poster_url": poster_url,
|
||||
}
|
||||
|
||||
|
||||
# --- the owner's writes --------------------------------------------------------------------
|
||||
# Each flushes and records its audit row; the handler commits.
|
||||
|
||||
|
||||
def expiry(days: int | None) -> datetime | None:
|
||||
"""None/0/negative → no expiry; otherwise N days out, capped at ~10 years."""
|
||||
if not days or days <= 0:
|
||||
return None
|
||||
return datetime.now(timezone.utc) + timedelta(days=min(days, 3650))
|
||||
|
||||
|
||||
def audit_view(link: DownloadLink) -> dict:
|
||||
"""A link's settings as the audit records them: whether it has a password, never which."""
|
||||
return {
|
||||
"allow_download": link.allow_download,
|
||||
"has_password": link.password_hash is not None,
|
||||
"expires_at": iso(link.expires_at),
|
||||
}
|
||||
|
||||
|
||||
def create(
|
||||
db: Session,
|
||||
user: User,
|
||||
*,
|
||||
password: str,
|
||||
allow_download: bool,
|
||||
expires_days: int | None,
|
||||
**target: int,
|
||||
) -> tuple[DownloadLink, str]:
|
||||
"""A link to `target` (`job_id=` or `folder_id=`, already checked to be the user's), and the
|
||||
token it is opened by — which is in hand only now."""
|
||||
token = new_token()
|
||||
link = DownloadLink(
|
||||
**target,
|
||||
created_by=user.id,
|
||||
allow_download=allow_download,
|
||||
password_hash=security.hash_password(password) if password else None,
|
||||
expires_at=expiry(expires_days),
|
||||
)
|
||||
seal(link, token)
|
||||
db.add(link)
|
||||
db.flush()
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_CREATE,
|
||||
target_type="download_link",
|
||||
target_id=link.id,
|
||||
after={**target, **audit_view(link)},
|
||||
)
|
||||
return link, token
|
||||
|
||||
|
||||
def update(
|
||||
db: Session,
|
||||
user: User,
|
||||
link: DownloadLink,
|
||||
*,
|
||||
allow_download: bool | None = None,
|
||||
expires_days: int | None = None,
|
||||
password: str | None = None,
|
||||
sent: frozenset[str] = frozenset(),
|
||||
) -> None:
|
||||
"""Change what the caller sent (`sent` names the fields, so `expires_days=None` can mean "never
|
||||
expires"). Setting, clearing or changing the password bumps its version, so any grant issued
|
||||
under the previous one stops working now rather than living out its TTL."""
|
||||
before = audit_view(link)
|
||||
if "allow_download" in sent:
|
||||
link.allow_download = bool(allow_download)
|
||||
if "expires_days" in sent:
|
||||
link.expires_at = expiry(expires_days)
|
||||
if "password" in sent:
|
||||
pw = (password or "").strip()
|
||||
link.password_hash = security.hash_password(pw) if pw else None
|
||||
link.password_version += 1
|
||||
after = audit_view(link)
|
||||
if after != before or "password" in sent:
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_UPDATE,
|
||||
target_type="download_link",
|
||||
target_id=link.id,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def revoke(db: Session, user: User, link: DownloadLink) -> None:
|
||||
link_id = link.id
|
||||
db.delete(link)
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_REVOKE,
|
||||
target_type="download_link",
|
||||
target_id=link_id,
|
||||
after={"job_id": link.job_id} if link.job_id is not None else {"folder_id": link.folder_id},
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def count_view(db: Session, link: DownloadLink) -> None:
|
||||
"""One more opening of the link. In SQL, not `+= 1` on the loaded row: two opens at once would
|
||||
both write the same count."""
|
||||
db.execute(
|
||||
sql_update(DownloadLink)
|
||||
.where(DownloadLink.id == link.id)
|
||||
.values(view_count=DownloadLink.view_count + 1)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
"""Capture's tables: the downloaded files (`MediaAsset`), each user's jobs over them and the folders
|
||||
they sit in, the shares and public links that hand them on, a user's quota, cookies and format
|
||||
profiles, and where they stopped playing one. Only this unit writes them; another unit reads them
|
||||
along a declared edge, or asks this unit's service."""
|
||||
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Date,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, validates
|
||||
|
||||
from app.db import Base
|
||||
from app.models import DISPLAY_TEXT_MAX, TimestampMixin, UpdatedAtMixin
|
||||
|
||||
|
||||
class LibraryState(Base):
|
||||
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
||||
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
||||
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
||||
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
||||
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
||||
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
||||
|
||||
__tablename__ = "library_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class MediaAsset(Base, TimestampMixin):
|
||||
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
||||
|
||||
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
|
||||
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
|
||||
the same video with a format that hashes to the same `format_sig`, their `download_job`
|
||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||
|
||||
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
||||
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
||||
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
||||
|
||||
__tablename__ = "media_assets"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
|
||||
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="pending", server_default="pending", index=True
|
||||
) # "pending" | "ready" | "error"
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||
# card / watch page / link preview can still show an image. Relative to files_root.
|
||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||
width: Mapped[int | None] = mapped_column(Integer)
|
||||
height: Mapped[int | None] = mapped_column(Integer)
|
||||
container: Mapped[str | None] = mapped_column(String(16))
|
||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
|
||||
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
|
||||
# still pending (kind is known once the file is probed at finalize). It is the single stored home
|
||||
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
|
||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
|
||||
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
|
||||
uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
||||
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
||||
# fall back to a URL derived from source_kind+source_ref).
|
||||
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
|
||||
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
|
||||
sha256: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
@validates("rel_path")
|
||||
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
|
||||
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
|
||||
self.sha256 = None
|
||||
return rel_path
|
||||
|
||||
|
||||
class DownloadProfile(Base, TimestampMixin):
|
||||
"""A named download preset — a reusable format profile.
|
||||
|
||||
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
|
||||
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
|
||||
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
|
||||
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
|
||||
`format_sig`."""
|
||||
|
||||
__tablename__ = "download_profiles"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
) # NULL for a builtin preset
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
spec: Mapped[dict] = mapped_column(JSON)
|
||||
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class FileFolder(Base, TimestampMixin):
|
||||
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
||||
|
||||
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
||||
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
|
||||
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
|
||||
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
||||
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
||||
|
||||
__tablename__ = "file_folders"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||
Index(
|
||||
"uq_file_folder_own_name",
|
||||
"user_id",
|
||||
func.lower(text("name")),
|
||||
unique=True,
|
||||
postgresql_where=text("name IS NOT NULL"),
|
||||
),
|
||||
# One Hub folder per user.
|
||||
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
|
||||
CheckConstraint(
|
||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
|
||||
name="ck_file_folder_kind",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
||||
# "site:" + a host name (DNS caps one at 253 characters).
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
retention_days: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
def _ready_on_insert(context) -> datetime | None:
|
||||
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
|
||||
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
|
||||
done = context.get_current_parameters().get("status") == "done"
|
||||
return datetime.now(timezone.utc) if done else None
|
||||
|
||||
|
||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||
|
||||
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
|
||||
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
|
||||
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
||||
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
||||
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
||||
(SET NULL if the asset is GC'd — the job then shows as expired).
|
||||
|
||||
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
|
||||
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
|
||||
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
|
||||
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
|
||||
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
||||
|
||||
__tablename__ = "download_jobs"
|
||||
__table_args__ = (
|
||||
Index("ix_download_jobs_claim", "status", "queue_pos"),
|
||||
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
|
||||
CheckConstraint(
|
||||
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512))
|
||||
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
|
||||
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
|
||||
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
|
||||
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||
source_job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="SET NULL")
|
||||
)
|
||||
source_asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL")
|
||||
)
|
||||
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
|
||||
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
|
||||
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
|
||||
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
||||
profile_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="queued", server_default="queued", index=True
|
||||
) # queued | running | paused | done | error | canceled
|
||||
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
|
||||
eta_s: Mapped[int | None] = mapped_column(Integer)
|
||||
phase: Mapped[str | None] = mapped_column(String(32))
|
||||
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
||||
error_code: Mapped[str | None] = mapped_column(
|
||||
String(24)
|
||||
) # category, see downloads.errors.classify
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
||||
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
||||
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
||||
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||
)
|
||||
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
||||
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
||||
# file must still know which source folder it may go back to. NULL only for a file placed before
|
||||
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# When the file became done — what the library's "Added" means: a download that failed for days
|
||||
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
|
||||
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
|
||||
ready_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), default=_ready_on_insert
|
||||
)
|
||||
|
||||
|
||||
class DownloadShare(Base, TimestampMixin):
|
||||
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
|
||||
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
||||
|
||||
__tablename__ = "download_shares"
|
||||
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
|
||||
from_user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
|
||||
class DownloadQuota(Base, UpdatedAtMixin):
|
||||
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
|
||||
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
|
||||
byte cap entirely (e.g. for the admin's own account)."""
|
||||
|
||||
__tablename__ = "download_quotas"
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
max_concurrent: Mapped[int] = mapped_column(Integer)
|
||||
max_jobs: Mapped[int] = mapped_column(Integer)
|
||||
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
class DownloadLink(Base, TimestampMixin):
|
||||
"""A public capability link to one download, or to one of its owner's own folders — the "share
|
||||
by link" surface.
|
||||
|
||||
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
|
||||
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
|
||||
the folder at the time it is opened. The token itself is never stored as written: a link is
|
||||
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
|
||||
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
|
||||
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
|
||||
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
|
||||
grants a *registered* user access via the in-app "Shared with me" list."""
|
||||
|
||||
__tablename__ = "download_links"
|
||||
__table_args__ = (
|
||||
# A link opens exactly one thing: a file or a folder.
|
||||
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
password_hash: Mapped[str | None] = mapped_column(Text)
|
||||
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
|
||||
# password immediately invalidates any grants issued under the old one (instead of them living
|
||||
# out the 6h grant TTL).
|
||||
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
|
||||
|
||||
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
|
||||
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
|
||||
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
|
||||
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
|
||||
the routes are write-only (host + updated_at only). See `downloads.credentials`.
|
||||
|
||||
Two load-bearing security controls live around this row, NOT on it:
|
||||
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
|
||||
user id, so the authed asset is private to that user and can never be cache-served to
|
||||
another user (`downloads.credentials.cookie_salt`).
|
||||
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
|
||||
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
|
||||
can't exfiltrate the user's session to a look-alike host.
|
||||
|
||||
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
|
||||
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
|
||||
|
||||
__tablename__ = "download_credentials"
|
||||
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
host_key: Mapped[str] = mapped_column(String(64))
|
||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
@@ -16,7 +16,7 @@ from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
|
||||
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
|
||||
|
||||
|
||||
@@ -28,7 +28,8 @@ from app.downloads.jobdb import (
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem
|
||||
from app.models import DISPLAY_TEXT_MAX, PlexItem
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.plex import lookups as plex_lookups
|
||||
from app.plex import paths as plex_paths
|
||||
from app.notifications import create_notification
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
"""Capture's reads: what the library, the download tray, the share dialogs and the public watch pages
|
||||
show. Each returns rows or plain values and writes nothing; whose a row is goes through
|
||||
`userscope.owned_by` / `owned_row`, so a row of someone else's reads as a missing one.
|
||||
|
||||
The HTTP layer (`routes.downloads`, `routes.files`, `routes.public`) asks here and serializes; the
|
||||
writes are the service's (`service`, `folders`, `links`, `quota`)."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.apitokens import FilesCaller, Right
|
||||
from app.collation import collated
|
||||
from app.downloads import folders, links, quota
|
||||
from app.downloads.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
DownloadProfile,
|
||||
DownloadQuota,
|
||||
DownloadShare,
|
||||
FileFolder,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
)
|
||||
from app.downloads.status import FINISHED, JobStatus
|
||||
from app.models import User
|
||||
from app.userscope import owned_by, owned_row
|
||||
from app.utils import like_escape
|
||||
|
||||
# Cap a listing so a heavy library can't return an unbounded result set (the old Download Center's
|
||||
# own cap; far beyond any real library).
|
||||
MAX_LIST = 500
|
||||
|
||||
|
||||
# --- jobs and their files ------------------------------------------------------------------
|
||||
|
||||
|
||||
def own_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
|
||||
return owned_row(db, DownloadJob, job_id, user)
|
||||
|
||||
|
||||
def job(db: Session, job_id: int) -> DownloadJob | None:
|
||||
"""A job by id, whoever's — for a public link, which already names its job."""
|
||||
return db.get(DownloadJob, job_id)
|
||||
|
||||
|
||||
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
|
||||
"""A file the caller may read: their own, or one shared with them."""
|
||||
row = db.get(DownloadJob, job_id)
|
||||
if row is None:
|
||||
return None
|
||||
if row.user_id == user.id or shared_with(db, job_id, user) is not None:
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob | None:
|
||||
"""A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's
|
||||
own in a folder it may read."""
|
||||
if caller.grants is None:
|
||||
return accessible_job(db, caller.user, job_id)
|
||||
row = own_job(db, caller.user, job_id)
|
||||
if row is None or not caller.may(row.folder_id, Right.READ):
|
||||
return None
|
||||
return row
|
||||
|
||||
|
||||
def asset_of(db: Session, job: DownloadJob | None) -> MediaAsset | None:
|
||||
"""The file under a job, if it has one yet."""
|
||||
return db.get(MediaAsset, job.asset_id) if job is not None and job.asset_id else None
|
||||
|
||||
|
||||
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
|
||||
ids = {j.asset_id for j in jobs if j.asset_id}
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
|
||||
return {a.id: a for a in rows}
|
||||
|
||||
|
||||
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
|
||||
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
|
||||
ids = [j.id for j in jobs]
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(
|
||||
select(LibraryState.job_id, LibraryState.position_seconds).where(
|
||||
owned_by(LibraryState.user_id, user), LibraryState.job_id.in_(ids)
|
||||
)
|
||||
).all()
|
||||
return {job_id: pos for job_id, pos in rows}
|
||||
|
||||
|
||||
def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
|
||||
"""Where each file came from (its source folder's key), for the list's Source column, the
|
||||
"back to its source folder" move and the download tray's site: a file in a source folder has its
|
||||
folder's key; one moved into an own folder (or someone else's, shared) is traced back by
|
||||
`folders.source_key_for`."""
|
||||
ids = {j.folder_id for j in jobs if j.folder_id is not None}
|
||||
by_id = (
|
||||
{f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
|
||||
if ids
|
||||
else {}
|
||||
)
|
||||
out: dict[int, str] = {}
|
||||
for j in jobs:
|
||||
folder = by_id.get(j.folder_id) if j.folder_id is not None else None
|
||||
if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
|
||||
out[j.id] = folder.source_key
|
||||
else:
|
||||
out[j.id] = folders.source_key_for(db, j)
|
||||
return out
|
||||
|
||||
|
||||
def queue(db: Session, user: User, recent: datetime) -> list[DownloadJob]:
|
||||
"""The download tray: what is waiting, running, paused or failed, and what finished (done or
|
||||
canceled) since `recent`. A done job is timed from when it became done, never from its last
|
||||
write: a rename, a move or the GC's expiry warning touches an old file's row, and the tray would
|
||||
announce it as just finished."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(
|
||||
owned_by(DownloadJob.user_id, user),
|
||||
DownloadJob.status.not_in(FINISHED)
|
||||
| ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent))
|
||||
| ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)),
|
||||
)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def statuses_by_source(db: Session, user: User, statuses: Sequence[str]) -> list[tuple[str, str]]:
|
||||
"""(source_ref, status) of each of `user`'s jobs in one of `statuses`."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob.source_ref, DownloadJob.status).where(
|
||||
owned_by(DownloadJob.user_id, user), DownloadJob.status.in_(tuple(statuses))
|
||||
)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def library(
|
||||
db: Session,
|
||||
user: User,
|
||||
*,
|
||||
folder_id: int | None,
|
||||
listable: set[int] | None,
|
||||
shared: bool,
|
||||
term: str,
|
||||
sort: str,
|
||||
order: str,
|
||||
lang: str,
|
||||
) -> list[DownloadJob]:
|
||||
"""The caller's done files (optionally one folder's, or only in the folders a token may list),
|
||||
or the ones shared with them; searched by name and channel, sorted by when they were added, by
|
||||
name (in the reader's alphabet) or by size. A file is added to its owner's list when it is done,
|
||||
not when it was asked for (a download that failed for days and then went through is new); one
|
||||
shared with the caller, when it was shared."""
|
||||
stmt = (
|
||||
select(DownloadJob)
|
||||
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(DownloadJob.status == JobStatus.DONE)
|
||||
)
|
||||
if shared:
|
||||
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
|
||||
owned_by(DownloadShare.to_user_id, user)
|
||||
)
|
||||
else:
|
||||
stmt = stmt.where(owned_by(DownloadJob.user_id, user))
|
||||
if folder_id is not None:
|
||||
stmt = stmt.where(DownloadJob.folder_id == folder_id)
|
||||
elif listable is not None:
|
||||
stmt = stmt.where(DownloadJob.folder_id.in_(listable))
|
||||
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
|
||||
if term:
|
||||
like = f"%{like_escape(term)}%"
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
name.ilike(like),
|
||||
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
|
||||
)
|
||||
)
|
||||
key = {
|
||||
"added": DownloadShare.created_at if shared else DownloadJob.ready_at,
|
||||
"name": collated(name, lang),
|
||||
"size": MediaAsset.size_bytes,
|
||||
}[sort]
|
||||
first = key.asc() if order == "asc" else key.desc()
|
||||
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
|
||||
return list(db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars())
|
||||
|
||||
|
||||
def recent_jobs(db: Session) -> list[DownloadJob]:
|
||||
"""Everyone's newest jobs, for the admin's list."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
# --- shares --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def shared_with(db: Session, job_id: int, user: User | int) -> DownloadShare | None:
|
||||
"""The grant that hands this job to `user`, if there is one."""
|
||||
return db.execute(
|
||||
select(DownloadShare).where(
|
||||
DownloadShare.job_id == job_id, owned_by(DownloadShare.to_user_id, user)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
|
||||
rows = db.execute(
|
||||
select(DownloadShare.job_id, DownloadShare.created_at).where(
|
||||
owned_by(DownloadShare.to_user_id, user),
|
||||
DownloadShare.job_id.in_([j.id for j in jobs]),
|
||||
)
|
||||
).all()
|
||||
return {job_id: at for job_id, at in rows}
|
||||
|
||||
|
||||
# --- folders -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def own_folder(db: Session, user: User, folder_id: int) -> FileFolder | None:
|
||||
return owned_row(db, FileFolder, folder_id, user)
|
||||
|
||||
|
||||
def folders_by_id(db: Session, ids: set[int]) -> dict[int, FileFolder]:
|
||||
"""These folders, by id — for a reader that already knows which ones (a token's grants)."""
|
||||
if not ids:
|
||||
return {}
|
||||
return {f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
|
||||
|
||||
|
||||
def folders_with_totals(
|
||||
db: Session, user: User, listable: set[int] | None
|
||||
) -> list[tuple[FileFolder, int, int]]:
|
||||
"""The user's folders (only `listable` ones, when given) with how many done files each holds and
|
||||
their size. A source folder its files have all left stays (count 0)."""
|
||||
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
|
||||
stmt = select(
|
||||
FileFolder,
|
||||
func.count(DownloadJob.id),
|
||||
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
|
||||
).where(owned_by(FileFolder.user_id, user))
|
||||
if listable is not None:
|
||||
stmt = stmt.where(FileFolder.id.in_(listable))
|
||||
rows = db.execute(
|
||||
stmt.outerjoin(DownloadJob, done)
|
||||
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.group_by(FileFolder.id)
|
||||
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
|
||||
).tuples()
|
||||
return [(f, count, int(size or 0)) for f, count, size in rows]
|
||||
|
||||
|
||||
def folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
|
||||
count, size = db.execute(
|
||||
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
|
||||
.select_from(DownloadJob)
|
||||
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
|
||||
).one()
|
||||
return count, int(size)
|
||||
|
||||
|
||||
def linked_folders(db: Session, folder_ids: list[int]) -> set[int]:
|
||||
"""Which of these folders a live public link opens (one not past its expiry)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = db.execute(
|
||||
select(DownloadLink.folder_id).where(
|
||||
DownloadLink.folder_id.in_(folder_ids),
|
||||
or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now),
|
||||
)
|
||||
).scalars()
|
||||
return {folder_id for folder_id in rows if folder_id is not None}
|
||||
|
||||
|
||||
# --- links ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def own_link(db: Session, user: User, link_id: int) -> DownloadLink | None:
|
||||
"""A link to one of the user's files or folders."""
|
||||
link = db.get(DownloadLink, link_id)
|
||||
if link is None:
|
||||
return None
|
||||
target = (
|
||||
db.get(DownloadJob, link.job_id)
|
||||
if link.job_id is not None
|
||||
else db.get(FileFolder, link.folder_id)
|
||||
)
|
||||
if target is None or target.user_id != user.id:
|
||||
return None
|
||||
return link
|
||||
|
||||
|
||||
def links_to(
|
||||
db: Session, *, job_id: int | None = None, folder_id: int | None = None
|
||||
) -> list[DownloadLink]:
|
||||
"""The links to one file or one folder, newest first."""
|
||||
target = (
|
||||
DownloadLink.job_id == job_id if job_id is not None else DownloadLink.folder_id == folder_id
|
||||
)
|
||||
return list(
|
||||
db.execute(select(DownloadLink).where(target).order_by(DownloadLink.id.desc())).scalars()
|
||||
)
|
||||
|
||||
|
||||
def link_folder(db: Session, link: DownloadLink) -> FileFolder | None:
|
||||
return db.get(FileFolder, link.folder_id) if link.folder_id is not None else None
|
||||
|
||||
|
||||
def folder_job(db: Session, folder: FileFolder, job_id: int) -> DownloadJob | None:
|
||||
"""One of the files a link to `folder` reaches (`links.in_folder`), while it is in it."""
|
||||
return db.execute(
|
||||
select(DownloadJob).where(DownloadJob.id == job_id, *links.in_folder(folder))
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def owner_suspended(db: Session, job: DownloadJob) -> bool:
|
||||
"""The job's owner is suspended (or gone): the worker leaves their queued jobs where they are."""
|
||||
owner = db.get(User, job.user_id)
|
||||
return owner is None or owner.is_suspended
|
||||
|
||||
|
||||
# --- profiles ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def profiles_for(db: Session, user: User) -> list[DownloadProfile]:
|
||||
"""The built-in format profiles, then the user's own."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadProfile)
|
||||
.where(DownloadProfile.is_builtin.is_(True) | owned_by(DownloadProfile.user_id, user))
|
||||
.order_by(
|
||||
DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def own_profile(db: Session, user: User, profile_id: int) -> DownloadProfile | None:
|
||||
"""One of the user's own profiles (a built-in is no one's to change)."""
|
||||
p = owned_row(db, DownloadProfile, profile_id, user)
|
||||
return None if p is None or p.is_builtin else p
|
||||
|
||||
|
||||
def usable_profile(db: Session, user: User, profile_id: int) -> DownloadProfile | None:
|
||||
"""A profile the user may enqueue with: a built-in, or their own."""
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or (not p.is_builtin and p.user_id != user.id):
|
||||
return None
|
||||
return p
|
||||
|
||||
|
||||
def default_profile(db: Session) -> DownloadProfile | None:
|
||||
"""The first built-in, for an enqueue that named no format."""
|
||||
return db.execute(
|
||||
select(DownloadProfile)
|
||||
.where(DownloadProfile.is_builtin.is_(True))
|
||||
.order_by(DownloadProfile.sort_order)
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
# --- storage and quota (admin) -------------------------------------------------------------
|
||||
|
||||
|
||||
def storage_totals(db: Session) -> tuple[int, int, int]:
|
||||
"""(files on disk, their bytes, every asset row). On disk is `quota.ON_DISK`: ready files, and
|
||||
uploads waiting for or failed in their conversion — the same disk the footprint and the GC cap
|
||||
count."""
|
||||
on_disk = db.execute(
|
||||
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).one()
|
||||
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar_one()
|
||||
return on_disk[0], int(on_disk[1]), total_assets
|
||||
|
||||
|
||||
def done_files_per_user(db: Session) -> dict[int, int]:
|
||||
"""How many finished files each user's Files lists."""
|
||||
return dict(
|
||||
db.execute(
|
||||
select(DownloadJob.user_id, func.count())
|
||||
.where(DownloadJob.status == JobStatus.DONE)
|
||||
.group_by(DownloadJob.user_id)
|
||||
)
|
||||
.tuples()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def custom_quota(db: Session, user_id: int) -> DownloadQuota | None:
|
||||
"""The admin's own limits for this user, if any were set."""
|
||||
return db.get(DownloadQuota, user_id)
|
||||
@@ -26,7 +26,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.userscope import owned_by
|
||||
|
||||
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
||||
# (error, canceled) don't count.
|
||||
@@ -85,7 +86,7 @@ def footprint(db: Session, user_id: int) -> int:
|
||||
held_assets = (
|
||||
select(DownloadJob.asset_id)
|
||||
.where(
|
||||
DownloadJob.user_id == user_id,
|
||||
owned_by(DownloadJob.user_id, user_id),
|
||||
or_(
|
||||
DownloadJob.status.in_(_HOLDING),
|
||||
and_(DownloadJob.job_kind == JobKind.UPLOAD, DownloadJob.status == JobStatus.ERROR),
|
||||
@@ -109,7 +110,7 @@ def _count(db: Session, user_id: int, states) -> int:
|
||||
db.execute(
|
||||
select(func.count())
|
||||
.select_from(DownloadJob)
|
||||
.where(DownloadJob.user_id == user_id, DownloadJob.status.in_(states))
|
||||
.where(owned_by(DownloadJob.user_id, user_id), DownloadJob.status.in_(states))
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
@@ -201,3 +202,46 @@ def usage(db: Session, user_id: int) -> dict:
|
||||
"max_jobs": lim.max_jobs,
|
||||
"unlimited": lim.unlimited,
|
||||
}
|
||||
|
||||
|
||||
def set_custom(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
*,
|
||||
max_bytes: int | None = None,
|
||||
max_concurrent: int | None = None,
|
||||
max_jobs: int | None = None,
|
||||
unlimited: bool | None = None,
|
||||
) -> None:
|
||||
"""The admin's own limits for a user: starts from what applies now, then each value given (a
|
||||
size floored at 0, a count at 1). The handler commits."""
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
cur = resolve(db, user_id)
|
||||
row = DownloadQuota(
|
||||
user_id=user_id,
|
||||
max_bytes=cur.max_bytes,
|
||||
max_concurrent=cur.max_concurrent,
|
||||
max_jobs=cur.max_jobs,
|
||||
unlimited=cur.unlimited,
|
||||
)
|
||||
db.add(row)
|
||||
if max_bytes is not None:
|
||||
row.max_bytes = max(0, max_bytes)
|
||||
if max_concurrent is not None:
|
||||
row.max_concurrent = max(1, max_concurrent)
|
||||
if max_jobs is not None:
|
||||
row.max_jobs = max(1, max_jobs)
|
||||
if unlimited is not None:
|
||||
row.unlimited = unlimited
|
||||
db.flush()
|
||||
|
||||
|
||||
def reset_custom(db: Session, user_id: int) -> bool:
|
||||
"""Back to the defaults; False when the user had no limits of their own."""
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
return False
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
@@ -23,7 +23,7 @@ from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats
|
||||
from app.downloads.status import JobKind
|
||||
from app.models import DownloadJob
|
||||
from app.downloads.models import DownloadJob
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
"""Enqueue + cache resolution for the download center (the per-user layer over MediaAsset).
|
||||
"""Capture's writes: enqueue + cache resolution, the job transitions, and what the library's
|
||||
dialogs change (profiles, resume points, shares, a file's display fields).
|
||||
|
||||
`enqueue` is the single entry point used by the API: it resolves (or creates) the shared
|
||||
MediaAsset for the requested source+format, links a per-user DownloadJob to it, and — on a
|
||||
cache hit (asset already ready) — marks the job done instantly with no re-download. The worker
|
||||
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
handles everything else. Every function here flushes and leaves the commit to its caller (the
|
||||
handler); the folder, link and quota writes are `folders`, `links` and `quota`.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import delete as delete_rows
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -21,17 +24,17 @@ from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import folders, formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobKind, JobStatus
|
||||
from app.models import (
|
||||
DISPLAY_TEXT_MAX,
|
||||
Channel,
|
||||
from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, User, Video
|
||||
from app.downloads.models import (
|
||||
DownloadJob,
|
||||
DownloadProfile,
|
||||
DownloadShare,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
PlexItem,
|
||||
PlexShow,
|
||||
Video,
|
||||
)
|
||||
from app.plex import lookups as plex_lookups
|
||||
from app.plex import paths as plex_paths
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
|
||||
class EditError(Exception):
|
||||
@@ -230,8 +233,7 @@ def enqueue(
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
db.flush()
|
||||
return job
|
||||
|
||||
|
||||
@@ -310,8 +312,7 @@ def enqueue_edit(
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
db.flush()
|
||||
return job
|
||||
|
||||
|
||||
@@ -399,8 +400,7 @@ def enqueue_clip(
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
db.flush()
|
||||
return job
|
||||
|
||||
|
||||
@@ -508,8 +508,8 @@ def release_asset(db: Session, job: DownloadJob) -> None:
|
||||
# --- Job state transitions ---------------------------------------------------------------------
|
||||
# The single home for the pause/resume/cancel/delete/redownload transitions + their asset
|
||||
# bookkeeping (release, failed-asset reset, fresh-asset rebuild), so the routes are thin callers and
|
||||
# the worker/GC could share the same definitions (B-sub-8). Each commits its own transition; the
|
||||
# route keeps only auth + HTTP mapping + serialization.
|
||||
# the worker/GC could share the same definitions (B-sub-8). Each flushes its transition and the caller
|
||||
# commits it; the route keeps only auth + HTTP mapping + serialization.
|
||||
|
||||
|
||||
def pause(db: Session, job: DownloadJob) -> None:
|
||||
@@ -517,7 +517,7 @@ def pause(db: Session, job: DownloadJob) -> None:
|
||||
in any other state."""
|
||||
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING):
|
||||
job.status = JobStatus.PAUSED
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
def resume(db: Session, job: DownloadJob) -> None:
|
||||
@@ -541,7 +541,7 @@ def resume(db: Session, job: DownloadJob) -> None:
|
||||
asset.status = "pending"
|
||||
asset.error = None
|
||||
job.status = JobStatus.QUEUED
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset:
|
||||
@@ -576,14 +576,14 @@ def cancel(db: Session, job: DownloadJob) -> None:
|
||||
if job.status not in (JobStatus.DONE, JobStatus.CANCELED):
|
||||
release_asset(db, job) # release while the job still counts as holding
|
||||
job.status = JobStatus.CANCELED
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
def delete(db: Session, job: DownloadJob) -> None:
|
||||
"""Delete a job, releasing its asset hold first (which frees the file iff this was the last holder)."""
|
||||
release_asset(db, job)
|
||||
db.delete(job)
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
def redownload(db: Session, job: DownloadJob) -> None:
|
||||
@@ -632,4 +632,181 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
||||
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
|
||||
job.expires_at = None # stamped again when the rebuild is done
|
||||
job.expiry_warned = False
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
# --- the library's writes ------------------------------------------------------------------
|
||||
# What the HTTP layer changes for a user: each flushes, the handler commits.
|
||||
|
||||
|
||||
def lock_own_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
|
||||
"""One of the user's jobs, re-read under a row lock, for a transition that moves its asset's ref
|
||||
count (resume, cancel, delete, redownload) or writes the status those decide on (pause). Two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
just released back into a holder; either can free a file another job still holds. None: not
|
||||
theirs, or gone.
|
||||
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
owned = owned_row(db, DownloadJob, job_id, user)
|
||||
if owned is None:
|
||||
return None
|
||||
asset_id = owned.asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
return None
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
|
||||
|
||||
def hold_for_share(db: Session, user: User, job_id: int) -> DownloadJob | None:
|
||||
"""One of the user's files, held (FOR SHARE) for the rest of the transaction: a delete of it at
|
||||
the same moment (the user's, or the retention sweep's) waits and takes the new link with it,
|
||||
rather than failing the link's foreign key — or, done first, leaves None."""
|
||||
if owned_row(db, DownloadJob, job_id, user) is None:
|
||||
return None
|
||||
return db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update(read=True)
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def create_profile(db: Session, user: User, name: str, spec: dict) -> DownloadProfile:
|
||||
"""A format profile of the user's own, after their others."""
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
|
||||
owned_by(DownloadProfile.user_id, user)
|
||||
)
|
||||
).scalar_one()
|
||||
p = DownloadProfile(
|
||||
user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1
|
||||
)
|
||||
db.add(p)
|
||||
db.flush()
|
||||
return p
|
||||
|
||||
|
||||
def update_profile(profile: DownloadProfile, *, name: str | None, spec: dict | None) -> None:
|
||||
"""Rename a profile and/or replace its spec; None leaves that one as it is."""
|
||||
if name is not None:
|
||||
profile.name = name[:80]
|
||||
if spec is not None:
|
||||
profile.spec = spec
|
||||
|
||||
|
||||
def delete_profile(db: Session, profile: DownloadProfile) -> None:
|
||||
db.delete(profile)
|
||||
db.flush()
|
||||
|
||||
|
||||
def set_position(db: Session, user: User, job_id: int, position: int) -> None:
|
||||
"""The library player's resume point in a file the user can play; 0 clears it.
|
||||
|
||||
A clear is a statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon
|
||||
of one stop may both clear it, and the second then deletes nothing rather than failing. A set is
|
||||
an upsert, one row per (user, job) even when two checkpoints race, not a read-then-insert that
|
||||
the second would fail on the unique key."""
|
||||
if not position:
|
||||
db.execute(
|
||||
delete_rows(LibraryState).where(
|
||||
owned_by(LibraryState.user_id, user), LibraryState.job_id == job_id
|
||||
)
|
||||
)
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
db.execute(
|
||||
pg_insert(LibraryState)
|
||||
.values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_library_user_job",
|
||||
set_={"position_seconds": position, "progress_updated_at": now},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def share(db: Session, job: DownloadJob, sender: User, recipient: User) -> bool:
|
||||
"""Hand a file to another user, with a notice for them; False when it was already theirs."""
|
||||
from app.notifications import create_notification
|
||||
|
||||
exists = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job.id, owned_by(DownloadShare.to_user_id, recipient)
|
||||
)
|
||||
).first()
|
||||
if exists is not None:
|
||||
return False
|
||||
db.add(DownloadShare(job_id=job.id, from_user_id=sender.id, to_user_id=recipient.id))
|
||||
create_notification(
|
||||
db,
|
||||
user_id=recipient.id,
|
||||
type="download_shared",
|
||||
title=(job.display_name or "A download") + " was shared with you",
|
||||
# The client names the notice in the reader's language from `data`; the title is the
|
||||
# English fallback only.
|
||||
data={
|
||||
"kind": "download_shared",
|
||||
"from": sender.email,
|
||||
"job_id": job.id,
|
||||
"title": job.display_name,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
|
||||
def unshare(db: Session, job_id: int, recipient: User) -> bool:
|
||||
"""Take a file back from one user (its owner's act, or the recipient's own dismissal); False
|
||||
when it was not theirs."""
|
||||
row = db.execute(
|
||||
select(DownloadShare).where(
|
||||
DownloadShare.job_id == job_id, owned_by(DownloadShare.to_user_id, recipient)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return False
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
|
||||
# The display fields a user may set on their file; the file on disk keeps its own name.
|
||||
DISPLAY_FIELDS = frozenset(
|
||||
{"display_name", "display_uploader", "display_uploader_url", "extra_links"}
|
||||
)
|
||||
|
||||
|
||||
def set_display(job: DownloadJob, **fields: object) -> None:
|
||||
"""Set a file's display fields (each already cleaned by the caller); a field not given is left."""
|
||||
unknown = set(fields) - DISPLAY_FIELDS
|
||||
if unknown:
|
||||
raise ValueError(f"not a display field: {sorted(unknown)}")
|
||||
for name, value in fields.items():
|
||||
setattr(job, name, value)
|
||||
|
||||
|
||||
def touch(asset: MediaAsset) -> None:
|
||||
"""The file was just served: the cache's idle clock starts again."""
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def store_sha256(db: Session, asset_id: int, digest: str) -> None:
|
||||
"""Keep a file's hash once it is known — only where none is, so a concurrent hash of the same
|
||||
file never overwrites the first."""
|
||||
db.execute(
|
||||
update(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
|
||||
.values(sha256=digest)
|
||||
)
|
||||
|
||||
@@ -37,7 +37,8 @@ from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
from app.models import display_text
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def _title_from_filename(filename: str) -> str:
|
||||
|
||||
+1
-360
@@ -1,4 +1,4 @@
|
||||
from datetime import date, datetime, timezone
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
@@ -18,7 +18,6 @@ from sqlalchemy import (
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
@@ -465,27 +464,6 @@ class VideoState(Base, UpdatedAtMixin):
|
||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class LibraryState(Base):
|
||||
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
||||
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
||||
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
||||
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
||||
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
||||
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
||||
|
||||
__tablename__ = "library_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class SearchFind(Base, TimestampMixin):
|
||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||
|
||||
@@ -861,141 +839,6 @@ class Notification(Base):
|
||||
)
|
||||
|
||||
|
||||
class MediaAsset(Base, TimestampMixin):
|
||||
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
||||
|
||||
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
|
||||
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
|
||||
the same video with a format that hashes to the same `format_sig`, their `download_job`
|
||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||
|
||||
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
||||
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
||||
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
||||
|
||||
__tablename__ = "media_assets"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
|
||||
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="pending", server_default="pending", index=True
|
||||
) # "pending" | "ready" | "error"
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||
# card / watch page / link preview can still show an image. Relative to files_root.
|
||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||
width: Mapped[int | None] = mapped_column(Integer)
|
||||
height: Mapped[int | None] = mapped_column(Integer)
|
||||
container: Mapped[str | None] = mapped_column(String(16))
|
||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
|
||||
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
|
||||
# still pending (kind is known once the file is probed at finalize). It is the single stored home
|
||||
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
|
||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
|
||||
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
|
||||
uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
||||
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
||||
# fall back to a URL derived from source_kind+source_ref).
|
||||
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
|
||||
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
|
||||
sha256: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
@validates("rel_path")
|
||||
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
|
||||
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
|
||||
self.sha256 = None
|
||||
return rel_path
|
||||
|
||||
|
||||
class DownloadProfile(Base, TimestampMixin):
|
||||
"""A named download preset — a reusable format profile.
|
||||
|
||||
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
|
||||
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
|
||||
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
|
||||
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
|
||||
`format_sig`."""
|
||||
|
||||
__tablename__ = "download_profiles"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
) # NULL for a builtin preset
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
spec: Mapped[dict] = mapped_column(JSON)
|
||||
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class FileFolder(Base, TimestampMixin):
|
||||
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
||||
|
||||
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
||||
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
|
||||
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
|
||||
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
||||
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
||||
|
||||
__tablename__ = "file_folders"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||
Index(
|
||||
"uq_file_folder_own_name",
|
||||
"user_id",
|
||||
func.lower(text("name")),
|
||||
unique=True,
|
||||
postgresql_where=text("name IS NOT NULL"),
|
||||
),
|
||||
# One Hub folder per user.
|
||||
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
|
||||
CheckConstraint(
|
||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
|
||||
name="ck_file_folder_kind",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
||||
# "site:" + a host name (DNS caps one at 253 characters).
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
retention_days: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
class ApiToken(Base, TimestampMixin):
|
||||
"""A user's token for a machine to reach their Files library with (`app.apitokens`): sent as
|
||||
`Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its
|
||||
@@ -1037,208 +880,6 @@ class ApiTokenGrant(Base):
|
||||
can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
def _ready_on_insert(context) -> datetime | None:
|
||||
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
|
||||
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
|
||||
done = context.get_current_parameters().get("status") == "done"
|
||||
return datetime.now(timezone.utc) if done else None
|
||||
|
||||
|
||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||
|
||||
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
|
||||
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
|
||||
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
||||
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
||||
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
||||
(SET NULL if the asset is GC'd — the job then shows as expired).
|
||||
|
||||
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
|
||||
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
|
||||
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
|
||||
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
|
||||
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
||||
|
||||
__tablename__ = "download_jobs"
|
||||
__table_args__ = (
|
||||
Index("ix_download_jobs_claim", "status", "queue_pos"),
|
||||
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
|
||||
CheckConstraint(
|
||||
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512))
|
||||
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
|
||||
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
|
||||
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
|
||||
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||
source_job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="SET NULL")
|
||||
)
|
||||
source_asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL")
|
||||
)
|
||||
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
|
||||
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
|
||||
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
|
||||
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
||||
profile_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="queued", server_default="queued", index=True
|
||||
) # queued | running | paused | done | error | canceled
|
||||
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
|
||||
eta_s: Mapped[int | None] = mapped_column(Integer)
|
||||
phase: Mapped[str | None] = mapped_column(String(32))
|
||||
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
||||
error_code: Mapped[str | None] = mapped_column(
|
||||
String(24)
|
||||
) # category, see downloads.errors.classify
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
||||
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
||||
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
||||
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||
)
|
||||
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
||||
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
||||
# file must still know which source folder it may go back to. NULL only for a file placed before
|
||||
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# When the file became done — what the library's "Added" means: a download that failed for days
|
||||
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
|
||||
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
|
||||
ready_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), default=_ready_on_insert
|
||||
)
|
||||
|
||||
|
||||
class DownloadShare(Base, TimestampMixin):
|
||||
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
|
||||
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
||||
|
||||
__tablename__ = "download_shares"
|
||||
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
|
||||
from_user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
|
||||
class DownloadQuota(Base, UpdatedAtMixin):
|
||||
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
|
||||
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
|
||||
byte cap entirely (e.g. for the admin's own account)."""
|
||||
|
||||
__tablename__ = "download_quotas"
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
max_concurrent: Mapped[int] = mapped_column(Integer)
|
||||
max_jobs: Mapped[int] = mapped_column(Integer)
|
||||
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
class DownloadLink(Base, TimestampMixin):
|
||||
"""A public capability link to one download, or to one of its owner's own folders — the "share
|
||||
by link" surface.
|
||||
|
||||
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
|
||||
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
|
||||
the folder at the time it is opened. The token itself is never stored as written: a link is
|
||||
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
|
||||
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
|
||||
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
|
||||
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
|
||||
grants a *registered* user access via the in-app "Shared with me" list."""
|
||||
|
||||
__tablename__ = "download_links"
|
||||
__table_args__ = (
|
||||
# A link opens exactly one thing: a file or a folder.
|
||||
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
password_hash: Mapped[str | None] = mapped_column(Text)
|
||||
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
|
||||
# password immediately invalidates any grants issued under the old one (instead of them living
|
||||
# out the 6h grant TTL).
|
||||
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
|
||||
|
||||
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
|
||||
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
|
||||
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
|
||||
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
|
||||
the routes are write-only (host + updated_at only). See `downloads.credentials`.
|
||||
|
||||
Two load-bearing security controls live around this row, NOT on it:
|
||||
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
|
||||
user id, so the authed asset is private to that user and can never be cache-served to
|
||||
another user (`downloads.credentials.cookie_salt`).
|
||||
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
|
||||
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
|
||||
can't exfiltrate the user's session to a look-alike host.
|
||||
|
||||
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
|
||||
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
|
||||
|
||||
__tablename__ = "download_credentials"
|
||||
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
host_key: Mapped[str] = mapped_column(String(64))
|
||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
|
||||
|
||||
# --- Plex integration (optional) -------------------------------------------------------------
|
||||
# A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is
|
||||
# from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states
|
||||
|
||||
@@ -16,6 +16,7 @@ from app.db import Base
|
||||
from app.models import POSITION_MAX_S, User
|
||||
from app.issues import gitea
|
||||
from app.refusal import refused
|
||||
from app.userscope import owned_row
|
||||
from app.youtube.client import YouTubeError, refusal_reason
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
@@ -90,8 +91,8 @@ def owned_or_404(
|
||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
||||
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
||||
obj = get_or_404(db, model, id_, what=what, lock=lock)
|
||||
if getattr(obj, owner_attr) != user.id:
|
||||
obj = owned_row(db, model, id_, user, owner_attr=owner_attr, lock=lock)
|
||||
if obj is None:
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
+106
-337
@@ -3,23 +3,22 @@
|
||||
All routes are behind `current_user`; admin routes add `admin_user`. Live job state is polled by the
|
||||
frontend (the worker is a separate process); this module never downloads — it enqueues, manages job
|
||||
state, and shares finished files. The library itself — listing, renaming, moving, deleting and
|
||||
serving a done file, and uploading one — is `routes.files`.
|
||||
serving a done file, and uploading one — is `routes.files`. Reads come from `downloads.queries`,
|
||||
writes go through Capture's service (`service`, `links`, `quota`); this module validates, answers in
|
||||
HTTP and commits.
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Literal
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated
|
||||
from app import audit, sysconfig
|
||||
from app.auth import admin_user, current_user
|
||||
from app.config import settings
|
||||
@@ -29,45 +28,32 @@ from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import folders
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads import queries, quota, service, storage
|
||||
from app.downloads import urlguard
|
||||
from app.downloads.folders import FolderKind
|
||||
from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus
|
||||
from app.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
FileFolder,
|
||||
DownloadProfile,
|
||||
DownloadQuota,
|
||||
DownloadShare,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
User,
|
||||
display_text,
|
||||
)
|
||||
from app.downloads.status import NOT_REFETCHABLE, JobStatus
|
||||
from app.models import User, display_text
|
||||
from app.routes._common import (
|
||||
ProgressIn,
|
||||
get_or_404,
|
||||
kept_position,
|
||||
not_found,
|
||||
owned_or_404,
|
||||
)
|
||||
from app.routes.files import (
|
||||
MAX_LIST,
|
||||
accessible_job,
|
||||
assets_for,
|
||||
own_folder,
|
||||
own_job,
|
||||
own_job_for_update,
|
||||
positions_for,
|
||||
quota_detail,
|
||||
serialize,
|
||||
serialize_job,
|
||||
source_keys,
|
||||
)
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.userscope import emails_of, is_messageable_user, messageable_others, user_by_email
|
||||
from app.refusal import refused
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.downloads.models import DownloadJob, DownloadLink, DownloadProfile, FileFolder
|
||||
|
||||
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
|
||||
admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"])
|
||||
|
||||
@@ -119,7 +105,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
# --- profiles ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _serialize_profile(p: DownloadProfile) -> dict:
|
||||
def _serialize_profile(p: "DownloadProfile") -> dict:
|
||||
return {
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
@@ -131,18 +117,7 @@ def _serialize_profile(p: DownloadProfile) -> dict:
|
||||
|
||||
@router.get("/profiles")
|
||||
def list_profiles(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
rows = (
|
||||
db.execute(
|
||||
select(DownloadProfile)
|
||||
.where((DownloadProfile.is_builtin.is_(True)) | (DownloadProfile.user_id == user.id))
|
||||
.order_by(
|
||||
DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return [_serialize_profile(p) for p in rows]
|
||||
return [_serialize_profile(p) for p in queries.profiles_for(db, user)]
|
||||
|
||||
|
||||
class CreateProfileIn(BaseModel):
|
||||
@@ -158,15 +133,7 @@ def create_profile(
|
||||
spec = body.spec
|
||||
if not name:
|
||||
raise refused(400, "profile_name_required")
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
|
||||
DownloadProfile.user_id == user.id
|
||||
)
|
||||
).scalar_one()
|
||||
p = DownloadProfile(
|
||||
user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1
|
||||
)
|
||||
db.add(p)
|
||||
p = service.create_profile(db, user, name, spec)
|
||||
db.commit()
|
||||
return _serialize_profile(p)
|
||||
|
||||
@@ -183,31 +150,32 @@ def update_profile(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise not_found("profile")
|
||||
p = _own_profile(db, user, profile_id)
|
||||
sent = body.model_fields_set
|
||||
name = None
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise refused(400, "profile_name_required")
|
||||
p.name = name[:80]
|
||||
if "spec" in sent:
|
||||
if body.spec is None:
|
||||
raise refused(400, "spec_not_object")
|
||||
p.spec = body.spec
|
||||
if "spec" in sent and body.spec is None:
|
||||
raise refused(400, "spec_not_object")
|
||||
service.update_profile(p, name=name, spec=body.spec if "spec" in sent else None)
|
||||
db.commit()
|
||||
return _serialize_profile(p)
|
||||
|
||||
|
||||
def _own_profile(db: Session, user: User, profile_id: int) -> "DownloadProfile":
|
||||
p = queries.own_profile(db, user, profile_id)
|
||||
if p is None:
|
||||
raise not_found("profile")
|
||||
return p
|
||||
|
||||
|
||||
@router.delete("/profiles/{profile_id}")
|
||||
def delete_profile(
|
||||
profile_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise not_found("profile")
|
||||
db.delete(p)
|
||||
service.delete_profile(db, _own_profile(db, user, profile_id))
|
||||
db.commit()
|
||||
return {"deleted": profile_id}
|
||||
|
||||
@@ -277,18 +245,13 @@ def _resolve_spec(
|
||||
"""Pick the format spec for an enqueue: an explicit profile_id (builtin or the user's own),
|
||||
an inline spec, or the first builtin as a fallback."""
|
||||
if profile_id is not None:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or (not p.is_builtin and p.user_id != user.id):
|
||||
p = queries.usable_profile(db, user, profile_id)
|
||||
if p is None:
|
||||
raise not_found("profile")
|
||||
return p.spec, p.id
|
||||
if spec is not None:
|
||||
return spec, None
|
||||
fallback = db.execute(
|
||||
select(DownloadProfile)
|
||||
.where(DownloadProfile.is_builtin.is_(True))
|
||||
.order_by(DownloadProfile.sort_order)
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
fallback = queries.default_profile(db)
|
||||
if fallback is None:
|
||||
raise refused(400, "no_profile")
|
||||
return fallback.spec, fallback.id
|
||||
@@ -326,6 +289,7 @@ def enqueue_download(
|
||||
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -355,6 +319,7 @@ def enqueue_edit(
|
||||
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
|
||||
except service.EditError as e:
|
||||
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -398,38 +363,22 @@ def enqueue_clip(
|
||||
raise HTTPException(status_code=status, detail=detail) from e
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
# How long a finished job stays in the queue: the tray polls it from every page and must see a job
|
||||
# turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's.
|
||||
# A done job is timed from when it became done, never from its last write: a rename, a move or the
|
||||
# GC's expiry warning touches an old file's row, and the tray would announce it as just finished.
|
||||
QUEUE_RECENT = timedelta(minutes=10)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_downloads(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
"""The queue: what is waiting, running, paused or failed, and what finished a moment ago."""
|
||||
recent = datetime.now(timezone.utc) - QUEUE_RECENT
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(
|
||||
DownloadJob.user_id == user.id,
|
||||
DownloadJob.status.not_in(FINISHED)
|
||||
| ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent))
|
||||
| ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)),
|
||||
)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assets = assets_for(db, jobs)
|
||||
positions = positions_for(db, user, jobs)
|
||||
sources = source_keys(db, jobs)
|
||||
jobs = queries.queue(db, user, datetime.now(timezone.utc) - QUEUE_RECENT)
|
||||
assets = queries.assets_for(db, jobs)
|
||||
positions = queries.positions_for(db, user, jobs)
|
||||
sources = queries.source_keys(db, jobs)
|
||||
return [
|
||||
{
|
||||
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
|
||||
@@ -453,14 +402,8 @@ _INDEX_RANK: dict[str, int] = {
|
||||
def my_download_index(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""source_ref -> best active status, for the feed's per-card "downloaded / queued" badge.
|
||||
Small + cheap so the feed can poll it without loading the full job list."""
|
||||
rows = db.execute(
|
||||
select(DownloadJob.source_ref, DownloadJob.status).where(
|
||||
DownloadJob.user_id == user.id,
|
||||
DownloadJob.status.in_(tuple(_INDEX_RANK)),
|
||||
)
|
||||
).all()
|
||||
out: dict[str, str] = {}
|
||||
for ref, status in rows:
|
||||
for ref, status in queries.statuses_by_source(db, user, tuple(_INDEX_RANK)):
|
||||
if _INDEX_RANK.get(status, 0) > _INDEX_RANK.get(out.get(ref, ""), 0):
|
||||
out[ref] = status
|
||||
return out
|
||||
@@ -472,6 +415,7 @@ def pause_download(
|
||||
) -> dict:
|
||||
job = own_job_for_update(db, user, job_id)
|
||||
service.pause(db, job)
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -481,6 +425,7 @@ def resume_download(
|
||||
) -> dict:
|
||||
job = own_job_for_update(db, user, job_id)
|
||||
service.resume(db, job)
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -508,6 +453,7 @@ def redownload_download(
|
||||
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED):
|
||||
raise refused(409, "still_downloading")
|
||||
service.redownload(db, job)
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -517,6 +463,7 @@ def cancel_download(
|
||||
) -> dict:
|
||||
job = own_job_for_update(db, user, job_id)
|
||||
service.cancel(db, job)
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -534,24 +481,7 @@ def set_library_progress(
|
||||
one shared with them). A position too early or too close to the end clears it instead."""
|
||||
accessible_job(db, user, job_id)
|
||||
position = kept_position(body)
|
||||
mine = (LibraryState.user_id == user.id) & (LibraryState.job_id == job_id)
|
||||
if not position:
|
||||
# A statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon of
|
||||
# one stop may both clear it, and the second then deletes nothing rather than failing.
|
||||
db.execute(delete(LibraryState).where(mine))
|
||||
db.commit()
|
||||
return {"ok": True, "job_id": job_id, "position_seconds": 0}
|
||||
now = datetime.now(timezone.utc)
|
||||
# One row per (user, job) even when two checkpoints race (the same pair): an upsert, not a
|
||||
# read-then-insert that the second would fail on the unique key.
|
||||
db.execute(
|
||||
pg_insert(LibraryState)
|
||||
.values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_library_user_job",
|
||||
set_={"position_seconds": position, "progress_updated_at": now},
|
||||
)
|
||||
)
|
||||
service.set_position(db, user, job_id, position)
|
||||
db.commit()
|
||||
return {"ok": True, "job_id": job_id, "position_seconds": position}
|
||||
|
||||
@@ -564,8 +494,7 @@ def storyboard_meta(
|
||||
|
||||
Best-effort: for a very long/high-res source the sprite may not generate in time — then
|
||||
`available` is False and the editor shows a plain timeline."""
|
||||
job = accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
asset = queries.asset_of(db, accessible_job(db, user, job_id))
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise refused(409, "download_not_ready")
|
||||
meta = editmod.ensure_storyboard(asset, settings.files_root)
|
||||
@@ -585,8 +514,7 @@ def storyboard_meta(
|
||||
@router.get("/{job_id}/poster.jpg")
|
||||
def poster_image(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
|
||||
"""The locally-generated poster frame for a download whose source had no thumbnail."""
|
||||
job = accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
asset = queries.asset_of(db, accessible_job(db, user, job_id))
|
||||
if asset is None or not asset.poster_path:
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||
@@ -599,8 +527,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session =
|
||||
def storyboard_image(
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
):
|
||||
job = accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
asset = queries.asset_of(db, accessible_job(db, user, job_id))
|
||||
if asset is None or not asset.storyboard_path:
|
||||
raise not_found("filmstrip")
|
||||
path = storage.safe_abs_path(settings.files_root, asset.storyboard_path)
|
||||
@@ -623,45 +550,20 @@ def share_download(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
job = own_job(db, user, job_id)
|
||||
if job.status != JobStatus.DONE or job.asset_id is None:
|
||||
raise refused(409, "share_unfinished")
|
||||
target = body.email.strip().lower()
|
||||
if not target:
|
||||
raise refused(400, "recipient_required")
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == target)
|
||||
).scalar_one_or_none()
|
||||
recipient = user_by_email(db, target)
|
||||
# Uniform "no user" for a missing OR non-messageable (suspended/deactivated) target, so
|
||||
# sharing can't be used to probe which addresses exist or to reach a suspended account.
|
||||
if not is_messageable_user(recipient):
|
||||
if recipient is None or not is_messageable_user(recipient):
|
||||
raise not_found("recipient")
|
||||
if recipient.id == user.id:
|
||||
raise refused(400, "share_own")
|
||||
exists = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id
|
||||
)
|
||||
).first()
|
||||
if exists is None:
|
||||
db.add(DownloadShare(job_id=job_id, from_user_id=user.id, to_user_id=recipient.id))
|
||||
from app.notifications import create_notification
|
||||
|
||||
create_notification(
|
||||
db,
|
||||
user_id=recipient.id,
|
||||
type="download_shared",
|
||||
title=(job.display_name or "A download") + " was shared with you",
|
||||
# The client names the notice in the reader's language from `data`; the title is the
|
||||
# English fallback only.
|
||||
data={
|
||||
"kind": "download_shared",
|
||||
"from": user.email,
|
||||
"job_id": job_id,
|
||||
"title": job.display_name,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
if service.share(db, job, user, recipient):
|
||||
db.commit()
|
||||
return {"ok": True, "shared_with": recipient.email}
|
||||
|
||||
@@ -673,22 +575,10 @@ def unshare_download(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
owned_or_404(
|
||||
db, DownloadJob, job_id, user, what="download"
|
||||
) # ownership guard (404s if not the caller's job)
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == email.strip().lower())
|
||||
).scalar_one_or_none()
|
||||
if recipient is not None:
|
||||
row = db.execute(
|
||||
select(DownloadShare).where(
|
||||
DownloadShare.job_id == job_id,
|
||||
DownloadShare.to_user_id == recipient.id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
own_job(db, user, job_id) # ownership guard (404s if not the caller's job)
|
||||
recipient = user_by_email(db, email)
|
||||
if recipient is not None and service.unshare(db, job_id, recipient):
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@@ -701,54 +591,26 @@ def share_recipients(
|
||||
) -> list[dict]:
|
||||
"""Registered users this user can share a download with (excludes self + any non-messageable
|
||||
account — suspended or deactivated — so the picker never leaks those addresses)."""
|
||||
rows = (
|
||||
db.execute(
|
||||
select(User)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(collated(User.email, lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
rows = messageable_others(db, user, lang_for_user(user))
|
||||
return [{"id": u.id, "email": u.email, "display_name": u.display_name} for u in rows]
|
||||
|
||||
|
||||
# --- public watch links (share by link) ----------------------------------------------------
|
||||
|
||||
|
||||
def _own_link(db: Session, user: User, link_id: int) -> DownloadLink:
|
||||
link = db.get(DownloadLink, link_id)
|
||||
def _own_link(db: Session, user: User, link_id: int) -> "DownloadLink":
|
||||
link = queries.own_link(db, user, link_id)
|
||||
if link is None:
|
||||
raise not_found("link")
|
||||
target = (
|
||||
db.get(DownloadJob, link.job_id)
|
||||
if link.job_id is not None
|
||||
else db.get(FileFolder, link.folder_id)
|
||||
)
|
||||
if target is None or target.user_id != user.id:
|
||||
raise not_found("link")
|
||||
return link
|
||||
|
||||
|
||||
def _link_expiry(days: int | None) -> datetime | None:
|
||||
# None/0/negative → no expiry; otherwise N days out, capped at ~10 years. The Pydantic model
|
||||
# already coerced/validated the type, so a non-number is a 422 before we get here.
|
||||
if not days or days <= 0:
|
||||
return None
|
||||
return datetime.now(timezone.utc) + timedelta(days=min(days, 3650))
|
||||
|
||||
|
||||
def _links_of(db: Session, *where) -> list[dict]:
|
||||
rows = db.execute(select(DownloadLink).where(*where).order_by(DownloadLink.id.desc()))
|
||||
return [linksmod.owner_view(row) for row in rows.scalars()]
|
||||
|
||||
|
||||
@router.get("/{job_id}/links")
|
||||
def list_links(
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
return _links_of(db, DownloadLink.job_id == job_id)
|
||||
own_job(db, user, job_id)
|
||||
return [linksmod.owner_view(link) for link in queries.links_to(db, job_id=job_id)]
|
||||
|
||||
|
||||
class CreateLinkIn(BaseModel):
|
||||
@@ -764,30 +626,18 @@ class CreateLinkIn(BaseModel):
|
||||
|
||||
def _new_link(
|
||||
db: Session, user: User, body: CreateLinkIn, **target: int
|
||||
) -> tuple[DownloadLink, str]:
|
||||
"""Make a link to `target` (`job_id=` or `folder_id=`, already checked to be the user's), and
|
||||
the token it is opened by — which is in hand only now."""
|
||||
pw = body.password.strip()
|
||||
token = linksmod.new_token()
|
||||
link = DownloadLink(
|
||||
**target,
|
||||
created_by=user.id,
|
||||
allow_download=body.allow_download,
|
||||
password_hash=hash_password(pw) if pw else None,
|
||||
expires_at=_link_expiry(body.expires_days),
|
||||
)
|
||||
linksmod.seal(link, token)
|
||||
db.add(link)
|
||||
db.flush()
|
||||
audit.record(
|
||||
) -> tuple["DownloadLink", str]:
|
||||
"""A link to `target` (`job_id=` or `folder_id=`, already checked to be the user's) and its
|
||||
token (`links.create`). The Pydantic model already coerced the expiry, so a non-number is a 422
|
||||
before this runs."""
|
||||
return linksmod.create(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_CREATE,
|
||||
target_type="download_link",
|
||||
target_id=link.id,
|
||||
after={**target, **_link_audit(link)},
|
||||
user,
|
||||
password=body.password.strip(),
|
||||
allow_download=body.allow_download,
|
||||
expires_days=body.expires_days,
|
||||
**target,
|
||||
)
|
||||
return link, token
|
||||
|
||||
|
||||
@router.post("/{job_id}/links")
|
||||
@@ -805,28 +655,20 @@ def create_link(
|
||||
return linksmod.owner_view(link, token=token)
|
||||
|
||||
|
||||
def _shareable_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""One of the user's files, held (FOR SHARE) for the rest of the transaction: a delete of it at
|
||||
the same moment (the user's, or the retention sweep's) waits and takes the new link with it,
|
||||
rather than failing the link's foreign key — or, done first, leaves a 404."""
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update(read=True)
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
def _shareable_job(db: Session, user: User, job_id: int) -> "DownloadJob":
|
||||
"""One of the user's files, held for the link's transaction (`service.hold_for_share`)."""
|
||||
job = service.hold_for_share(db, user, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder:
|
||||
def _shareable_folder(db: Session, user: User, folder_id: int) -> "FileFolder":
|
||||
"""One of the user's own folders. A source folder lets its files go on its own clock, and the Hub
|
||||
is the user's channel to a machine of theirs: neither is a shelf to hand out. Held (FOR SHARE)
|
||||
for the rest of the transaction, so a delete of it at the same moment waits and takes the new
|
||||
link with it, rather than failing the link's foreign key."""
|
||||
owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||
own_folder(db, user, folder_id)
|
||||
try:
|
||||
folder = folders.own_folder(db, user.id, folder_id)
|
||||
except folders.FolderError as e:
|
||||
@@ -840,8 +682,8 @@ def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder:
|
||||
def list_folder_links(
|
||||
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||
return _links_of(db, DownloadLink.folder_id == folder_id)
|
||||
own_folder(db, user, folder_id)
|
||||
return [linksmod.owner_view(link) for link in queries.links_to(db, folder_id=folder_id)]
|
||||
|
||||
|
||||
@router.post("/folders/{folder_id}/links")
|
||||
@@ -857,15 +699,6 @@ def create_folder_link(
|
||||
return linksmod.owner_view(link, token=token)
|
||||
|
||||
|
||||
def _link_audit(link: DownloadLink) -> dict:
|
||||
"""A link's settings as the audit records them: whether it has a password, never which."""
|
||||
return {
|
||||
"allow_download": link.allow_download,
|
||||
"has_password": link.password_hash is not None,
|
||||
"expires_at": iso(link.expires_at),
|
||||
}
|
||||
|
||||
|
||||
class UpdateLinkIn(BaseModel):
|
||||
allow_download: bool | None = None
|
||||
expires_days: int | None = None
|
||||
@@ -880,29 +713,15 @@ def update_link(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
link = _own_link(db, user, link_id)
|
||||
before = _link_audit(link)
|
||||
sent = body.model_fields_set
|
||||
if "allow_download" in sent:
|
||||
link.allow_download = bool(body.allow_download)
|
||||
if "expires_days" in sent:
|
||||
link.expires_at = _link_expiry(body.expires_days)
|
||||
if "password" in sent:
|
||||
pw = (body.password or "").strip()
|
||||
link.password_hash = hash_password(pw) if pw else None
|
||||
# Bump the version so any grant issued under the previous password stops working now, rather
|
||||
# than living out its 6h TTL (setting/clearing/changing the password all count as a rotation).
|
||||
link.password_version += 1
|
||||
after = _link_audit(link)
|
||||
if after != before or "password" in sent:
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_UPDATE,
|
||||
target_type="download_link",
|
||||
target_id=link_id,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
linksmod.update(
|
||||
db,
|
||||
user,
|
||||
link,
|
||||
allow_download=body.allow_download,
|
||||
expires_days=body.expires_days,
|
||||
password=body.password,
|
||||
sent=frozenset(body.model_fields_set),
|
||||
)
|
||||
db.commit()
|
||||
return linksmod.owner_view(link)
|
||||
|
||||
@@ -911,16 +730,7 @@ def update_link(
|
||||
def revoke_link(
|
||||
link_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
link = _own_link(db, user, link_id)
|
||||
db.delete(link)
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_REVOKE,
|
||||
target_type="download_link",
|
||||
target_id=link_id,
|
||||
after={"job_id": link.job_id} if link.job_id is not None else {"folder_id": link.folder_id},
|
||||
)
|
||||
linksmod.revoke(db, user, _own_link(db, user, link_id))
|
||||
db.commit()
|
||||
return {"ok": True, "deleted": link_id}
|
||||
|
||||
@@ -932,20 +742,9 @@ def revoke_link(
|
||||
def admin_list_downloads(
|
||||
admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assets = assets_for(db, jobs)
|
||||
emails = {
|
||||
u.id: u.email
|
||||
for u in db.execute(select(User).where(User.id.in_({j.user_id for j in jobs}))).scalars()
|
||||
}
|
||||
jobs = queries.recent_jobs(db)
|
||||
assets = queries.assets_for(db, jobs)
|
||||
emails = emails_of(db, {j.user_id for j in jobs})
|
||||
out = []
|
||||
for j in jobs:
|
||||
data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None)
|
||||
@@ -957,27 +756,11 @@ def admin_list_downloads(
|
||||
|
||||
@admin_router.get("/storage")
|
||||
def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
# Every file on disk (`quota.ON_DISK`): ready ones, and uploads waiting for or failed in their
|
||||
# conversion — the same disk the footprint and the GC cap count.
|
||||
on_disk = db.execute(
|
||||
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).one()
|
||||
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar()
|
||||
files_on_disk, total_bytes, total_assets = queries.storage_totals(db)
|
||||
# EVERY user, not only those holding a file: a quota is set before anyone downloads.
|
||||
emails: dict[int, str] = {
|
||||
uid: email
|
||||
for uid, email in db.execute(select(User.id, User.email).order_by(User.email)).tuples()
|
||||
}
|
||||
emails = emails_of(db)
|
||||
uids = list(emails)
|
||||
# What each one's Files lists: their finished files.
|
||||
files: dict[int, int] = {
|
||||
uid: n
|
||||
for uid, n in db.execute(
|
||||
select(DownloadJob.user_id, func.count())
|
||||
.where(DownloadJob.status == JobStatus.DONE)
|
||||
.group_by(DownloadJob.user_id)
|
||||
).tuples()
|
||||
}
|
||||
files = queries.done_files_per_user(db)
|
||||
|
||||
def row(uid: int) -> dict:
|
||||
lim = quota.resolve(db, uid)
|
||||
@@ -991,8 +774,8 @@ def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_d
|
||||
}
|
||||
|
||||
return {
|
||||
"files_on_disk": on_disk[0],
|
||||
"total_bytes": int(on_disk[1]),
|
||||
"files_on_disk": files_on_disk,
|
||||
"total_bytes": total_bytes,
|
||||
"total_assets": total_assets,
|
||||
"total_cap_bytes": sysconfig.get_int(db, "download_total_max_bytes"),
|
||||
"per_user": [row(uid) for uid in uids],
|
||||
@@ -1004,10 +787,9 @@ def admin_get_quota(
|
||||
user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
lim = quota.resolve(db, user_id)
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
return {
|
||||
"user_id": user_id,
|
||||
"custom": row is not None,
|
||||
"custom": queries.custom_quota(db, user_id) is not None,
|
||||
"max_bytes": lim.max_bytes,
|
||||
"max_concurrent": lim.max_concurrent,
|
||||
"max_jobs": lim.max_jobs,
|
||||
@@ -1032,25 +814,14 @@ def admin_set_quota(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
get_or_404(db, User, user_id, what="user")
|
||||
cur = quota.resolve(db, user_id)
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
row = DownloadQuota(
|
||||
user_id=user_id,
|
||||
max_bytes=cur.max_bytes,
|
||||
max_concurrent=cur.max_concurrent,
|
||||
max_jobs=cur.max_jobs,
|
||||
unlimited=cur.unlimited,
|
||||
)
|
||||
db.add(row)
|
||||
if body.max_bytes is not None:
|
||||
row.max_bytes = max(0, body.max_bytes)
|
||||
if body.max_concurrent is not None:
|
||||
row.max_concurrent = max(1, body.max_concurrent)
|
||||
if body.max_jobs is not None:
|
||||
row.max_jobs = max(1, body.max_jobs)
|
||||
if body.unlimited is not None:
|
||||
row.unlimited = body.unlimited
|
||||
quota.set_custom(
|
||||
db,
|
||||
user_id,
|
||||
max_bytes=body.max_bytes,
|
||||
max_concurrent=body.max_concurrent,
|
||||
max_jobs=body.max_jobs,
|
||||
unlimited=body.unlimited,
|
||||
)
|
||||
db.commit()
|
||||
return admin_get_quota(user_id, admin, db)
|
||||
|
||||
@@ -1059,8 +830,6 @@ def admin_set_quota(
|
||||
def admin_reset_quota(
|
||||
user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
if quota.reset_custom(db, user_id):
|
||||
db.commit()
|
||||
return admin_get_quota(user_id, admin, db)
|
||||
|
||||
+91
-244
@@ -5,8 +5,10 @@ the per-user layer (name, folder, kept, expiry, shares and links all hang off it
|
||||
bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk.
|
||||
|
||||
In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above
|
||||
the routes are shared with that module — serialization, the ownership locks, the upload spool —
|
||||
because a download and a library file are one row at two moments.
|
||||
the routes are shared with that module — serialization, the ownership checks, the upload spool —
|
||||
because a download and a library file are one row at two moments. What is read comes from
|
||||
`downloads.queries`, what is written goes through Capture's service (`service`, `folders`); this
|
||||
module validates, answers in HTTP and commits.
|
||||
|
||||
A machine with an API token (`app.apitokens`) may use the routes that take `files_caller`: list the
|
||||
folders and files, fetch a file and its hash, upload, delete — each only in a folder its token was
|
||||
@@ -19,51 +21,37 @@ import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, Literal
|
||||
from typing import TYPE_CHECKING, BinaryIO, Literal
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, sysconfig
|
||||
from app.apitokens import FilesCaller, Right, files_caller
|
||||
from app.audit import AuditAction
|
||||
from app.auth import current_user
|
||||
from app.collation import collated
|
||||
from app.config import settings
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.downloads import folders, preview, quota, service, storage
|
||||
from app.downloads import folders, preview, queries, quota, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.status import JobStatus
|
||||
from app.mailtext import lang_for_user
|
||||
from app.models import (
|
||||
DISPLAY_TEXT_MAX,
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
DownloadShare,
|
||||
FileFolder,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
User,
|
||||
display_text,
|
||||
)
|
||||
from app.models import DISPLAY_TEXT_MAX, User, display_text
|
||||
from app.refusal import refused
|
||||
from app.routes._common import capped_stream_to, not_found, owned_or_404
|
||||
from app.utils import iso, like_escape
|
||||
from app.routes._common import capped_stream_to, not_found
|
||||
from app.utils import iso
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||
|
||||
router = APIRouter(prefix="/api/files", tags=["files"])
|
||||
|
||||
# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's
|
||||
# own cap; far beyond any real library).
|
||||
MAX_LIST = 500
|
||||
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
|
||||
_MAX_RETENTION_DAYS = 3650
|
||||
_HASH_CHUNK = 1 << 20
|
||||
@@ -75,7 +63,7 @@ _HASHING = threading.BoundedSemaphore(_HASHERS)
|
||||
# --- shared with routes.downloads -------------------------------------------------------
|
||||
|
||||
|
||||
def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
def serialize(job: "DownloadJob", asset: "MediaAsset | None") -> dict:
|
||||
ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
|
||||
return {
|
||||
"id": job.id,
|
||||
@@ -135,58 +123,18 @@ def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
|
||||
ids = {j.asset_id for j in jobs if j.asset_id}
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
|
||||
return {a.id: a for a in rows}
|
||||
|
||||
|
||||
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
|
||||
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
|
||||
ids = [j.id for j in jobs]
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(
|
||||
select(LibraryState.job_id, LibraryState.position_seconds).where(
|
||||
LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
|
||||
)
|
||||
).all()
|
||||
return {job_id: pos for job_id, pos in rows}
|
||||
|
||||
|
||||
def serialize_job(db: Session, job: DownloadJob) -> dict:
|
||||
def serialize_job(db: Session, job: "DownloadJob") -> dict:
|
||||
"""Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
|
||||
every single-job handler shares."""
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
return serialize(job, asset)
|
||||
return serialize(job, queries.asset_of(db, job))
|
||||
|
||||
|
||||
def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
just released back into a holder; either can free a file another job still holds.
|
||||
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`service.finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
def own_job_for_update(db: Session, user: User, job_id: int) -> "DownloadJob":
|
||||
"""One of the user's jobs under its row lock (`service.lock_own_job`), or a 404."""
|
||||
job = service.lock_own_job(db, user, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
def quota_detail(e: quota.QuotaExceeded) -> dict:
|
||||
@@ -322,30 +270,32 @@ async def spool_upload(
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""A file the caller may read: their own, or one shared with them."""
|
||||
job = db.get(DownloadJob, job_id)
|
||||
def accessible_job(db: Session, user: User, job_id: int) -> "DownloadJob":
|
||||
"""A file the caller may read (`queries.accessible_job`: their own, or shared with them)."""
|
||||
job = queries.accessible_job(db, user, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
if job.user_id == user.id:
|
||||
return job
|
||||
shared = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
|
||||
)
|
||||
).first()
|
||||
if shared is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob:
|
||||
"""A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's
|
||||
own in a folder it may read."""
|
||||
if caller.grants is None:
|
||||
return accessible_job(db, caller.user, job_id)
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None or job.user_id != caller.user.id or not caller.may(job.folder_id, Right.READ):
|
||||
def own_job(db: Session, user: User, job_id: int) -> "DownloadJob":
|
||||
job = queries.own_job(db, user, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
def own_folder(db: Session, user: User, folder_id: int) -> "FileFolder":
|
||||
folder = queries.own_folder(db, user, folder_id)
|
||||
if folder is None:
|
||||
raise not_found("folder")
|
||||
return folder
|
||||
|
||||
|
||||
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> "DownloadJob":
|
||||
"""A file `caller` may read (`queries.readable_job`), or a 404."""
|
||||
job = queries.readable_job(db, caller, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
@@ -353,7 +303,7 @@ def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob:
|
||||
# --- folders -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool = False) -> dict:
|
||||
def _serialize_folder(folder: "FileFolder", count: int, size: int, linked: bool = False) -> dict:
|
||||
return {
|
||||
"id": folder.id,
|
||||
"kind": folder.kind,
|
||||
@@ -369,18 +319,6 @@ def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool =
|
||||
}
|
||||
|
||||
|
||||
def _linked(db: Session, folder_ids: list[int]) -> set[int]:
|
||||
"""Which of these folders a live public link opens (one not past its expiry)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = db.execute(
|
||||
select(DownloadLink.folder_id).where(
|
||||
DownloadLink.folder_id.in_(folder_ids),
|
||||
or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now),
|
||||
)
|
||||
).scalars()
|
||||
return {folder_id for folder_id in rows if folder_id is not None}
|
||||
|
||||
|
||||
@router.get("/folders")
|
||||
def list_folders(
|
||||
caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
||||
@@ -388,23 +326,9 @@ def list_folders(
|
||||
"""The caller's folders with how many files each holds and their size (a token: the ones it may
|
||||
list). A source folder its files have all left stays (count 0): the client decides whether to
|
||||
show it."""
|
||||
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
|
||||
stmt = select(
|
||||
FileFolder,
|
||||
func.count(DownloadJob.id),
|
||||
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
|
||||
).where(FileFolder.user_id == caller.user.id)
|
||||
listable = caller.folders_with(Right.LIST)
|
||||
if listable is not None:
|
||||
stmt = stmt.where(FileFolder.id.in_(listable))
|
||||
rows = db.execute(
|
||||
stmt.outerjoin(DownloadJob, done)
|
||||
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.group_by(FileFolder.id)
|
||||
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
|
||||
).all()
|
||||
linked = _linked(db, [f.id for f, _, _ in rows])
|
||||
return [_serialize_folder(f, count, int(size), f.id in linked) for f, count, size in rows]
|
||||
rows = queries.folders_with_totals(db, caller.user, caller.folders_with(Right.LIST))
|
||||
linked = queries.linked_folders(db, [f.id for f, _, _ in rows])
|
||||
return [_serialize_folder(f, count, size, f.id in linked) for f, count, size in rows]
|
||||
|
||||
|
||||
class FolderIn(BaseModel):
|
||||
@@ -422,13 +346,10 @@ def _folder_name(raw: str) -> str:
|
||||
def create_folder(
|
||||
body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name))
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(folder)
|
||||
db.flush()
|
||||
except IntegrityError as e: # the unique (user, lower(name)): one folder per name
|
||||
raise refused(409, "file_folder_name_taken") from e
|
||||
folder = folders.create_own(db, user.id, _folder_name(body.name))
|
||||
except folders.FolderError as e:
|
||||
raise refused(409, e.reason) from e
|
||||
audit.record(
|
||||
db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id
|
||||
)
|
||||
@@ -451,17 +372,13 @@ def update_folder(
|
||||
) -> dict:
|
||||
"""Rename one of the caller's own folders (a source folder's name is its origin's), and/or set
|
||||
how long any of their folders keeps a file (`folders.set_retention`)."""
|
||||
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||
folder = own_folder(db, user, folder_id)
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
if folder.kind != folders.FolderKind.OWN:
|
||||
raise refused(409, "file_folder_not_own")
|
||||
folder.name = _folder_name(body.name or "")
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.flush()
|
||||
except IntegrityError as e:
|
||||
raise refused(409, "file_folder_name_taken") from e
|
||||
folders.rename(db, folder, _folder_name(body.name or ""))
|
||||
except folders.FolderError as e:
|
||||
raise refused(409, e.reason) from e
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
@@ -486,18 +403,9 @@ def update_folder(
|
||||
after={"retention_days": body.retention_days},
|
||||
)
|
||||
db.commit()
|
||||
count, size = _folder_totals(db, folder.id)
|
||||
return _serialize_folder(folder, count, size, folder.id in _linked(db, [folder.id]))
|
||||
|
||||
|
||||
def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
|
||||
count, size = db.execute(
|
||||
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
|
||||
.select_from(DownloadJob)
|
||||
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
|
||||
).one()
|
||||
return count, int(size)
|
||||
count, size = queries.folder_totals(db, folder.id)
|
||||
linked = folder.id in queries.linked_folders(db, [folder.id])
|
||||
return _serialize_folder(folder, count, size, linked)
|
||||
|
||||
|
||||
@router.delete("/folders/{folder_id}")
|
||||
@@ -505,7 +413,7 @@ def delete_folder(
|
||||
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Delete one of the caller's own folders; its files go back to their source folders."""
|
||||
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||
folder = own_folder(db, user, folder_id)
|
||||
try:
|
||||
moved = folders.delete_own(db, folder)
|
||||
except folders.FolderError as e:
|
||||
@@ -547,13 +455,7 @@ def remove_shared_with_me(
|
||||
"""Remove a file that was shared WITH this user from their 'Shared with me' list. Deletes
|
||||
only the recipient's share grant — the owner's job and the physical file are untouched (this is
|
||||
a per-user dismissal, not a delete)."""
|
||||
row = db.execute(
|
||||
select(DownloadShare).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
if service.unshare(db, job_id, user):
|
||||
db.commit()
|
||||
return {"ok": True, "removed": job_id}
|
||||
|
||||
@@ -582,46 +484,22 @@ def list_files(
|
||||
raise refused(403, "api_token_scope")
|
||||
if folder_id is not None and folder_id not in listable:
|
||||
raise not_found("folder")
|
||||
stmt = (
|
||||
select(DownloadJob)
|
||||
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(DownloadJob.status == JobStatus.DONE)
|
||||
)
|
||||
shared = scope == "shared"
|
||||
if shared:
|
||||
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
|
||||
DownloadShare.to_user_id == user.id
|
||||
)
|
||||
else:
|
||||
stmt = stmt.where(DownloadJob.user_id == user.id)
|
||||
if folder_id is not None:
|
||||
stmt = stmt.where(DownloadJob.folder_id == folder_id)
|
||||
elif listable is not None:
|
||||
stmt = stmt.where(DownloadJob.folder_id.in_(listable))
|
||||
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
|
||||
term = q.strip()
|
||||
if term:
|
||||
like = f"%{like_escape(term)}%"
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
name.ilike(like),
|
||||
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
|
||||
)
|
||||
)
|
||||
# A file is added to its owner's list when it is done, not when it was asked for (a download that
|
||||
# failed for days and then went through is new); one shared with the caller, when it was shared.
|
||||
key = {
|
||||
"added": DownloadShare.created_at if shared else DownloadJob.ready_at,
|
||||
"name": collated(name, lang_for_user(user)),
|
||||
"size": MediaAsset.size_bytes,
|
||||
}[sort]
|
||||
first = key.asc() if order == "asc" else key.desc()
|
||||
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
|
||||
jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all()
|
||||
assets = assets_for(db, jobs)
|
||||
positions = positions_for(db, user, jobs)
|
||||
shared_at = _shared_at(db, user, jobs) if shared else {}
|
||||
sources = source_keys(db, jobs)
|
||||
jobs = queries.library(
|
||||
db,
|
||||
user,
|
||||
folder_id=folder_id,
|
||||
listable=listable,
|
||||
shared=shared,
|
||||
term=q.strip(),
|
||||
sort=sort,
|
||||
order=order,
|
||||
lang=lang_for_user(user),
|
||||
)
|
||||
assets = queries.assets_for(db, jobs)
|
||||
positions = queries.positions_for(db, user, jobs)
|
||||
shared_at = queries.shared_at(db, user, jobs) if shared else {}
|
||||
sources = queries.source_keys(db, jobs)
|
||||
return [
|
||||
{
|
||||
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
|
||||
@@ -634,36 +512,6 @@ def list_files(
|
||||
]
|
||||
|
||||
|
||||
def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
|
||||
"""Where each file came from (its source folder's key), for the list's Source column, the
|
||||
"back to its source folder" move and the download tray's site: a file in a source folder has its folder's key; one moved into
|
||||
an own folder (or someone else's, shared) is traced back by `folders.source_key_for`."""
|
||||
ids = {j.folder_id for j in jobs if j.folder_id is not None}
|
||||
by_id = (
|
||||
{f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
|
||||
if ids
|
||||
else {}
|
||||
)
|
||||
out: dict[int, str] = {}
|
||||
for j in jobs:
|
||||
folder = by_id.get(j.folder_id) if j.folder_id is not None else None
|
||||
if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
|
||||
out[j.id] = folder.source_key
|
||||
else:
|
||||
out[j.id] = folders.source_key_for(db, j)
|
||||
return out
|
||||
|
||||
|
||||
def _shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
|
||||
rows = db.execute(
|
||||
select(DownloadShare.job_id, DownloadShare.created_at).where(
|
||||
DownloadShare.to_user_id == user.id,
|
||||
DownloadShare.job_id.in_([j.id for j in jobs]),
|
||||
)
|
||||
).all()
|
||||
return {job_id: at for job_id, at in rows}
|
||||
|
||||
|
||||
_MAX_EXTRA_LINKS = 8
|
||||
|
||||
|
||||
@@ -711,18 +559,20 @@ def update_file(
|
||||
"""Rename a file and its channel, set its reference links (all display-only: the file on disk
|
||||
keeps its own name), move it to another folder (`folders.move`), or keep it until deleted /
|
||||
hand it back to its folder's retention."""
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
job = own_job(db, user, job_id)
|
||||
sent = body.model_fields_set
|
||||
display: dict[str, object] = {}
|
||||
if "name" in sent:
|
||||
job.display_name = display_text(body.name)
|
||||
display["display_name"] = display_text(body.name)
|
||||
if "uploader" in sent:
|
||||
job.display_uploader = display_text(body.uploader)
|
||||
display["display_uploader"] = display_text(body.uploader)
|
||||
if "uploader_url" in sent:
|
||||
job.display_uploader_url = _clean_url(body.uploader_url)
|
||||
display["display_uploader_url"] = _clean_url(body.uploader_url)
|
||||
if "extra_links" in sent:
|
||||
raw = body.extra_links or []
|
||||
cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
|
||||
job.extra_links = cleaned or None
|
||||
display["extra_links"] = cleaned or None
|
||||
service.set_display(job, **display)
|
||||
if "folder_id" in sent:
|
||||
if body.folder_id is None:
|
||||
raise refused(400, "file_folder_gone")
|
||||
@@ -768,7 +618,8 @@ def delete_file(
|
||||
target_id=job_id,
|
||||
summary=caller.via,
|
||||
)
|
||||
service.delete(db, job) # commits, the audit row with it
|
||||
service.delete(db, job)
|
||||
db.commit() # the audit row with it
|
||||
return {"ok": True, "deleted": job_id}
|
||||
|
||||
|
||||
@@ -787,7 +638,7 @@ def upload_file(
|
||||
return serialize_job(db, ingest_spooled(db, upload))
|
||||
|
||||
|
||||
def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
|
||||
def ingest_spooled(db: Session, upload: SpooledUpload) -> "DownloadJob":
|
||||
"""The spooled upload into the library (`upload.ingest_upload`, which commits its own rows),
|
||||
its refusals answered as HTTP."""
|
||||
if upload.size == 0:
|
||||
@@ -811,8 +662,8 @@ def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
|
||||
return job
|
||||
|
||||
|
||||
def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
def _ready_file(db: Session, job: "DownloadJob") -> tuple["MediaAsset", Path]:
|
||||
asset = queries.asset_of(db, job)
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise refused(409, "download_not_ready")
|
||||
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||
@@ -821,7 +672,7 @@ def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
|
||||
return asset, path
|
||||
|
||||
|
||||
def serve_file(db: Session, job: DownloadJob) -> FileResponse:
|
||||
def serve_file(db: Session, job: "DownloadJob") -> FileResponse:
|
||||
"""The file of `job` (the caller may read it), range-aware, named by the user's display name.
|
||||
|
||||
The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
|
||||
@@ -839,7 +690,7 @@ def serve_file(db: Session, job: DownloadJob) -> FileResponse:
|
||||
if asset.sha256:
|
||||
digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode()
|
||||
headers["Repr-Digest"] = f"sha-256=:{digest}:"
|
||||
asset.last_access_at = datetime.now(timezone.utc) # the handler commits it
|
||||
service.touch(asset) # the handler commits it
|
||||
media_type = storage.served_media_type(asset.media_kind, asset.container)
|
||||
if media_type is None:
|
||||
return FileResponse(
|
||||
@@ -894,13 +745,13 @@ def preview_file(
|
||||
text = preview.text_utf8(head)
|
||||
if text is None:
|
||||
raise refused(415, "file_no_preview")
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
service.touch(asset)
|
||||
db.commit()
|
||||
return Response(text, media_type=preview.media_type(kind, asset.container), headers=headers)
|
||||
filename = storage.download_filename(
|
||||
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||
)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
service.touch(asset)
|
||||
db.commit()
|
||||
return FileResponse(
|
||||
path,
|
||||
@@ -936,10 +787,6 @@ def get_sha256(
|
||||
finally:
|
||||
_HASHING.release()
|
||||
digest = h.hexdigest()
|
||||
db.execute(
|
||||
update(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
|
||||
.values(sha256=digest)
|
||||
)
|
||||
service.store_sha256(db, asset_id, digest)
|
||||
db.commit()
|
||||
return {"sha256": digest}
|
||||
|
||||
@@ -6,23 +6,26 @@ the app or any account data. Password-protected links exchange the password once
|
||||
a short-lived signed grant that the media URL carries (a `<video src>` can't send a header).
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.downloads import queries, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset, User
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import verify_password
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
|
||||
router = APIRouter(prefix="/api/public", tags=["public"])
|
||||
|
||||
# Slow password guessing on a link (the token itself is 192-bit, so it needs no throttle). Counted per
|
||||
@@ -37,26 +40,25 @@ _unlock_limit = RateLimiter(max_events=10, window_seconds=300)
|
||||
_unlock_failures = RateLimiter(max_events=50, window_seconds=300)
|
||||
|
||||
|
||||
def _link_or_404(db: Session, token: str) -> DownloadLink:
|
||||
def _link_or_404(db: Session, token: str) -> "DownloadLink":
|
||||
link = linksmod.find(db, token)
|
||||
if link is None or linksmod.is_expired(link):
|
||||
raise refused(404, "link_gone")
|
||||
return link
|
||||
|
||||
|
||||
def _being_made(db: Session, job: DownloadJob) -> bool:
|
||||
def _being_made(db: Session, job: "DownloadJob") -> bool:
|
||||
"""The job is running, or queued for a worker that will take it: the worker leaves a suspended
|
||||
owner's queued jobs where they are (`worker._claim_job`), as long as the suspension lasts."""
|
||||
if job.status == JobStatus.RUNNING:
|
||||
return True
|
||||
if job.status != JobStatus.QUEUED:
|
||||
return False
|
||||
owner = db.get(User, job.user_id)
|
||||
return owner is not None and not owner.is_suspended
|
||||
return not queries.owner_suspended(db, job)
|
||||
|
||||
|
||||
def _ready_job(db: Session, job: DownloadJob | None) -> tuple[DownloadJob, MediaAsset]:
|
||||
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
||||
def _ready_job(db: Session, job: "DownloadJob | None") -> tuple["DownloadJob", "MediaAsset"]:
|
||||
asset = queries.asset_of(db, job)
|
||||
if (
|
||||
job is not None
|
||||
and asset is not None
|
||||
@@ -78,25 +80,24 @@ def _ready_job(db: Session, job: DownloadJob | None) -> tuple[DownloadJob, Media
|
||||
return job, asset
|
||||
|
||||
|
||||
def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAsset]:
|
||||
def _ready_target(db: Session, link: "DownloadLink") -> tuple["DownloadJob", "MediaAsset"]:
|
||||
"""A file link's file. A folder link names no one file: its files are asked for by id."""
|
||||
if link.job_id is None:
|
||||
raise refused(404, "video_gone")
|
||||
return _ready_job(db, db.get(DownloadJob, link.job_id))
|
||||
return _ready_job(db, queries.job(db, link.job_id))
|
||||
|
||||
|
||||
def _link_folder(db: Session, link: DownloadLink) -> FileFolder:
|
||||
folder = db.get(FileFolder, link.folder_id) if link.folder_id is not None else None
|
||||
def _link_folder(db: Session, link: "DownloadLink") -> "FileFolder":
|
||||
folder = queries.link_folder(db, link)
|
||||
if folder is None:
|
||||
raise refused(404, "link_gone")
|
||||
return folder
|
||||
|
||||
|
||||
def _folder_job(db: Session, link: DownloadLink, job_id: int) -> tuple[DownloadJob, MediaAsset]:
|
||||
folder = _link_folder(db, link)
|
||||
job = db.execute(
|
||||
select(DownloadJob).where(DownloadJob.id == job_id, *linksmod.in_folder(folder))
|
||||
).scalar_one_or_none()
|
||||
def _folder_job(
|
||||
db: Session, link: "DownloadLink", job_id: int
|
||||
) -> tuple["DownloadJob", "MediaAsset"]:
|
||||
job = queries.folder_job(db, _link_folder(db, link), job_id)
|
||||
if job is None:
|
||||
raise refused(404, "video_gone")
|
||||
return _ready_job(db, job)
|
||||
@@ -113,7 +114,7 @@ def _file_url(token: str, grant: str | None = None, job_id: int | None = None) -
|
||||
|
||||
|
||||
def _poster_url(
|
||||
token: str, asset: MediaAsset, grant: str | None = None, job_id: int | None = None
|
||||
token: str, asset: "MediaAsset", grant: str | None = None, job_id: int | None = None
|
||||
) -> str | None:
|
||||
if not asset.poster_path:
|
||||
return None
|
||||
@@ -121,7 +122,7 @@ def _poster_url(
|
||||
return f"{url}?g={grant}" if grant else url
|
||||
|
||||
|
||||
def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
|
||||
def _folder_meta(db: Session, link: "DownloadLink", token: str, grant: str | None) -> dict:
|
||||
"""A folder link's page: the folder's name and each file's own watch metadata
|
||||
(`linksmod.folder_files`: ready, on disk, newest first)."""
|
||||
folder = _link_folder(db, link)
|
||||
@@ -147,7 +148,7 @@ def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None)
|
||||
}
|
||||
|
||||
|
||||
def _file_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
|
||||
def _file_meta(db: Session, link: "DownloadLink", token: str, grant: str | None) -> dict:
|
||||
job, asset = _ready_target(db, link)
|
||||
return {
|
||||
"needs_password": False,
|
||||
@@ -158,20 +159,14 @@ def _file_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -
|
||||
}
|
||||
|
||||
|
||||
def _opened(db: Session, link: DownloadLink, token: str, grant: str | None = None) -> dict:
|
||||
def _opened(db: Session, link: "DownloadLink", token: str, grant: str | None = None) -> dict:
|
||||
"""The page an open link shows, counted as one view (the handler commits the count)."""
|
||||
meta = (
|
||||
_folder_meta(db, link, token, grant)
|
||||
if link.folder_id is not None
|
||||
else _file_meta(db, link, token, grant)
|
||||
)
|
||||
# In SQL, not `+= 1` on the loaded row: two opens at once would both write the same count.
|
||||
db.execute(
|
||||
update(DownloadLink)
|
||||
.where(DownloadLink.id == link.id)
|
||||
.values(view_count=DownloadLink.view_count + 1)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
linksmod.count_view(db, link)
|
||||
return meta
|
||||
|
||||
|
||||
@@ -217,13 +212,13 @@ def watch_unlock(
|
||||
return {"ok": True, **meta}
|
||||
|
||||
|
||||
def _granted(link: DownloadLink, token: str, g: str | None) -> None:
|
||||
def _granted(link: "DownloadLink", token: str, g: str | None) -> None:
|
||||
"""A password link's files want the grant its unlock handed out."""
|
||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
||||
raise refused(403, "share_password_required")
|
||||
|
||||
|
||||
def _serve(link: DownloadLink, asset: MediaAsset) -> FileResponse:
|
||||
def _serve(link: "DownloadLink", asset: "MediaAsset") -> FileResponse:
|
||||
"""Range-aware stream of a shared file. Inline by default (plays in the page); an
|
||||
`allow_download` link serves it as an attachment."""
|
||||
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||
@@ -252,7 +247,7 @@ def _serve(link: DownloadLink, asset: MediaAsset) -> FileResponse:
|
||||
)
|
||||
|
||||
|
||||
def _poster(asset: MediaAsset) -> FileResponse:
|
||||
def _poster(asset: "MediaAsset") -> FileResponse:
|
||||
if not asset.poster_path:
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||
|
||||
@@ -21,8 +21,8 @@ from app.apitokens import GRANT_COLUMNS, Right
|
||||
from app.audit import AuditAction
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.downloads import folders
|
||||
from app.models import ApiToken, ApiTokenGrant, FileFolder, User
|
||||
from app.downloads import folders, queries
|
||||
from app.models import ApiToken, ApiTokenGrant, User
|
||||
from app.refusal import refused
|
||||
from app.routes._common import owned_or_404
|
||||
from app.security import hash_token
|
||||
@@ -50,12 +50,16 @@ class TokenIn(BaseModel):
|
||||
|
||||
|
||||
def _serialize(db: Session, token: ApiToken) -> dict:
|
||||
rows = db.execute(
|
||||
select(ApiTokenGrant, FileFolder)
|
||||
.join(FileFolder, FileFolder.id == ApiTokenGrant.folder_id)
|
||||
.where(ApiTokenGrant.token_id == token.id)
|
||||
.order_by(FileFolder.id)
|
||||
).all()
|
||||
grants = list(
|
||||
db.execute(
|
||||
select(ApiTokenGrant)
|
||||
.where(ApiTokenGrant.token_id == token.id)
|
||||
.order_by(ApiTokenGrant.folder_id)
|
||||
).scalars()
|
||||
)
|
||||
# The folders are Capture's: read through its queries, not its table.
|
||||
by_id = queries.folders_by_id(db, {g.folder_id for g in grants})
|
||||
rows = [(g, by_id[g.folder_id]) for g in grants if g.folder_id in by_id]
|
||||
now = datetime.now(timezone.utc)
|
||||
return {
|
||||
"id": token.id,
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Every table, registered on `Base.metadata`.
|
||||
|
||||
Each unit declares its tables in its own `models` module, so importing `app.models` alone no longer
|
||||
names the whole schema. Whatever needs all of it — the migrations, the test database, a check over
|
||||
every mapper — imports this module.
|
||||
"""
|
||||
|
||||
import app.downloads.models # Capture's tables
|
||||
import app.models # noqa: F401 — the core tables
|
||||
@@ -1,13 +1,54 @@
|
||||
"""Shared "who is an active, non-suspended account" predicate.
|
||||
"""Whose a row is, and who an account is to the others — one place for both.
|
||||
|
||||
Used wherever a user must be filtered out of a recipient list or rejected as a target: the download
|
||||
share-by-email and recipient-picker endpoints. Kept in one place so the two forms (SQL clauses for a
|
||||
query, Python check for a loaded row) can't drift apart, and so a route module doesn't have to import
|
||||
another route module just to reuse the rule.
|
||||
**Ownership.** A user-owned table says whose a row is in one column (`user_id`, or `to_user_id` for
|
||||
something handed to them). A unit's reads and writes name that column through `owned_by` and fetch a
|
||||
row by id through `owned_row`, never with an inline `Model.user_id == …`: the rule "a row owned by
|
||||
someone else is answered like a missing one" then lives here, and a contract test
|
||||
(`test_unit_fences.py`) holds a switched-on unit to it.
|
||||
|
||||
**Recipients.** "An active, non-suspended account" — the download share-by-email and the recipient
|
||||
picker filter on it. Kept here so the two forms (SQL clauses for a query, a Python check for a loaded
|
||||
row) can't drift apart.
|
||||
"""
|
||||
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from sqlalchemy import ColumnElement, func, select
|
||||
from sqlalchemy.orm import InstrumentedAttribute, Session
|
||||
|
||||
from app.collation import collated
|
||||
from app.db import Base
|
||||
from app.models import User
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
|
||||
|
||||
def _owner_id(owner: User | int) -> int:
|
||||
return owner if isinstance(owner, int) else owner.id
|
||||
|
||||
|
||||
def owned_by(column: InstrumentedAttribute, owner: User | int) -> ColumnElement[bool]:
|
||||
"""`column` — a table's owner column — names `owner`."""
|
||||
return column == _owner_id(owner)
|
||||
|
||||
|
||||
def owned_row(
|
||||
db: Session,
|
||||
model: type[M],
|
||||
id_: Any,
|
||||
owner: User | int,
|
||||
*,
|
||||
owner_attr: str = "user_id",
|
||||
lock: bool = False,
|
||||
) -> M | None:
|
||||
"""The row of `model` with this primary key when `owner` owns it, else None — a row of someone
|
||||
else's is the same answer as a missing one, so a caller can never tell the two apart. `lock`
|
||||
takes it FOR UPDATE (past the identity map)."""
|
||||
row = db.get(model, id_, with_for_update=True if lock else None)
|
||||
if row is None or getattr(row, owner_attr) != _owner_id(owner):
|
||||
return None
|
||||
return row
|
||||
|
||||
|
||||
def messageable_clauses() -> list:
|
||||
"""SQL clauses for "an active, non-suspended account" — splat into a recipient query."""
|
||||
@@ -20,3 +61,30 @@ def messageable_clauses() -> list:
|
||||
def is_messageable_user(u: User | None) -> bool:
|
||||
"""Python mirror of `messageable_clauses()` for an already-loaded row (a share recipient)."""
|
||||
return bool(u and u.is_active and not u.is_suspended)
|
||||
|
||||
|
||||
def user_by_email(db: Session, email: str) -> User | None:
|
||||
"""The account with this address, matched without regard to case."""
|
||||
return db.execute(
|
||||
select(User).where(func.lower(User.email) == email.strip().lower())
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def messageable_others(db: Session, user: User, lang: str) -> list[User]:
|
||||
"""Every account `user` may hand something to: active, not suspended, not themselves; in the
|
||||
reader's alphabet by address."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(User)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(collated(User.email, lang))
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def emails_of(db: Session, ids: set[int] | None = None) -> dict[int, str]:
|
||||
"""Each account's address by id, in address order: these ids, or every account."""
|
||||
stmt = select(User.id, User.email).order_by(User.email)
|
||||
if ids is not None:
|
||||
stmt = stmt.where(User.id.in_(ids))
|
||||
return dict(db.execute(stmt).tuples().all())
|
||||
|
||||
@@ -46,7 +46,8 @@ from app.downloads.jobdb import (
|
||||
_staging_dir,
|
||||
)
|
||||
from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_stop = threading.Event()
|
||||
|
||||
@@ -25,7 +25,7 @@ from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine, event, text
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
import app.models # register every model on Base.metadata
|
||||
import app.tables # every table on Base.metadata
|
||||
import app.state as app_state
|
||||
from app import apitokens, security
|
||||
from app.auth import current_user
|
||||
|
||||
@@ -25,16 +25,8 @@ from app.config import settings
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.main import app
|
||||
from app.notifications import create_notification
|
||||
from app.models import (
|
||||
ApiToken,
|
||||
ApiTokenGrant,
|
||||
AuditLog,
|
||||
DownloadJob,
|
||||
FileFolder,
|
||||
MediaAsset,
|
||||
Notification,
|
||||
User,
|
||||
)
|
||||
from app.models import ApiToken, ApiTokenGrant, AuditLog, Notification, User
|
||||
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes import tokens as tokens_routes
|
||||
from app.routes.auth import bump_session_epoch
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
"""Capture's service and queries, one use case each, below the HTTP layer.
|
||||
|
||||
The routes are thin callers now, and their own tests stay the contract a client sees. These hold
|
||||
what each write does and leaves on its own: that it flushes and leaves the commit to its caller,
|
||||
that a row of someone else's reads as a missing one, and the rule each write carries (a hash kept
|
||||
once, a password change that retires the old grants, a quota floored). DB-backed (real Postgres).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.audit import AuditAction
|
||||
from app.downloads import folders, links, queries, quota, service
|
||||
from app.downloads.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
DownloadQuota,
|
||||
DownloadShare,
|
||||
FileFolder,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
)
|
||||
from app.models import AuditLog, Notification
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
|
||||
def _file(db, user, *, sha256=None) -> DownloadJob:
|
||||
asset = MediaAsset(
|
||||
source_kind="url",
|
||||
source_ref=f"https://example.com/{user.id}.mp4",
|
||||
format_sig=f"sig-{user.id}",
|
||||
status="ready",
|
||||
rel_path=f"f-{user.id}.mp4",
|
||||
ref_count=1,
|
||||
media_kind="video",
|
||||
container="mp4",
|
||||
sha256=sha256,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
job = DownloadJob(
|
||||
user_id=user.id,
|
||||
asset_id=asset.id,
|
||||
source_kind="url",
|
||||
source_ref=asset.source_ref,
|
||||
status="done",
|
||||
profile_snapshot={},
|
||||
display_name="Clip",
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
return job
|
||||
|
||||
|
||||
def _uncommitted(db):
|
||||
"""What the caller still has to commit: the session holds pending work."""
|
||||
return bool(db.new or db.dirty or db.deleted) or db.in_transaction()
|
||||
|
||||
|
||||
# --- whose a row is ------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_row_of_someone_elses_reads_as_a_missing_one(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert owned_row(db, DownloadJob, job.id, owner) is job
|
||||
assert owned_row(db, DownloadJob, job.id, stranger) is None
|
||||
assert owned_row(db, DownloadJob, job.id + 1000, owner) is None
|
||||
mine = db.execute(select(DownloadJob.id).where(owned_by(DownloadJob.user_id, owner.id)))
|
||||
assert mine.scalars().all() == [job.id]
|
||||
|
||||
|
||||
def test_a_file_is_readable_by_its_owner_and_whoever_it_was_shared_with(db, make_user):
|
||||
owner, friend, stranger = make_user(), make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.share(db, job, owner, friend) is True
|
||||
db.commit()
|
||||
assert queries.accessible_job(db, owner, job.id) is job
|
||||
assert queries.accessible_job(db, friend, job.id) is job
|
||||
assert queries.accessible_job(db, stranger, job.id) is None
|
||||
assert queries.own_job(db, friend, job.id) is None # shared is not owned
|
||||
|
||||
|
||||
def test_the_tray_and_the_library_list_only_the_callers_own_files(db, make_user):
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
mine, theirs = make_user(), make_user()
|
||||
own, other = _file(db, mine), _file(db, theirs)
|
||||
since = datetime.now(timezone.utc) - timedelta(days=1)
|
||||
for job in (own, other):
|
||||
job.ready_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
assert [j.id for j in queries.queue(db, mine, since)] == [own.id]
|
||||
library = queries.library(
|
||||
db,
|
||||
mine,
|
||||
folder_id=None,
|
||||
listable=None,
|
||||
shared=False,
|
||||
term="",
|
||||
sort="added",
|
||||
order="desc",
|
||||
lang="en",
|
||||
)
|
||||
assert [j.id for j in library] == [own.id]
|
||||
assert [r for r, _ in queries.statuses_by_source(db, mine, ["done"])] == [own.source_ref]
|
||||
|
||||
|
||||
# --- locks -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_job_lock_answers_none_for_a_job_not_the_callers(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.lock_own_job(db, stranger, job.id) is None
|
||||
assert service.lock_own_job(db, owner, job.id) is job
|
||||
assert service.hold_for_share(db, stranger, job.id) is None
|
||||
assert service.hold_for_share(db, owner, job.id) is job
|
||||
|
||||
|
||||
# --- profiles --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_profile_is_made_after_the_users_others_and_only_theirs_to_change(db, make_user):
|
||||
user, other = make_user(), make_user()
|
||||
first = service.create_profile(db, user, "x" * 100, {"kind": "video"})
|
||||
second = service.create_profile(db, user, "Audio", {"kind": "audio"})
|
||||
assert _uncommitted(db)
|
||||
db.commit()
|
||||
assert first.name == "x" * 80 # names are capped
|
||||
assert second.sort_order == first.sort_order + 1
|
||||
assert queries.own_profile(db, other, first.id) is None
|
||||
service.update_profile(first, name="Video", spec=None)
|
||||
db.commit()
|
||||
assert (first.name, first.spec) == ("Video", {"kind": "video"})
|
||||
service.delete_profile(db, first)
|
||||
db.commit()
|
||||
assert [p.id for p in queries.profiles_for(db, user)] == [second.id]
|
||||
|
||||
|
||||
# --- resume points ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_resume_point_is_one_row_per_user_and_file_and_a_zero_clears_it(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
service.set_position(db, user, job.id, 40)
|
||||
service.set_position(db, user, job.id, 90)
|
||||
db.commit()
|
||||
rows = db.execute(select(LibraryState.position_seconds)).scalars().all()
|
||||
assert rows == [90]
|
||||
assert queries.positions_for(db, user, [job]) == {job.id: 90}
|
||||
service.set_position(db, user, job.id, 0)
|
||||
service.set_position(db, user, job.id, 0) # the beacon's second clear deletes nothing
|
||||
db.commit()
|
||||
assert db.execute(select(LibraryState)).first() is None
|
||||
|
||||
|
||||
# --- shares ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_share_is_made_once_with_one_notice_and_undone_by_either_side(db, make_user):
|
||||
owner, friend = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.share(db, job, owner, friend) is True
|
||||
assert service.share(db, job, owner, friend) is False
|
||||
assert _uncommitted(db)
|
||||
db.commit()
|
||||
notices = db.execute(select(Notification).where(Notification.user_id == friend.id)).scalars()
|
||||
assert [n.data["job_id"] for n in notices] == [job.id]
|
||||
assert service.unshare(db, job.id, friend) is True
|
||||
assert service.unshare(db, job.id, friend) is False
|
||||
db.commit()
|
||||
assert db.execute(select(DownloadShare)).first() is None
|
||||
|
||||
|
||||
# --- a file's display fields, its access clock, its hash ------------------------------------
|
||||
|
||||
|
||||
def test_only_display_fields_can_be_set_on_a_file(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
service.set_display(job, display_name="New", extra_links=None)
|
||||
assert job.display_name == "New"
|
||||
with pytest.raises(ValueError):
|
||||
service.set_display(job, status="error")
|
||||
assert job.status == "done"
|
||||
|
||||
|
||||
def test_a_hash_is_kept_once_and_never_overwritten(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
service.store_sha256(db, job.asset_id, "a" * 64)
|
||||
service.store_sha256(db, job.asset_id, "b" * 64)
|
||||
db.commit()
|
||||
assert db.get(MediaAsset, job.asset_id).sha256 == "a" * 64
|
||||
|
||||
|
||||
def test_serving_a_file_restarts_its_idle_clock(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
asset = queries.asset_of(db, job)
|
||||
before = asset.last_access_at
|
||||
service.touch(asset)
|
||||
assert asset.last_access_at is not None and asset.last_access_at != before
|
||||
|
||||
|
||||
# --- links -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _actions(db) -> list[str]:
|
||||
return db.execute(select(AuditLog.action).order_by(AuditLog.id)).scalars().all()
|
||||
|
||||
|
||||
def test_a_link_is_made_changed_and_revoked_with_an_audit_row_each(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
link, token = links.create(
|
||||
db, user, password="pw", allow_download=False, expires_days=3, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
assert links.find(db, token) is link
|
||||
assert link.expires_at is not None and link.password_hash is not None
|
||||
version = link.password_version
|
||||
links.update(db, user, link, password="other", sent=frozenset({"password"}))
|
||||
db.commit()
|
||||
assert link.password_version == version + 1 # the old grants stop working
|
||||
links.update(db, user, link, allow_download=True, sent=frozenset()) # nothing was sent
|
||||
assert link.allow_download is False
|
||||
assert [link.id for link in queries.links_to(db, job_id=job.id)] == [link.id]
|
||||
links.revoke(db, user, link)
|
||||
db.commit()
|
||||
assert db.execute(select(DownloadLink)).first() is None
|
||||
assert _actions(db) == [
|
||||
AuditAction.DOWNLOAD_LINK_CREATE,
|
||||
AuditAction.DOWNLOAD_LINK_UPDATE,
|
||||
AuditAction.DOWNLOAD_LINK_REVOKE,
|
||||
]
|
||||
|
||||
|
||||
def test_a_link_is_the_callers_only_through_a_file_or_folder_of_theirs(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
link, _ = links.create(
|
||||
db, owner, password="", allow_download=True, expires_days=None, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
assert queries.own_link(db, owner, link.id) is link
|
||||
assert queries.own_link(db, stranger, link.id) is None
|
||||
|
||||
|
||||
def test_a_view_is_counted_in_sql(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
link, _ = links.create(
|
||||
db, user, password="", allow_download=True, expires_days=None, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
links.count_view(db, link)
|
||||
links.count_view(db, link)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
assert db.get(DownloadLink, link.id).view_count == 2
|
||||
|
||||
|
||||
# --- folders ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_folder_name_is_one_per_user_and_a_source_folder_keeps_its_own(db, make_user):
|
||||
user, other = make_user(), make_user()
|
||||
mine = folders.create_own(db, user.id, "Trips")
|
||||
with pytest.raises(folders.FolderError, match="file_folder_name_taken"):
|
||||
folders.create_own(db, user.id, "trips")
|
||||
folders.create_own(db, other.id, "Trips") # another user's namesake is theirs
|
||||
db.commit()
|
||||
source = folders.source_folder(db, user.id, folders.SourceKey.UPLOAD)
|
||||
with pytest.raises(folders.FolderError, match="file_folder_not_own"):
|
||||
folders.rename(db, source, "Mine now")
|
||||
folders.rename(db, mine, "Holidays")
|
||||
db.commit()
|
||||
assert db.get(FileFolder, mine.id).name == "Holidays"
|
||||
assert queries.own_folder(db, other, mine.id) is None
|
||||
|
||||
|
||||
# --- quota -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_custom_quota_starts_from_what_applies_and_floors_each_value(db, make_user):
|
||||
user = make_user()
|
||||
defaults = quota.resolve(db, user.id)
|
||||
quota.set_custom(db, user.id, max_concurrent=0, max_bytes=-5)
|
||||
db.commit()
|
||||
row = queries.custom_quota(db, user.id)
|
||||
assert (row.max_concurrent, row.max_bytes, row.max_jobs) == (1, 0, defaults.max_jobs)
|
||||
assert quota.reset_custom(db, user.id) is True
|
||||
assert quota.reset_custom(db, user.id) is False
|
||||
db.commit()
|
||||
assert db.get(DownloadQuota, user.id) is None
|
||||
@@ -18,7 +18,8 @@ from app.downloads import errors as dlerrors
|
||||
from app.downloads import formats, pipeline, quota, runner, service
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads.edit import INPUT_GUARD
|
||||
from app.models import Channel, DownloadJob, MediaAsset, PlexItem, PlexLibrary, PlexShow, Video
|
||||
from app.models import Channel, PlexItem, PlexLibrary, PlexShow, Video
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.plex import paths as plex_paths
|
||||
|
||||
# --- the spec -----------------------------------------------------------------------------------
|
||||
|
||||
@@ -18,7 +18,8 @@ from cryptography.fernet import Fernet
|
||||
|
||||
from app import security
|
||||
from app.downloads import credentials, formats, service
|
||||
from app.models import DownloadCredential, DownloadJob, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadCredential, DownloadJob, MediaAsset
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -24,7 +24,8 @@ from app import security
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.downloads import credentials, formats, pipeline, service
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.worker import _process_job
|
||||
|
||||
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
|
||||
|
||||
@@ -18,7 +18,8 @@ from sqlalchemy import select
|
||||
from app.apitokens import FilesCaller
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import service
|
||||
from app.models import DownloadJob, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.routes.downloads import cancel_download, pause_download
|
||||
from app.routes.files import delete_file, own_job_for_update
|
||||
|
||||
@@ -88,6 +89,7 @@ def test_overlapping_requests_for_one_job_release_its_hold_once(
|
||||
# The second request waits for the first one's locks instead of deciding on the same row.
|
||||
assert not finished.wait(0.5), "the second request did not wait for the first"
|
||||
transition(first, locked)
|
||||
first.commit() # the handler's commit: the service only flushes
|
||||
other.join(10)
|
||||
assert finished.is_set()
|
||||
finally:
|
||||
|
||||
@@ -13,7 +13,8 @@ its OWN session, so every test COMMITS its setup before calling it.
|
||||
from app.config import settings
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import formats, pipeline, quota
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.worker import _claim_job
|
||||
|
||||
|
||||
@@ -179,6 +180,7 @@ def test_a_suspended_users_queued_jobs_wait(db):
|
||||
|
||||
|
||||
def test_list_downloads_is_capped(db, monkeypatch):
|
||||
from app.downloads import queries
|
||||
from app.routes import downloads as dl
|
||||
|
||||
u = User(email="heavy@example.com")
|
||||
@@ -188,7 +190,7 @@ def test_list_downloads_is_capped(db, monkeypatch):
|
||||
_job(db, u.id, queue_pos=i, status="done")
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(dl, "MAX_LIST", 2)
|
||||
monkeypatch.setattr(queries, "MAX_LIST", 2)
|
||||
rows = dl.list_downloads(user=u, db=db)
|
||||
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
|
||||
|
||||
|
||||
@@ -13,7 +13,8 @@ DB lane (`db` fixture) — skipped without Postgres.
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from app.downloads import service
|
||||
from app.models import DownloadJob, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def _user(db, email="ref@example.com"):
|
||||
|
||||
@@ -4,7 +4,8 @@ you") — the only place the title lived, so a Hungarian reader got the English.
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import DownloadJob, MediaAsset, Notification
|
||||
from app.models import Notification
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user):
|
||||
|
||||
@@ -8,7 +8,8 @@ behaviour-equivalently. DB lane (`db` fixture) — skipped without Postgres.
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.models import DownloadJob, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None):
|
||||
|
||||
@@ -11,7 +11,8 @@ from sqlalchemy import text
|
||||
|
||||
from app.downloads import service
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob
|
||||
|
||||
|
||||
def _running_job(db) -> DownloadJob:
|
||||
|
||||
@@ -13,7 +13,7 @@ from app.config import settings
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import pipeline, quota, service
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
|
||||
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ import yt_dlp
|
||||
from app import logs, sysconfig, worker
|
||||
from app.downloads import pipeline
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.plex import stream as plex_stream
|
||||
|
||||
|
||||
|
||||
@@ -17,7 +17,8 @@ from app.downloads import folders, formats, service
|
||||
from app.downloads import gc as gcmod
|
||||
from app.db import SessionLocal
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, FileFolder, MediaAsset, Notification
|
||||
from app.models import Notification
|
||||
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@ from app.config import settings
|
||||
from app.downloads import folders, service
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset
|
||||
from app.models import AuditLog
|
||||
from app.downloads.models import DownloadJob, DownloadShare, FileFolder, MediaAsset
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ from sqlalchemy import select
|
||||
from app.config import settings
|
||||
from app.downloads import folders, preview, service
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadShare, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadShare, MediaAsset
|
||||
from app.routes.files import TRUNCATED_HEADER
|
||||
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@ from app.config import settings
|
||||
from app.db import engine
|
||||
from app.downloads import folders, og
|
||||
from app.downloads import probe as probemod
|
||||
from app.models import AuditLog, DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
from app.models import AuditLog
|
||||
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
|
||||
_VIDEO = {
|
||||
"streams": [
|
||||
@@ -193,7 +194,7 @@ def test_every_open_counts_as_a_view(shelf, client):
|
||||
|
||||
def test_the_folder_list_is_capped_like_the_owners_files_list():
|
||||
from app.downloads import links as linksmod
|
||||
from app.routes.files import MAX_LIST
|
||||
from app.downloads.queries import MAX_LIST
|
||||
|
||||
assert linksmod.FOLDER_MAX == MAX_LIST
|
||||
|
||||
@@ -317,7 +318,7 @@ def test_a_password_file_link_says_it_opens_a_file(shelf, client):
|
||||
def test_a_file_whose_bytes_are_gone_is_not_offered(shelf, client, db):
|
||||
from pathlib import Path
|
||||
|
||||
from app.models import MediaAsset
|
||||
from app.downloads.models import MediaAsset
|
||||
|
||||
token = _link(shelf)["token"]
|
||||
gone = shelf["inside"][0]
|
||||
|
||||
@@ -9,7 +9,7 @@ cannot reach, is ever read or written. DB-backed (real Postgres).
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import DownloadJob, DownloadShare, LibraryState, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadShare, LibraryState, MediaAsset
|
||||
from app.routes._common import ProgressIn, kept_position
|
||||
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@ from app import security
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.downloads import probe as probemod
|
||||
from app.models import AuditLog, DownloadLink
|
||||
from app.models import AuditLog
|
||||
from app.downloads.models import DownloadLink
|
||||
|
||||
_VIDEO = {
|
||||
"streams": [
|
||||
|
||||
@@ -7,7 +7,7 @@ the shared one, so every other owner check kept the English. Needs the DB.
|
||||
"""
|
||||
|
||||
from app.auth import require_write_scope
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user):
|
||||
|
||||
@@ -17,7 +17,7 @@ import pytest
|
||||
from app.config import settings
|
||||
from app.downloads import pipeline, playable, probe, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import pipeline, playable, probe
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
APP = Path(__file__).resolve().parents[1] / "app"
|
||||
|
||||
|
||||
@@ -11,13 +11,18 @@ until its entry is lowered (the fence only tightens, as `baseline.json` does for
|
||||
empty `ALLOWED` is the end state — a file listed there is a debt, not an exemption.
|
||||
|
||||
A core route (identity, admin, ops) is not a unit's HTTP layer and may use the session; it may
|
||||
still not reach into a unit's tables.
|
||||
still not reach into a unit's tables. A route may NAME a unit's model as a type (under
|
||||
`TYPE_CHECKING`), never use one as a value.
|
||||
|
||||
Below the HTTP layer, a fenced unit's own code keeps to its edges (`FENCED`, `READS`): see the
|
||||
second half of this file.
|
||||
"""
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
from app.models import Base
|
||||
import app.tables # noqa: F401 — every unit's tables, for the ownership check
|
||||
from app.db import Base
|
||||
|
||||
APP = Path(__file__).parent.parent / "app"
|
||||
ROUTES = APP / "routes"
|
||||
@@ -107,9 +112,7 @@ UNIT_ROUTES = {
|
||||
ALLOWED = {
|
||||
"channels.py": 83,
|
||||
"discover.py": 7,
|
||||
"downloads.py": 88,
|
||||
"feed.py": 36,
|
||||
"files.py": 58,
|
||||
"playlists.py": 72,
|
||||
"plex/_shared.py": 1,
|
||||
"plex/admin.py": 12,
|
||||
@@ -120,14 +123,12 @@ ALLOWED = {
|
||||
"plex/people.py": 20,
|
||||
"plex/playback.py": 42,
|
||||
"plex/playlists.py": 80,
|
||||
"public.py": 12,
|
||||
"runs.py": 1,
|
||||
"saved_views.py": 17,
|
||||
"scheduler.py": 3,
|
||||
"search.py": 46,
|
||||
"sync.py": 38,
|
||||
"tags.py": 10,
|
||||
"tokens.py": 1,
|
||||
}
|
||||
|
||||
# The session methods that read or write data. `commit` and `rollback` stay with the handler.
|
||||
@@ -184,6 +185,20 @@ def _session_names(tree: ast.AST) -> set[str]:
|
||||
return names
|
||||
|
||||
|
||||
def _type_only_imports(tree: ast.AST) -> set[ast.AST]:
|
||||
"""The imports under `if TYPE_CHECKING:` — they name a type for an annotation and are never
|
||||
loaded at run time, so a model imported there cannot be queried or written."""
|
||||
out: set[ast.AST] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.If):
|
||||
test = node.test
|
||||
name = test.id if isinstance(test, ast.Name) else getattr(test, "attr", None)
|
||||
if name == "TYPE_CHECKING":
|
||||
for child in node.body:
|
||||
out |= set(ast.walk(child))
|
||||
return out
|
||||
|
||||
|
||||
def _root(node: ast.expr) -> ast.expr:
|
||||
while isinstance(node, (ast.Attribute, ast.Subscript)):
|
||||
node = node.value
|
||||
@@ -196,8 +211,9 @@ def violations(source: str, unit: str | None) -> list[str]:
|
||||
tree = ast.parse(source)
|
||||
found = []
|
||||
sql_names = set()
|
||||
typing_only = _type_only_imports(tree)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
if isinstance(node, ast.ImportFrom) and node not in typing_only:
|
||||
module = node.module or ""
|
||||
for alias in node.names:
|
||||
bound = alias.asname or alias.name
|
||||
@@ -295,6 +311,9 @@ DETECTOR_CASES = {
|
||||
"from app.capture.models import DownloadJob as J": 1,
|
||||
"def f(cache): cache.get('k')": 0,
|
||||
"from sqlalchemy.exc import IntegrityError\ntry:\n pass\nexcept IntegrityError:\n pass": 0,
|
||||
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.downloads.models import Job": 0,
|
||||
"import typing\nif typing.TYPE_CHECKING:\n from app.models import DownloadJob": 0,
|
||||
"if TYPE_CHECKING:\n pass\nfrom app.downloads.models import DownloadJob": 1,
|
||||
}
|
||||
|
||||
|
||||
@@ -304,3 +323,185 @@ def test_the_detector_sees_each_kind_of_database_work():
|
||||
assert violations("from app.models import Video\nfrom sqlalchemy import select", None) == [
|
||||
"1: imports catalog's Video"
|
||||
]
|
||||
|
||||
|
||||
# --- inside a unit: its package --------------------------------------------------------------
|
||||
# A unit whose sprint has landed is fenced beyond its HTTP layer too. Its own code reads another
|
||||
# unit's tables only along a declared edge (the feed joins Video with VideoState in one query; a
|
||||
# boundary that split it would cost the speed the app is for), and writes another unit's tables
|
||||
# never — that goes through the owner's service. Whose a row is, it says through `userscope`, and
|
||||
# it commits only where a job owns its session.
|
||||
|
||||
# Each unit's code outside the HTTP layer: its package, and the entry point that runs its jobs.
|
||||
UNIT_PACKAGES = {"capture": ("downloads/", "worker.py")}
|
||||
# The units fenced so far; the rest join with their sprint.
|
||||
FENCED = {"capture"}
|
||||
# Whose tables a unit's own code may READ. Core code reads a fenced unit's tables through its queries.
|
||||
READS = {"capture": {"catalog", "plex"}}
|
||||
# The columns that say whose a row is.
|
||||
OWNER_COLUMNS = {"user_id", "to_user_id", "from_user_id", "created_by"}
|
||||
# Where a fenced unit's code may commit: the jobs that own their session (the worker and what it
|
||||
# runs, the GC, the backfill — none of them inside a request), and the one write that must commit
|
||||
# where it stands: an upload's rows and the file it placed on disk land together or not at all, so
|
||||
# a failed commit can remove the file (`upload.ingest_upload`). Everything else flushes; the handler
|
||||
# commits.
|
||||
UNIT_COMMITS = {
|
||||
"capture": {
|
||||
"worker.py",
|
||||
"downloads/backfill.py",
|
||||
"downloads/gc.py",
|
||||
"downloads/jobdb.py",
|
||||
"downloads/pipeline.py",
|
||||
"downloads/upload.py:ingest_upload",
|
||||
}
|
||||
}
|
||||
SQL_WRITES = {"insert", "update", "delete", "pg_insert"}
|
||||
|
||||
|
||||
def _package_unit(rel: str) -> str | None:
|
||||
for unit, prefixes in UNIT_PACKAGES.items():
|
||||
if any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in prefixes):
|
||||
return unit
|
||||
return None
|
||||
|
||||
|
||||
def _app_files():
|
||||
for path in sorted(APP.rglob("*.py")):
|
||||
rel = path.relative_to(APP).as_posix()
|
||||
if not rel.startswith("routes/"):
|
||||
yield rel, path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _imported_models(tree: ast.AST) -> dict[str, str]:
|
||||
"""The unit models a module imports at run time (a `TYPE_CHECKING` import names a type only),
|
||||
by the name it binds them to."""
|
||||
typing_only = _type_only_imports(tree)
|
||||
out = {}
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node not in typing_only:
|
||||
module = node.module or ""
|
||||
if module.startswith("app.") and "models" in module:
|
||||
for alias in node.names:
|
||||
if alias.name in UNIT_MODEL_NAMES:
|
||||
out[alias.asname or alias.name] = alias.name
|
||||
return out
|
||||
|
||||
|
||||
def _enclosing_functions(tree: ast.AST) -> dict[ast.AST, str]:
|
||||
out = {}
|
||||
for fn in ast.walk(tree):
|
||||
if isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
for node in ast.walk(fn):
|
||||
out.setdefault(node, fn.name)
|
||||
return out
|
||||
|
||||
|
||||
def package_violations(rel: str, source: str) -> list[str]:
|
||||
"""What a module outside the HTTP layer does against a fenced unit's boundary, as `line: what`."""
|
||||
tree = ast.parse(source)
|
||||
mine = _package_unit(rel)
|
||||
found = []
|
||||
models = _imported_models(tree)
|
||||
for name in models.values():
|
||||
theirs = UNIT_MODEL_NAMES[name]
|
||||
if theirs == mine or (mine not in FENCED and theirs not in FENCED):
|
||||
continue
|
||||
if theirs not in READS.get(mine or "", set()):
|
||||
found.append(f"0: {mine or 'core'} imports {theirs}'s {name}")
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name):
|
||||
continue
|
||||
written = None
|
||||
if node.func.id in models:
|
||||
written = models[node.func.id]
|
||||
elif node.func.id in SQL_WRITES and node.args and isinstance(node.args[0], ast.Name):
|
||||
written = models.get(node.args[0].id)
|
||||
if written is None:
|
||||
continue
|
||||
theirs = UNIT_MODEL_NAMES[written]
|
||||
if theirs != mine and (mine in FENCED or theirs in FENCED):
|
||||
found.append(f"{node.lineno}: {mine or 'core'} writes {theirs}'s {written}")
|
||||
if mine in FENCED:
|
||||
found += _owner_filters(tree)
|
||||
functions = _enclosing_functions(tree)
|
||||
allowed = UNIT_COMMITS.get(mine, set())
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "commit"
|
||||
and rel not in allowed
|
||||
and f"{rel}:{functions.get(node)}" not in allowed
|
||||
):
|
||||
found.append(f"{node.lineno}: commits in {functions.get(node)}")
|
||||
return sorted(found, key=lambda v: int(v.split(":")[0]))
|
||||
|
||||
|
||||
def _owner_filters(tree: ast.AST) -> list[str]:
|
||||
"""A comparison on a table's owner column written inline (`Model.user_id == …`), not through
|
||||
`userscope.owned_by`."""
|
||||
found = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Compare):
|
||||
for side in (node.left, *node.comparators):
|
||||
if (
|
||||
isinstance(side, ast.Attribute)
|
||||
and isinstance(side.value, ast.Name)
|
||||
and side.value.id in set(UNIT_MODEL_NAMES) | CORE_MODELS
|
||||
and side.attr in OWNER_COLUMNS
|
||||
):
|
||||
found.append(f"{node.lineno}: filters {side.value.id}.{side.attr} inline")
|
||||
return found
|
||||
|
||||
|
||||
def test_a_fenced_units_code_keeps_to_its_edges():
|
||||
hits = {rel: found for rel, src in _app_files() if (found := package_violations(rel, src))}
|
||||
assert hits == {}, (
|
||||
"read another unit's tables only along READS, write them only through their service, say"
|
||||
" whose a row is through userscope, commit only where UNIT_COMMITS says"
|
||||
)
|
||||
|
||||
|
||||
def test_a_fenced_units_routes_filter_owners_through_userscope():
|
||||
hits = {
|
||||
rel: found
|
||||
for rel, src in _http_files()
|
||||
if _unit_of(rel) in FENCED and (found := _owner_filters(ast.parse(src)))
|
||||
}
|
||||
assert hits == {}
|
||||
|
||||
|
||||
def test_every_unit_package_exists():
|
||||
for prefixes in UNIT_PACKAGES.values():
|
||||
for p in prefixes:
|
||||
assert (APP / p).exists(), p
|
||||
for commits in UNIT_COMMITS.values():
|
||||
for entry in commits:
|
||||
assert (APP / entry.split(":")[0]).exists(), entry
|
||||
|
||||
|
||||
PACKAGE_CASES = {
|
||||
# (module path, source): violations
|
||||
("downloads/x.py", "from app.models import Video"): 0, # capture reads catalog
|
||||
("downloads/x.py", "from app.models import Playlist"): 1, # no capture→reader edge
|
||||
("downloads/x.py", "from app.models import Video\nVideo(id=1)"): 1, # a write across
|
||||
("downloads/x.py", "from app.models import Video\nupdate(Video)"): 1,
|
||||
("downloads/x.py", "from app.downloads.models import DownloadJob\nDownloadJob()"): 0,
|
||||
("search.py", "from app.downloads.models import DownloadJob"): 1, # core reads capture
|
||||
("search.py", "from app.models import Video\nVideo()"): 0, # neither side fenced yet
|
||||
("downloads/x.py", "q = DownloadJob.user_id == 1"): 1,
|
||||
("downloads/x.py", "q = owned_by(DownloadJob.user_id, 1)"): 0,
|
||||
("downloads/x.py", "def f(db):\n db.commit()"): 1,
|
||||
("downloads/gc.py", "def f(db):\n db.commit()"): 0,
|
||||
("downloads/upload.py", "def ingest_upload(db):\n db.commit()"): 0,
|
||||
("downloads/upload.py", "def other(db):\n db.commit()"): 1,
|
||||
(
|
||||
"downloads/x.py",
|
||||
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.models import Playlist",
|
||||
): 0,
|
||||
}
|
||||
|
||||
|
||||
def test_the_package_detector_sees_each_kind():
|
||||
for (rel, source), expected in PACKAGE_CASES.items():
|
||||
assert len(package_violations(rel, source)) == expected, (rel, source)
|
||||
|
||||
@@ -33,7 +33,8 @@ from app.downloads import mediakind, pipeline, playable, quota, service, storage
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
|
||||
|
||||
# --- pure: the kind invariant ---------------------------------------------------------------
|
||||
|
||||
@@ -21,7 +21,8 @@ from app.downloads import pipeline, quota, storage
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, Notification
|
||||
from app.models import Notification
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -12,7 +12,7 @@ from app.config import settings
|
||||
from app.downloads import pipeline, runner, service, storage
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ project checks needs a row.
|
||||
| CVE gate | HIGH/CRITICAL with a fix available only | `--ignore-unfixed` + `--severity HIGH,CRITICAL`: a finding with no fix cannot be acted on by an upgrade; it shows in the scan output. A fixable one fails the publish, and the only way past it is `.trivyignore.yaml`. | 0 open (E32 S1) |
|
||||
| npm audit | `--omit=dev --audit-level=high` | What ships to the browser, at high or above. The dev toolchain's advisories were burned down by hand in E32 S1 (18 → 0). | 0 |
|
||||
| knip | `entry` — one file added | Not a narrowing: knip's Vite plugin reads entries from index.html only, so the shared link's page (`watch.html` → `src/watchMain.tsx`) is named here, or the lane reports that whole entry unused. It adds files to the lane, never takes any out. | 1 entry |
|
||||
| unit fence | the database work the HTTP layer still does, pinned | `backend/tests/test_unit_fences.py`: a unit's route file may not build SQL, call the session's data methods, set a model's attributes or import a unit's tables; a core route may not import a unit's tables. Switched on unit by unit: `ALLOWED` pins each file's count exactly, so a rise fails and a fall fails until the pin is lowered — the list may only shrink. End state: empty, and the pin mechanism goes. | 826 in 23 files (E42 S0) |
|
||||
| unit fence | the database work the HTTP layer still does, pinned | `backend/tests/test_unit_fences.py`: a unit's route file may not build SQL, call the session's data methods, set a model's attributes or import a unit's tables; a core route may not import a unit's tables. Switched on unit by unit: `ALLOWED` pins each file's count exactly, so a rise fails and a fall fails until the pin is lowered — the list may only shrink. End state: empty, and the pin mechanism goes. A unit in `FENCED` is held below its HTTP layer too: its code reads another unit's tables only along `READS`, writes them never, filters owners through `userscope`, and commits only where `UNIT_COMMITS` names a job's own session. | 826 in 23 files (E42 S0) → 667 in 19 (S1: Capture 0, fenced) |
|
||||
| React Compiler | the functions it skips, pinned | `components/compilerCoverage.contract.test.ts` — the list may only shrink. End state: only VirtualGrid (TanStack Virtual, an IncompatibleLibrary) remains. | 93 (E32 S1) → 29 (E32 S3) → 1, VirtualGrid (E32 S4) |
|
||||
|
||||
## Accepted findings
|
||||
|
||||
@@ -29,8 +29,10 @@ describe("display-name bound", () => {
|
||||
it("is the width the columns are actually declared with", () => {
|
||||
// The mutation guard for the check above: both constants could agree on a number neither
|
||||
// column uses. `String(DISPLAY_TEXT_MAX)` is how the three display columns are declared, so a
|
||||
// literal creeping back into one of them fails here.
|
||||
const models = src("../../../../backend/app/models.py");
|
||||
// literal creeping back into one of them fails here. `User.display_name` is a core table; the
|
||||
// job's two are Capture's.
|
||||
const models =
|
||||
src("../../../../backend/app/models.py") + src("../../../../backend/app/downloads/models.py");
|
||||
const declared = [...models.matchAll(/display_(?:name|uploader): Mapped.*?String\((.*?)\)/g)];
|
||||
expect(declared).toHaveLength(3);
|
||||
for (const m of declared) expect(m[1]).toBe("DISPLAY_TEXT_MAX");
|
||||
|
||||
@@ -10,6 +10,11 @@ import { FILE_SORTS, FILES_LIST_CAP, listQuery } from "./files";
|
||||
// matches everything. The route's parameter names are read from its source, as text — the backend
|
||||
// cannot be imported here.
|
||||
const routes = readFileSync(resolve(__dirname, "../../../../backend/app/routes/files.py"), "utf-8");
|
||||
// The cap is the query's (backend downloads/queries.py), which the route lists through.
|
||||
const queries = readFileSync(
|
||||
resolve(__dirname, "../../../../backend/app/downloads/queries.py"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
function routeParams(): Map<string, string> {
|
||||
const sig = routes.match(/^def list_files\(([\s\S]*?)\) -> /m);
|
||||
@@ -43,8 +48,8 @@ describe("the Files list's request", () => {
|
||||
|
||||
it("knows the cap the route lists up to", () => {
|
||||
// A list as long as the cap may be missing older files; the page then searches on the server.
|
||||
const cap = routes.match(/^MAX_LIST = (\d+)$/m);
|
||||
expect(cap, "routes/files.py no longer defines MAX_LIST").toBeTruthy();
|
||||
const cap = queries.match(/^MAX_LIST = (\d+)$/m);
|
||||
expect(cap, "downloads/queries.py no longer defines MAX_LIST").toBeTruthy();
|
||||
expect(FILES_LIST_CAP).toBe(Number(cap![1]));
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user