peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(files): the Files library API — folders, moving and keeping files, upload into a folder, the file's hash

Browse Source 
/api/files is the library's home: list a user's done files (by folder, searched, sorted, or the
ones shared with them), rename / move / keep / delete a file, make, rename and delete own folders
(a deleted folder's files go back to their source folders), and set how long any folder keeps a
file — done files shift by the difference, never closer than the GC's warning window.

An upload can name its folder (an own folder or Uploads), checked before the body is read and again
before the file lands; it is hashed while it streams, and any file can be hashed on first ask. The
download says the hash (Repr-Digest) once it is known; a new file at an asset's path forgets it.

Uploads, deletes, keeps, moves and folder changes are audited (ids only). The fileless "expired"
rows of the old retention are deleted by migration 0103: an expired file now leaves the library.

The shared route helpers moved to routes/files.py; the old /api/downloads library routes delegate
to the new ones until the client moves over.
This commit is contained in:
peter
2026-10-11 04:42:55 +02:00
parent bf99444ded
commit 1a1665b8ec
24 changed files with 1483 additions and 399 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.basedpyright/baseline.json
-40
View File
@@ -727,30 +727,6 @@
}
],
"./app/routes/downloads.py": [
{
"code": "reportArgumentType",
"range": {
"startColumn": 29,
"endColumn": 33,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 29,
"endColumn": 33,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 40,
"endColumn": 50,
"lineCount": 1
}
},
{
"code": "reportOptionalMemberAccess",
"range": {
@@ -790,22 +766,6 @@
"endColumn": 54,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 29,
"endColumn": 33,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 40,
"endColumn": 50,
"lineCount": 1
}
}
],
"./app/routes/playlists.py": [
backend/alembic/versions/0103_files_sha256.py
+32
View File
@@ -0,0 +1,32 @@
"""The Files library: a file's SHA-256, and the expired rows retired
Revision ID: 0103_files_sha256
Revises: 0102_file_folders
Create Date: 2026-10-11
`media_assets.sha256` holds the file's hash once it is known (an upload is hashed while it streams
in, any other file on the first ask), so a machine fetching a file can check what it got.
An expired file now leaves its user's library (`downloads.gc`). Before, an expired download stayed
as a fileless "expired · download again" row (a done job with no asset). Those rows hold nothing on
disk and the library no longer shows them, so they are deleted here rather than left behind
unseen; their shares and public links go with them (ON DELETE CASCADE), and those already served
nothing. The downgrade cannot bring them back.
"""
import sqlalchemy as sa
from alembic import op
revision: str = "0103_files_sha256"
down_revision: str | None = "0102_file_folders"
branch_labels: str | None = None
depends_on: str | None = None
def upgrade() -> None:
op.add_column("media_assets", sa.Column("sha256", sa.String(64)))
op.execute("DELETE FROM download_jobs WHERE status = 'done' AND asset_id IS NULL")
def downgrade() -> None:
op.drop_column("media_assets", "sha256")
backend/app/audit.py
+11
View File
@@ -58,6 +58,17 @@ class AuditAction:
# A user sent a debug session's issue to the tracker — the one act that takes their log
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
ISSUE_CREATE = "issue_create"
# The Files library (`routes.files`): what a user put in, took out, kept or reorganised. Ids
# only — a file's or folder's name is the user's own text, not the admin's to read here.
FILE_UPLOAD = "file_upload"
FILE_DELETE = "file_delete"
FILE_KEEP = "file_keep"
FILE_UNKEEP = "file_unkeep"
FILE_MOVE = "file_move"
FILE_FOLDER_CREATE = "file_folder_create"
FILE_FOLDER_RENAME = "file_folder_rename"
FILE_FOLDER_RETENTION = "file_folder_retention"
FILE_FOLDER_DELETE = "file_folder_delete"
def record(
backend/app/bodylimit.py
+1
View File
@@ -94,6 +94,7 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
# sending it at all, off the `upload_max_bytes` that `GET /api/downloads/usage` reports.
("POST", "/api/downloads/upload"): settings.download_upload_max_bytes,
("POST", "/api/files/upload"): settings.download_upload_max_bytes,
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
}
backend/app/downloads/folders.py
+89 -1
View File
@@ -9,7 +9,9 @@ shared asset can be held by several users), so a rename or a move is a column up
- an OWN folder is the user's, with the name they gave it.
Every file of a user is in exactly one folder. The folder a new file lands in is decided here
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once.
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file
moves only into one of the user's own folders or back to its own source folder (`move`): a YouTube
file in the TikTok folder would be a lie the folder's name tells.
Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A
@@ -26,6 +28,7 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app import sysconfig
from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset
@@ -137,3 +140,88 @@ def stamp_expiry(db: Session, job: DownloadJob) -> None:
"""Stamp a job that just became done (in this session) with its expiry."""
job.expires_at = done_expiry(db, job.folder_id, job.kept)
job.expiry_warned = False
class FolderError(Exception):
"""A folder change the rules refuse; `reason` is the refusal the route answers with."""
def __init__(self, reason: str) -> None:
super().__init__(reason)
self.reason = reason
def upload_target(db: Session, user_id: int, folder_id: int) -> FileFolder:
"""The folder an upload asked to land in, if it may: one of the user's own folders, or their
Uploads folder. A file put in a site's folder would claim an origin it does not have."""
folder = db.get(FileFolder, folder_id)
if folder is None or folder.user_id != user_id:
raise FolderError("file_folder_gone")
if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
raise FolderError("file_folder_not_its_source")
return folder
def restamp(db: Session, job: DownloadJob) -> None:
"""Start a file's clock again from now: it moved to another folder, or its user stopped keeping
it. A file not done yet gets its expiry when it finishes (`stamp_expiry`), so it has none now."""
done = job.status == JobStatus.DONE
job.expires_at = done_expiry(db, job.folder_id, job.kept) if done else None
job.expiry_warned = False
def set_kept(db: Session, job: DownloadJob, kept: bool) -> None:
"""Keep a file until its user deletes it, or hand it back to its folder's retention."""
job.kept = kept
restamp(db, job)
def move(db: Session, job: DownloadJob, folder: FileFolder) -> None:
"""Put a file in `folder` — one of its user's own folders, or the source folder it came from.
The caller has checked that `folder` is the job's user's."""
if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job):
raise FolderError("file_folder_not_its_source")
if folder.id == job.folder_id:
return
job.folder_id = folder.id
restamp(db, job)
def set_retention(db: Session, folder: FileFolder, days: int | None) -> None:
"""Change how long `folder` keeps a file (None = until deleted). The done files already in it
move by the difference — each keeps the age it has — but none ends up closer than the GC's
warning window, so a shortened retention still warns before it takes anything."""
old = folder.retention_days
if old == days:
return
folder.retention_days = days
now = datetime.now(timezone.utc)
floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days"))
jobs = db.execute(
select(DownloadJob).where(
DownloadJob.folder_id == folder.id,
DownloadJob.status == JobStatus.DONE,
DownloadJob.kept.is_(False),
)
).scalars()
for job in jobs:
if days is None:
job.expires_at = None
elif old is None or job.expires_at is None:
job.expires_at = max(now + timedelta(days=days), floor)
else:
job.expires_at = max(job.expires_at + timedelta(days=days - old), floor)
job.expiry_warned = False
def delete_own(db: Session, folder: FileFolder) -> int:
"""Delete one of a user's own folders. Its files are not deleted: each goes back to the source
folder it came from, its clock starting there now. Returns how many moved."""
if folder.kind != FolderKind.OWN:
raise FolderError("file_folder_not_own")
jobs = db.execute(select(DownloadJob).where(DownloadJob.folder_id == folder.id)).scalars().all()
for job in jobs:
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
restamp(db, job)
db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete
db.delete(folder)
return len(jobs)
backend/app/downloads/upload.py
+18 -1
View File
@@ -31,6 +31,8 @@ from pathlib import Path
from sqlalchemy.orm import Session
from app import audit
from app.audit import AuditAction
from app.config import settings
from app.downloads import edit as editmod
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
@@ -53,14 +55,22 @@ def ingest_upload(
src_path: Path,
size: int,
display_name: str | None = None,
sha256: str | None = None,
folder_id: int | None = None,
) -> DownloadJob:
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return its job —
`done`, or `queued` for the worker's phone-safe conversion (see the module docstring). `src_path`
is MOVED into place on success (the caller stages it to a temp file first).
is MOVED into place on success (the caller stages it to a temp file first). `sha256` is the
hash of those bytes, when the caller took it on the way in; `folder_id` the folder it asked for
(`folders.upload_target`), else its Uploads folder.
Raises folders.FolderError when that folder is not one an upload may land in (or is gone).
Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a
rejected upload leaves nothing behind and the caller just cleans up its temp file."""
quota.check_upload(db, user_id, size) # the file's size is known → charge it up front
if folder_id is not None:
folders.upload_target(db, user_id, folder_id)
data = probe.probe_json(src_path)
vcodec, acodec, width, height, duration = probe.facts(data)
@@ -93,6 +103,9 @@ def ingest_upload(
asset.status = "pending" if convert else "ready"
asset.rel_path = rel
# The hash names the bytes that arrived: a conversion replaces them, so it is kept only
# when the file stays as it came.
asset.sha256 = None if convert else sha256
asset.size_bytes = actual_size
asset.container = ext or None
asset.vcodec = vcodec
@@ -124,9 +137,13 @@ def ingest_upload(
)
db.add(job)
folders.place(db, job)
if folder_id is not None:
job.folder_id = folder_id
if not convert:
folders.stamp_expiry(db, job)
service.incr_ref_count(db, asset.id)
db.flush()
audit.record(db, user_id, AuditAction.FILE_UPLOAD, target_type="file", target_id=job.id)
db.commit()
except Exception:
db.rollback()
backend/app/main.py
+2
View File
@@ -42,6 +42,7 @@ from app.routes import ( # noqa: E402 — logging is configured first
discover as discover_routes,
downloads,
feed,
files as files_routes,
health,
me,
notifications,
@@ -316,6 +317,7 @@ app.include_router(saved_views.router)
app.include_router(plex_routes.router)
app.include_router(downloads.router)
app.include_router(downloads.admin_router)
app.include_router(files_routes.router)
app.include_router(public_routes.router)
app.include_router(admin.router)
app.include_router(config_routes.router)
backend/app/models.py
+9
View File
@@ -883,6 +883,15 @@ class MediaAsset(Base, TimestampMixin):
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
sha256: Mapped[str | None] = mapped_column(String(64))
@validates("rel_path")
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
self.sha256 = None
return rel_path
class DownloadProfile(Base, TimestampMixin):
backend/app/routes/_common.py
+8 -2
View File
@@ -6,7 +6,7 @@ and "fetch a row this user owns or 404" idioms have one definition, one status c
one existence-leak policy.
"""
from typing import IO, Any, TypeVar
from typing import Any, Protocol, TypeVar
from fastapi import HTTPException, Request
from pydantic import BaseModel, ConfigDict, Field
@@ -122,8 +122,14 @@ async def capped_body(request: Request, limit: int) -> bytes:
return bytes(body)
class ByteSink(Protocol):
"""Where `capped_stream_to` writes: a file, or anything that takes the bytes on their way to one."""
def write(self, data: bytes, /) -> object: ...
async def capped_stream_to(
request: Request, out: IO[bytes], limit: int, refusal: HTTPException
request: Request, out: ByteSink, limit: int, refusal: HTTPException
) -> int:
"""`capped_body`'s sibling for a body too large to hold: the same counting rule, written
straight to `out` instead of accumulated, returning the size.
backend/app/routes/downloads.py
+59 -325
View File
@@ -5,17 +5,12 @@ frontend (the worker is a separate process); this module never downloads — it
state, serves finished files (range-aware, with the user's custom display name), and shares them.
"""
import os
import re
import tempfile
from typing import Literal
from collections.abc import AsyncIterator, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from urllib.parse import parse_qs, unquote, urlparse
from urllib.parse import parse_qs, urlparse
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from sqlalchemy import delete, func, select
@@ -27,13 +22,12 @@ from app.collation import collated
from app import audit, sysconfig
from app.auth import admin_user, current_user
from app.config import settings
from app.db import get_db, release_read_connection
from app.db import get_db
from app.downloads import clip as clipmod
from app.downloads import credentials as dlcreds
from app.downloads import edit as editmod
from app.downloads import links as linksmod
from app.downloads import quota, service, storage
from app.downloads import upload as uploadmod
from app.downloads import urlguard
from app.downloads.status import NOT_REFETCHABLE, JobStatus
from app.models import (
@@ -49,15 +43,28 @@ from app.models import (
)
from app.routes._common import (
ProgressIn,
capped_stream_to,
get_or_404,
kept_position,
not_found,
owned_or_404,
)
from app.routes.files import (
MAX_LIST,
SpooledUpload,
accessible_job,
assets_for,
delete_file,
own_job_for_update,
positions_for,
quota_detail,
serialize,
serialize_job,
serve_file,
spool_upload,
ingest_spooled,
)
from app.security import hash_password
from app.userscope import is_messageable_user, messageable_clauses
from app.utils import iso
from app.refusal import refused
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
@@ -65,11 +72,6 @@ admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"]
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
# Cap the per-user job listings so a heavy library can't return an unbounded result set. The
# Download Center renders newest-first; 500 is far beyond any real manual-download history (and
# matches the admin listing's own cap).
_MAX_LIST = 500
# --- source + serialization ----------------------------------------------------------------
@@ -113,118 +115,6 @@ def resolve_source(raw: str) -> tuple[str, str]:
raise refused(400, "invalid_link")
def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
return {
"id": job.id,
"status": job.status,
"progress": job.progress,
"speed_bps": job.speed_bps,
"eta_s": job.eta_s,
"phase": job.phase,
"error": job.error,
"error_code": job.error_code,
"error_detail": job.error_detail,
"queue_pos": job.queue_pos,
"created_at": iso(job.created_at),
"source_kind": job.source_kind,
"source_ref": job.source_ref,
"source_url": service.reference_url(job, asset),
# A locally-generated poster to fall back to when the source had no thumbnail.
"poster_url": f"/api/downloads/{job.id}/poster.jpg"
if (asset is not None and asset.poster_path)
else None,
"profile_id": job.profile_id,
"spec": job.profile_snapshot,
"display_name": job.display_name,
"display_uploader": job.display_uploader,
"display_uploader_url": job.display_uploader_url,
"extra_links": job.extra_links or [],
"job_kind": job.job_kind,
"source_job_id": job.source_job_id,
"edit_spec": job.edit_spec,
"can_download": ready and job.status == JobStatus.DONE,
"expired": job.asset_id is None and job.status == JobStatus.DONE,
# The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
"folder_id": job.folder_id,
"kept": job.kept,
"expires_at": iso(job.expires_at),
"asset": None
if asset is None
else {
"id": asset.id,
"status": asset.status,
"title": asset.title,
"uploader": asset.uploader,
"upload_date": iso(asset.upload_date),
"duration_s": asset.duration_s,
"size_bytes": asset.size_bytes,
"width": asset.width,
"height": asset.height,
"container": asset.container,
# "video" | "audio" | "file" — the library gates play/edit on this (media only); a
# generic uploaded file is download-only. Null while still in flight (set at finalize).
"media_kind": asset.media_kind,
"uploader_url": asset.uploader_url,
"thumbnail_url": asset.thumbnail_url,
},
}
def _assets_for(db: Session, jobs: list[DownloadJob]) -> dict[int, MediaAsset]:
ids = {j.asset_id for j in jobs if j.asset_id}
if not ids:
return {}
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
return {a.id: a for a in rows}
def _positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
ids = [j.id for j in jobs]
if not ids:
return {}
rows = db.execute(
select(LibraryState.job_id, LibraryState.position_seconds).where(
LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
)
).all()
return {job_id: pos for job_id, pos in rows}
def _serialize_job(db: Session, job: DownloadJob) -> dict:
"""Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
every single-job handler shares."""
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
return _serialize(job, asset)
def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
or writes the status those decide on (pause). The job is re-read under a row lock, so two
overlapping requests for one job run one after the other and the second decides on the first's
committed result — never both releasing the same hold, and never a pause turning a job a cancel
just released back into a holder; either can free a file another job still holds.
The asset is locked BEFORE the job: the order a finishing worker child takes them in
(`service.finalize_if_running`), so the two can never deadlock."""
while True:
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
if asset_id is not None:
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
job = db.execute(
select(DownloadJob)
.where(DownloadJob.id == job_id)
.with_for_update()
.execution_options(populate_existing=True)
).scalar_one_or_none()
if job is None:
raise not_found("download")
if job.asset_id == asset_id:
return job
db.rollback() # another request moved the job to another asset meanwhile: lock that one
# --- profiles ------------------------------------------------------------------------------
@@ -434,91 +324,8 @@ def enqueue_download(
try:
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
return _serialize_job(db, job)
# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous
# caller could make the process write the whole per-file allowance to temp, per request (SEC-20
# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
# declared AFTER the auth one, is the only shape that closes it — the same one
# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
# every route with a raised ceiling rather than leaving it to a comment like this one.
#
# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
# Hungarian filename would otherwise be unsendable.
UPLOAD_FILENAME_HEADER = "x-upload-filename"
UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
@dataclass(frozen=True)
class SpooledUpload:
"""What the body dependency hands the handler: the bytes already on disk, plus the two names
that came alongside them."""
path: Path
size: int
user_id: int
filename: str
display_name: str | None
def _header_text(request: Request, name: str) -> str:
"""One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
own fallback names the file."""
raw = request.headers.get(name, "")
try:
return unquote(raw, errors="strict").strip()
except (UnicodeDecodeError, ValueError):
# silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
# cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
# a label; logging it would hand any client a free log line per request.
return ""
async def spool_upload(
request: Request,
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> AsyncIterator[SpooledUpload]:
"""Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
`user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so
naming `current_user` here makes the auth check a precondition of reading the body at all — a
stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest
MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit,
empty, quota, probe or move error, a client that vanished mid-upload — it is removed."""
cap = sysconfig.get_int(db, "download_upload_max_bytes")
user_id = user.id
# Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
# for why, and for the precondition it enforces. Everything this dependency still needs is
# already an int or a header in hand: touching an ORM attribute below would reload `user` and
# check a connection straight back out.
release_read_connection(db)
staging = Path(settings.files_root) / ".staging" / "uploads"
staging.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(staging))
tmp_path = Path(tmp)
try:
with os.fdopen(fd, "wb") as out:
size = await capped_stream_to(
request,
out,
cap,
HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
)
yield SpooledUpload(
path=tmp_path,
size=size,
user_id=user_id,
filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
)
finally:
tmp_path.unlink(missing_ok=True)
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
return serialize_job(db, job)
@router.post("/upload")
@@ -526,30 +333,9 @@ def upload_download(
upload: SpooledUpload = Depends(spool_upload),
db: Session = Depends(get_db),
) -> dict:
"""Ingest a local file into the caller's library. Any file type is accepted; media
(video/audio) gets the full function set, a generic file is download-only. The body IS the file
(see the header note above); `spool_upload` has already authenticated the caller and streamed it
to a temp path under the download volume, so what is left here is the non-yt-dlp ingest which
probes + places it and returns its job — `done`, or `queued` for the worker's phone-safe
conversion (`downloads.upload`).
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
plus sync DB writes — so it must NOT run on the event loop. The streaming half is async because
reading the body is; keeping them in separate callables is what lets each run where it belongs."""
if upload.size == 0:
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
try:
job = uploadmod.ingest_upload(
db,
upload.user_id,
filename=upload.filename,
src_path=upload.path,
size=upload.size,
display_name=upload.display_name,
)
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
return _serialize_job(db, job)
"""The Files library's upload (`routes.files.upload_file`) at its old address, until the client
moves to the new one."""
return serialize_job(db, ingest_spooled(db, upload))
class EnqueueEditIn(BaseModel):
@@ -567,7 +353,7 @@ def enqueue_edit(
The source may be the user's own download OR one shared with them — either way the resulting
clip is a NEW per-user derivative in the editor's own library (counts against their quota); the
source file is only read, never modified, so editing a shared video is safe."""
source_job = _accessible_job(db, user, body.source_job_id)
source_job = accessible_job(db, user, body.source_job_id)
if source_job.status != JobStatus.DONE or source_job.asset_id is None:
raise refused(409, "edit_unfinished")
spec = body.edit_spec
@@ -575,10 +361,10 @@ def enqueue_edit(
try:
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
except service.EditError as e:
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e
return _serialize_job(db, job)
return serialize_job(db, job)
class ClipCropIn(BaseModel):
@@ -620,14 +406,8 @@ def enqueue_clip(
detail = {"code": "clip", "reason": e.reason, **e.params}
raise HTTPException(status_code=status, detail=detail) from e
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
return _serialize_job(db, job)
def _quota_detail(e: quota.QuotaExceeded) -> dict:
"""Structured error the trilingual client localizes (reason key + numbers), instead of a
pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current}
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
return serialize_job(db, job)
@router.get("")
@@ -637,16 +417,16 @@ def list_downloads(user: User = Depends(current_user), db: Session = Depends(get
select(DownloadJob)
.where(DownloadJob.user_id == user.id)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
.limit(_MAX_LIST)
.limit(MAX_LIST)
)
.scalars()
.all()
)
assets = _assets_for(db, jobs)
positions = _positions_for(db, user, jobs)
assets = assets_for(db, jobs)
positions = positions_for(db, user, jobs)
return [
{
**_serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
"position_seconds": positions.get(j.id, 0),
}
for j in jobs
@@ -705,16 +485,16 @@ def shared_with_me(user: User = Depends(current_user), db: Session = Depends(get
.join(DownloadShare, DownloadShare.job_id == DownloadJob.id)
.where(DownloadShare.to_user_id == user.id)
.order_by(DownloadShare.created_at.desc(), DownloadJob.id.desc())
.limit(_MAX_LIST)
.limit(MAX_LIST)
)
.scalars()
.all()
)
assets = _assets_for(db, jobs)
positions = _positions_for(db, user, jobs)
assets = assets_for(db, jobs)
positions = positions_for(db, user, jobs)
out = []
for j in jobs:
data = _serialize(j, assets.get(j.asset_id))
data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None)
data["shared"] = True
data["position_seconds"] = positions.get(j.id, 0)
out.append(data)
@@ -778,25 +558,25 @@ def update_download_meta(
cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
job.extra_links = cleaned or None
db.commit()
return _serialize_job(db, job)
return serialize_job(db, job)
@router.post("/{job_id}/pause")
def pause_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
job = _own_job_for_update(db, user, job_id)
job = own_job_for_update(db, user, job_id)
service.pause(db, job)
return _serialize_job(db, job)
return serialize_job(db, job)
@router.post("/{job_id}/resume")
def resume_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
job = _own_job_for_update(db, user, job_id)
job = own_job_for_update(db, user, job_id)
service.resume(db, job)
return _serialize_job(db, job)
return serialize_job(db, job)
@router.post("/{job_id}/redownload")
@@ -815,7 +595,7 @@ def redownload_download(
under the current format/naming rules — never silently reuses a shared cached file — and (b)
never deletes a file another job/user still shares: the old asset is only removed if THIS job
was its last holder."""
job = _own_job_for_update(db, user, job_id)
job = own_job_for_update(db, user, job_id)
if job.job_kind in NOT_REFETCHABLE:
# None has a re-fetchable source: an editor clip is derived locally, an upload IS the
# source, an ad-hoc clip is a range of a source the clip itself does not re-fetch.
@@ -823,82 +603,36 @@ def redownload_download(
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED):
raise refused(409, "still_downloading")
service.redownload(db, job)
return _serialize_job(db, job)
return serialize_job(db, job)
@router.post("/{job_id}/cancel")
def cancel_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
job = _own_job_for_update(db, user, job_id)
job = own_job_for_update(db, user, job_id)
service.cancel(db, job)
return _serialize_job(db, job)
return serialize_job(db, job)
@router.delete("/{job_id}")
def delete_download(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
job = _own_job_for_update(db, user, job_id)
service.delete(db, job)
return {"deleted": job_id}
"""The Files library's delete (`routes.files.delete_file`, audited) at its old address, until the
client moves to the new one."""
delete_file(job_id, user, db)
return {"ok": True, "deleted": job_id}
# --- file download (range-aware, custom display name) --------------------------------------
def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
job = db.get(DownloadJob, job_id)
if job is None:
raise not_found("download")
if job.user_id == user.id:
return job
shared = db.execute(
select(DownloadShare.id).where(
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
)
).first()
if shared is None:
raise not_found("download")
return job
@router.get("/{job_id}/file")
def download_file(
job_id: int,
request: Request,
user: User = Depends(current_user),
db: Session = Depends(get_db),
):
job = _accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
if path is None:
raise not_found("file")
filename = storage.download_filename(
job.display_name or asset.title or asset.source_ref, asset.container, path
)
asset.last_access_at = datetime.now(timezone.utc)
def download_file(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
response = serve_file(db, accessible_job(db, user, job_id))
db.commit()
# The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
# (storage.served_media_type), NEVER guessed from the file's own extension. A generic 'file' (and
# any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html")
# returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on
# this origin. Recognised media is served with its real type so the in-app player streams it. Both
# are range-aware (FileResponse → 206).
media_type = storage.served_media_type(asset.media_kind, asset.container)
if media_type is None:
return FileResponse(
path,
filename=filename,
media_type="application/octet-stream",
content_disposition_type="attachment",
)
return FileResponse(path, filename=filename, media_type=media_type)
return response
# --- editor filmstrip (scrub track) --------------------------------------------------------
@@ -913,7 +647,7 @@ def set_library_progress(
) -> dict:
"""Checkpoint the library player's resume point in a download the caller can play (their own, or
one shared with them). A position too early or too close to the end clears it instead."""
_accessible_job(db, user, job_id)
accessible_job(db, user, job_id)
position = kept_position(body)
mine = (LibraryState.user_id == user.id) & (LibraryState.job_id == job_id)
if not position:
@@ -945,7 +679,7 @@ def storyboard_meta(
Best-effort: for a very long/high-res source the sprite may not generate in time — then
`available` is False and the editor shows a plain timeline."""
job = _accessible_job(db, user, job_id)
job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
@@ -966,7 +700,7 @@ def storyboard_meta(
@router.get("/{job_id}/poster.jpg")
def poster_image(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
"""The locally-generated poster frame for a download whose source had no thumbnail."""
job = _accessible_job(db, user, job_id)
job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.poster_path:
raise not_found("poster")
@@ -980,7 +714,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session =
def storyboard_image(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
):
job = _accessible_job(db, user, job_id)
job = accessible_job(db, user, job_id)
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.storyboard_path:
raise not_found("filmstrip")
@@ -1236,19 +970,19 @@ def admin_list_downloads(
db.execute(
select(DownloadJob)
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
.limit(_MAX_LIST)
.limit(MAX_LIST)
)
.scalars()
.all()
)
assets = _assets_for(db, jobs)
assets = assets_for(db, jobs)
emails = {
u.id: u.email
for u in db.execute(select(User).where(User.id.in_({j.user_id for j in jobs}))).scalars()
}
out = []
for j in jobs:
data = _serialize(j, assets.get(j.asset_id))
data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None)
data["user_email"] = emails.get(j.user_id)
out.append(data)
return out
backend/app/routes/files.py
+691
View File
@@ -0,0 +1,691 @@
"""The Files library HTTP API — a user's files, the folders they sit in, and the file itself.
A file in the library is a user's DONE job (`DownloadJob`) over its shared `MediaAsset`: the job is
the per-user layer (name, folder, kept, expiry, shares and links all hang off it), the asset the
bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk.
In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above
the routes are shared with that module — serialization, the ownership locks, the upload spool —
because a download and a library file are one row at two moments.
"""
import base64
import hashlib
import os
import tempfile
from collections.abc import AsyncIterator, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import BinaryIO, Literal
from urllib.parse import unquote
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from sqlalchemy import func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app import audit, sysconfig
from app.audit import AuditAction
from app.auth import current_user
from app.collation import collated
from app.config import settings
from app.db import get_db, release_read_connection
from app.downloads import folders, quota, service, storage
from app.downloads import upload as uploadmod
from app.downloads.status import JobStatus
from app.mailtext import lang_for_user
from app.models import (
DISPLAY_TEXT_MAX,
DownloadJob,
DownloadShare,
FileFolder,
LibraryState,
MediaAsset,
User,
display_text,
)
from app.refusal import refused
from app.routes._common import capped_stream_to, not_found, owned_or_404
from app.utils import iso, like_escape
router = APIRouter(prefix="/api/files", tags=["files"])
# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's
# own cap; far beyond any real library).
MAX_LIST = 500
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
_MAX_RETENTION_DAYS = 3650
_HASH_CHUNK = 1 << 20
# --- shared with routes.downloads -------------------------------------------------------
def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
return {
"id": job.id,
"status": job.status,
"progress": job.progress,
"speed_bps": job.speed_bps,
"eta_s": job.eta_s,
"phase": job.phase,
"error": job.error,
"error_code": job.error_code,
"error_detail": job.error_detail,
"queue_pos": job.queue_pos,
"created_at": iso(job.created_at),
"source_kind": job.source_kind,
"source_ref": job.source_ref,
"source_url": service.reference_url(job, asset),
# A locally-generated poster to fall back to when the source had no thumbnail.
"poster_url": f"/api/downloads/{job.id}/poster.jpg"
if (asset is not None and asset.poster_path)
else None,
"profile_id": job.profile_id,
"spec": job.profile_snapshot,
"display_name": job.display_name,
"display_uploader": job.display_uploader,
"display_uploader_url": job.display_uploader_url,
"extra_links": job.extra_links or [],
"job_kind": job.job_kind,
"source_job_id": job.source_job_id,
"edit_spec": job.edit_spec,
"can_download": ready and job.status == JobStatus.DONE,
"expired": job.asset_id is None and job.status == JobStatus.DONE,
# The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
"folder_id": job.folder_id,
"kept": job.kept,
"expires_at": iso(job.expires_at),
"asset": None
if asset is None
else {
"id": asset.id,
"status": asset.status,
"title": asset.title,
"uploader": asset.uploader,
"upload_date": iso(asset.upload_date),
"duration_s": asset.duration_s,
"size_bytes": asset.size_bytes,
"width": asset.width,
"height": asset.height,
"container": asset.container,
# "video" | "audio" | "file" — the library gates play/edit on this (media only); a
# generic uploaded file is download-only. Null while still in flight (set at finalize).
"media_kind": asset.media_kind,
"uploader_url": asset.uploader_url,
"thumbnail_url": asset.thumbnail_url,
},
}
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
ids = {j.asset_id for j in jobs if j.asset_id}
if not ids:
return {}
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
return {a.id: a for a in rows}
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
ids = [j.id for j in jobs]
if not ids:
return {}
rows = db.execute(
select(LibraryState.job_id, LibraryState.position_seconds).where(
LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
)
).all()
return {job_id: pos for job_id, pos in rows}
def serialize_job(db: Session, job: DownloadJob) -> dict:
"""Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
every single-job handler shares."""
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
return serialize(job, asset)
def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
or writes the status those decide on (pause). The job is re-read under a row lock, so two
overlapping requests for one job run one after the other and the second decides on the first's
committed result — never both releasing the same hold, and never a pause turning a job a cancel
just released back into a holder; either can free a file another job still holds.
The asset is locked BEFORE the job: the order a finishing worker child takes them in
(`service.finalize_if_running`), so the two can never deadlock."""
while True:
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
if asset_id is not None:
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
job = db.execute(
select(DownloadJob)
.where(DownloadJob.id == job_id)
.with_for_update()
.execution_options(populate_existing=True)
).scalar_one_or_none()
if job is None:
raise not_found("download")
if job.asset_id == asset_id:
return job
db.rollback() # another request moved the job to another asset meanwhile: lock that one
def quota_detail(e: quota.QuotaExceeded) -> dict:
"""Structured error the trilingual client localizes (reason key + numbers), instead of a
pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current}
# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous
# caller could make the process write the whole per-file allowance to temp, per request (SEC-20
# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
# declared AFTER the auth one, is the only shape that closes it — the same one
# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
# every route with a raised ceiling rather than leaving it to a comment like this one.
#
# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
# Hungarian filename would otherwise be unsendable.
UPLOAD_FILENAME_HEADER = "x-upload-filename"
UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
# The folder the file goes in (its id); without it, the user's Uploads folder.
UPLOAD_FOLDER_HEADER = "x-upload-folder"
@dataclass(frozen=True)
class SpooledUpload:
"""What the body dependency hands the handler: the bytes already on disk and their hash, plus
the names and the folder that came alongside them."""
path: Path
size: int
sha256: str
user_id: int
filename: str
display_name: str | None
folder_id: int | None
def _header_text(request: Request, name: str) -> str:
"""One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
own fallback names the file."""
raw = request.headers.get(name, "")
try:
return unquote(raw, errors="strict").strip()
except (UnicodeDecodeError, ValueError):
# silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
# cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
# a label; logging it would hand any client a free log line per request.
return ""
def _upload_folder(db: Session, user_id: int, raw: str) -> int | None:
"""The folder id an upload asked for (`UPLOAD_FOLDER_HEADER`), checked; None when it asked for
none."""
if not raw.strip():
return None
try:
return folders.upload_target(db, user_id, int(raw)).id
except ValueError as e:
raise refused(409, "file_folder_gone") from e
except folders.FolderError as e:
raise refused(409, e.reason) from e
class _HashingSink:
"""The upload's file, hashing the bytes on their way in: the hash costs no second read."""
def __init__(self, out: BinaryIO) -> None:
self.out = out
self.hash = hashlib.sha256()
def write(self, data: bytes, /) -> int:
self.hash.update(data)
return self.out.write(data)
async def spool_upload(
request: Request,
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> AsyncIterator[SpooledUpload]:
"""Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
`user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so
naming `current_user` here makes the auth check a precondition of reading the body at all — a
stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest
MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit,
empty, quota, probe or move error, a client that vanished mid-upload — it is removed."""
cap = sysconfig.get_int(db, "download_upload_max_bytes")
user_id = user.id
# A folder the file may not go in is refused before the transfer, not after it.
folder_id = _upload_folder(db, user_id, request.headers.get(UPLOAD_FOLDER_HEADER, ""))
# Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
# for why, and for the precondition it enforces. Everything this dependency still needs is
# already an int or a header in hand: touching an ORM attribute below would reload `user` and
# check a connection straight back out.
release_read_connection(db)
staging = Path(settings.files_root) / ".staging" / "uploads"
staging.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(staging))
tmp_path = Path(tmp)
try:
with os.fdopen(fd, "wb") as out:
sink = _HashingSink(out)
size = await capped_stream_to(
request,
sink,
cap,
HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
)
yield SpooledUpload(
path=tmp_path,
size=size,
sha256=sink.hash.hexdigest(),
user_id=user_id,
filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
folder_id=folder_id,
)
finally:
tmp_path.unlink(missing_ok=True)
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
"""A file the caller may read: their own, or one shared with them."""
job = db.get(DownloadJob, job_id)
if job is None:
raise not_found("download")
if job.user_id == user.id:
return job
shared = db.execute(
select(DownloadShare.id).where(
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
)
).first()
if shared is None:
raise not_found("download")
return job
# --- folders -------------------------------------------------------------------------------
def _serialize_folder(folder: FileFolder, count: int, size: int) -> dict:
return {
"id": folder.id,
"kind": folder.kind,
# A source folder's name is the client's to render from its key, in the reader's language.
"source_key": folder.source_key,
"name": folder.name,
"retention_days": folder.retention_days,
"count": count,
"size_bytes": size,
"created_at": iso(folder.created_at),
}
def _own_folder_or_404(db: Session, user: User, folder_id: int) -> FileFolder:
return owned_or_404(db, FileFolder, folder_id, user, what="folder")
@router.get("/folders")
def list_folders(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
"""The caller's folders with how many files each holds and their size. A source folder its
files have all left stays (count 0): the client decides whether to show it."""
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
rows = db.execute(
select(
FileFolder,
func.count(DownloadJob.id),
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
)
.outerjoin(DownloadJob, done)
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(FileFolder.user_id == user.id)
.group_by(FileFolder.id)
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
).all()
return [_serialize_folder(f, count, int(size)) for f, count, size in rows]
class FolderIn(BaseModel):
name: str = Field(min_length=1, max_length=DISPLAY_TEXT_MAX)
def _folder_name(raw: str) -> str:
name = display_text(raw)
if not name:
raise refused(400, "file_folder_name_required")
return name
@router.post("/folders")
def create_folder(
body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name))
try:
with db.begin_nested():
db.add(folder)
db.flush()
except IntegrityError as e: # the unique (user, lower(name)): one folder per name
raise refused(409, "file_folder_name_taken") from e
audit.record(
db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id
)
db.commit()
return _serialize_folder(folder, 0, 0)
class UpdateFolderIn(BaseModel):
# Presence (model_fields_set) decides what is touched; retention_days null = keep forever.
name: str | None = Field(default=None, min_length=1, max_length=DISPLAY_TEXT_MAX)
retention_days: int | None = Field(default=None, ge=1, le=_MAX_RETENTION_DAYS)
@router.patch("/folders/{folder_id}")
def update_folder(
folder_id: int,
body: UpdateFolderIn,
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
"""Rename one of the caller's own folders (a source folder's name is its origin's), and/or set
how long any of their folders keeps a file (`folders.set_retention`)."""
folder = _own_folder_or_404(db, user, folder_id)
sent = body.model_fields_set
if "name" in sent:
if folder.kind != folders.FolderKind.OWN:
raise refused(409, "file_folder_not_own")
folder.name = _folder_name(body.name or "")
try:
with db.begin_nested():
db.flush()
except IntegrityError as e:
raise refused(409, "file_folder_name_taken") from e
audit.record(
db,
user.id,
AuditAction.FILE_FOLDER_RENAME,
target_type="file_folder",
target_id=folder.id,
)
if "retention_days" in sent and body.retention_days != folder.retention_days:
before = folder.retention_days
folders.set_retention(db, folder, body.retention_days)
audit.record(
db,
user.id,
AuditAction.FILE_FOLDER_RETENTION,
target_type="file_folder",
target_id=folder.id,
before={"retention_days": before},
after={"retention_days": body.retention_days},
)
db.commit()
count, size = _folder_totals(db, folder.id)
return _serialize_folder(folder, count, size)
def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
count, size = db.execute(
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
.select_from(DownloadJob)
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
).one()
return count, int(size)
@router.delete("/folders/{folder_id}")
def delete_folder(
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
"""Delete one of the caller's own folders; its files go back to their source folders."""
folder = _own_folder_or_404(db, user, folder_id)
try:
moved = folders.delete_own(db, folder)
except folders.FolderError as e:
raise refused(409, e.reason) from e
audit.record(
db,
user.id,
AuditAction.FILE_FOLDER_DELETE,
target_type="file_folder",
target_id=folder_id,
after={"files_moved": moved},
)
db.commit()
return {"ok": True, "deleted": folder_id, "files_moved": moved}
# --- files ---------------------------------------------------------------------------------
@router.get("")
def list_files(
folder_id: int | None = None,
q: str = Query("", max_length=200),
sort: Literal["added", "name", "size"] = "added",
order: Literal["asc", "desc"] = "desc",
scope: Literal["mine", "shared"] = "mine",
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> list[dict]:
"""The caller's done files (`scope=mine`, optionally one folder's), or the ones shared with
them (`scope=shared`); searched by name and channel, sorted by when they were added, by name
(in the reader's alphabet) or by size."""
stmt = (
select(DownloadJob)
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
.where(DownloadJob.status == JobStatus.DONE)
)
if scope == "shared":
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
DownloadShare.to_user_id == user.id
)
else:
stmt = stmt.where(DownloadJob.user_id == user.id)
if folder_id is not None:
stmt = stmt.where(DownloadJob.folder_id == folder_id)
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
term = q.strip()
if term:
like = f"%{like_escape(term)}%"
stmt = stmt.where(
or_(
name.ilike(like),
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
)
)
key = {
"added": DownloadJob.created_at,
"name": collated(name, lang_for_user(user)),
"size": MediaAsset.size_bytes,
}[sort]
first = key.asc() if order == "asc" else key.desc()
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all()
assets = assets_for(db, jobs)
positions = positions_for(db, user, jobs)
shared = scope == "shared"
return [
{
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
"position_seconds": positions.get(j.id, 0),
"shared": shared,
}
for j in jobs
]
class UpdateFileIn(BaseModel):
# Presence (model_fields_set) decides what is touched; a null/"" name falls back to the title.
name: str | None = None
folder_id: int | None = None
kept: bool | None = None
@router.patch("/{job_id}")
def update_file(
job_id: int,
body: UpdateFileIn,
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
"""Rename a file (its display name; the file on disk keeps its own), move it to another folder
(`folders.move`), or keep it until deleted / hand it back to its folder's retention."""
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
sent = body.model_fields_set
if "name" in sent:
job.display_name = display_text(body.name)
if "folder_id" in sent:
if body.folder_id is None:
raise refused(400, "file_folder_gone")
folder = db.get(FileFolder, body.folder_id)
if folder is None or folder.user_id != user.id:
raise refused(409, "file_folder_gone")
before = job.folder_id
try:
folders.move(db, job, folder)
except folders.FolderError as e:
raise refused(409, e.reason) from e
if job.folder_id != before:
audit.record(
db,
user.id,
AuditAction.FILE_MOVE,
target_type="file",
target_id=job.id,
before={"folder_id": before},
after={"folder_id": job.folder_id},
)
if "kept" in sent and body.kept is not None and body.kept != job.kept:
folders.set_kept(db, job, body.kept)
action = AuditAction.FILE_KEEP if body.kept else AuditAction.FILE_UNKEEP
audit.record(db, user.id, action, target_type="file", target_id=job.id)
db.commit()
return serialize_job(db, job)
@router.delete("/{job_id}")
def delete_file(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
"""Delete a file from the caller's library; the bytes go with their last holder."""
job = own_job_for_update(db, user, job_id)
audit.record(db, user.id, AuditAction.FILE_DELETE, target_type="file", target_id=job_id)
service.delete(db, job) # commits, the audit row with it
return {"ok": True, "deleted": job_id}
@router.post("/upload")
def upload_file(
upload: SpooledUpload = Depends(spool_upload),
db: Session = Depends(get_db),
) -> dict:
"""Put a local file in the caller's library — in the folder `UPLOAD_FOLDER_HEADER` names, else
their Uploads folder. Any file type; media gets the player, a generic file is download-only.
The body IS the file (see the header note above `UPLOAD_FILENAME_HEADER`); `spool_upload` has
authenticated the caller, checked the folder and streamed it to disk already.
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
plus sync DB writes — so it must NOT run on the event loop."""
return serialize_job(db, ingest_spooled(db, upload))
def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
"""The spooled upload into the library (`upload.ingest_upload`, which commits its own rows),
its refusals answered as HTTP."""
if upload.size == 0:
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
try:
job = uploadmod.ingest_upload(
db,
upload.user_id,
filename=upload.filename,
src_path=upload.path,
size=upload.size,
display_name=upload.display_name,
sha256=upload.sha256,
folder_id=upload.folder_id,
)
except quota.QuotaExceeded as e:
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
except folders.FolderError as e: # the folder went while the file was on its way
raise refused(409, e.reason) from e
return job
def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
if path is None:
raise not_found("file")
return asset, path
def serve_file(db: Session, job: DownloadJob) -> FileResponse:
"""The file of `job` (the caller may read it), range-aware, named by the user's display name.
The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
(`storage.served_media_type`), NEVER guessed from the file's own extension. A generic 'file' (and
any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html")
returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on
this origin. Recognised media is served with its real type so the in-app player streams it. Both
are range-aware (FileResponse → 206). A file whose hash is known says so (`Repr-Digest`, RFC
9530: the digest of the whole file, whichever range is asked for)."""
asset, path = _ready_file(db, job)
filename = storage.download_filename(
job.display_name or asset.title or asset.source_ref, asset.container, path
)
headers = {}
if asset.sha256:
digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode()
headers["Repr-Digest"] = f"sha-256=:{digest}:"
asset.last_access_at = datetime.now(timezone.utc) # the handler commits it
media_type = storage.served_media_type(asset.media_kind, asset.container)
if media_type is None:
return FileResponse(
path,
filename=filename,
media_type="application/octet-stream",
content_disposition_type="attachment",
headers=headers,
)
return FileResponse(path, filename=filename, media_type=media_type, headers=headers)
@router.get("/{job_id}/file")
def get_file(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
response = serve_file(db, accessible_job(db, user, job_id))
db.commit()
return response
@router.get("/{job_id}/sha256")
def get_sha256(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
"""The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder.
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool."""
asset, path = _ready_file(db, accessible_job(db, user, job_id))
if not asset.sha256:
h = hashlib.sha256()
with path.open("rb") as f:
while chunk := f.read(_HASH_CHUNK):
h.update(chunk)
asset.sha256 = h.hexdigest()
db.commit()
return {"sha256": asset.sha256}
backend/tests/conftest.py
+6 -2
View File
@@ -254,20 +254,24 @@ def as_user(db, override_dep, _configured):
@pytest.fixture
def upload_headers():
"""Build the headers `POST /api/downloads/upload` takes (the file is the raw body).
"""Build the headers the upload routes take (the file is the raw body).
Shared, because two test modules drive that route and a second copy of this is exactly the drift
that lets the wire shape change in one place only. The body IS the file now; the names ride as
percent-encoded headers (`quote` mirrors the client's `encodeURIComponent`, since headers are
latin-1 and a Hungarian filename would not survive raw)."""
def _headers(filename: str, display_name: str | None = None) -> dict[str, str]:
def _headers(
filename: str, display_name: str | None = None, folder: int | str | None = None
) -> dict[str, str]:
headers = {
"content-type": "application/octet-stream",
"x-upload-filename": quote(filename),
}
if display_name is not None:
headers["x-upload-display-name"] = quote(display_name)
if folder is not None:
headers["x-upload-folder"] = str(folder)
return headers
return _headers
backend/tests/test_body_limit.py
+3 -3
View File
@@ -221,7 +221,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes():
CAPPED_BODY_SITES = {
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
("downloads.py", "spool_upload", "capped_stream_to", "cap"),
("files.py", "spool_upload", "capped_stream_to", "cap"),
}
@@ -467,7 +467,7 @@ def test_every_body_reading_site_hands_its_pooled_connection_back_first():
assert not missing, (
"a route reads its own request body while still inside its read transaction, pinning a "
f"pooled connection for the whole transfer: {missing}. Call "
"db.release_read_connection(db) before the await — see routes/downloads.py:spool_upload."
"db.release_read_connection(db) before the await — see routes/files.py:spool_upload."
)
@@ -540,7 +540,7 @@ def test_no_route_with_a_raised_ceiling_declares_a_body_parameter(method, templa
"its dependencies, so FastAPI reads it before the first dependency runs and the raised "
"ceiling is all that stands between an anonymous caller and the whole allowance. Read the "
"body yourself, from a dependency declared AFTER the auth one — see "
"routes/downloads.py:spool_upload."
"routes/files.py:spool_upload."
)
backend/tests/test_download_job_locks.py
+5 -8
View File
@@ -5,7 +5,7 @@ unlocked read: two overlapping requests for one job (a double-click, a second ta
gateway timeout) both saw it holding its asset and both released the hold. The count then reaches 0
while another job still holds the asset, and GC or the ready-file delete removes it under that job.
A pause racing a cancel did the same one step later: its unlocked write turned the released job back
into a holder, which a later delete releases again. `_own_job_for_update` makes the second request
into a holder, which a later delete releases again. `own_job_for_update` makes the second request
wait and decide on the first one's committed result.
"""
@@ -18,12 +18,8 @@ from sqlalchemy import select
from app.db import SessionLocal
from app.downloads import service
from app.models import DownloadJob, MediaAsset, User
from app.routes.downloads import (
_own_job_for_update,
cancel_download,
delete_download,
pause_download,
)
from app.routes.downloads import cancel_download, delete_download, pause_download
from app.routes.files import delete_file, own_job_for_update
_REF = "https://example.com/video"
@@ -59,6 +55,7 @@ def _asset_held_by_two_jobs(db, status="error"):
[
("error", service.cancel, cancel_download, "canceled"),
("error", service.delete, delete_download, None),
("error", service.delete, delete_file, None),
("queued", service.cancel, pause_download, "canceled"),
],
)
@@ -80,7 +77,7 @@ def test_overlapping_requests_for_one_job_release_its_hold_once(
finished.set()
try:
locked = _own_job_for_update(first, user, job_id)
locked = own_job_for_update(first, user, job_id)
other = threading.Thread(target=the_other_request)
other.start()
# The second request waits for the first one's locks instead of deciding on the same row.
backend/tests/test_download_limits.py
+1 -1
View File
@@ -188,7 +188,7 @@ def test_list_downloads_is_capped(db, monkeypatch):
_job(db, u.id, queue_pos=i, status="done")
db.commit()
monkeypatch.setattr(dl, "_MAX_LIST", 2)
monkeypatch.setattr(dl, "MAX_LIST", 2)
rows = dl.list_downloads(user=u, db=db)
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
backend/tests/test_files_api.py
+432
View File
@@ -0,0 +1,432 @@
"""The Files library API (`routes.files`), DB-lane: folders (list, own create/rename/delete, a
folder's retention), the file list (done only, by folder, searched, sorted, shared with me), a
file's rename / move / keep / delete, the upload into a folder, and the file's hash.
Every route is also held to the caller's own rows: another user's folder or file answers like a
missing one.
"""
import base64
import hashlib
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import select
from app import sysconfig
from app.config import settings
from app.downloads import folders, service
from app.downloads import upload as uploadmod
from app.downloads.status import JobStatus
from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset
NOW = datetime.now(timezone.utc)
@pytest.fixture
def _root(tmp_path, monkeypatch):
monkeypatch.setattr(settings, "files_root", str(tmp_path))
return tmp_path
def _file(db, user_id, *, kind="youtube", ref=None, title="A title", size=10, rel=None, **kw):
"""A done library file: an asset with bytes on disk-path terms, and the user's job over it."""
n = db.execute(select(MediaAsset.id)).all()
ref = ref or f"vid{len(n):08d}"
asset = MediaAsset(
source_kind=kind,
source_ref=ref,
format_sig=f"sig-{ref}",
status="ready",
rel_path=rel or f"{ref}.mp4",
size_bytes=size,
ref_count=0,
title=title,
media_kind="video",
container="mp4",
)
db.add(asset)
db.flush()
job = DownloadJob(
user_id=user_id,
asset_id=asset.id,
source_kind=kind,
source_ref=ref,
profile_snapshot={},
status=kw.pop("status", JobStatus.DONE),
**kw,
)
db.add(job)
db.flush()
folders.place(db, job)
folders.stamp_expiry(db, job)
service.incr_ref_count(db, asset.id)
db.commit()
return job
def _own(db, user_id, name="Saját"):
folder = FileFolder(user_id=user_id, kind=folders.FolderKind.OWN, name=name)
db.add(folder)
db.commit()
return folder
def _audits(db, action):
return db.execute(select(AuditLog).where(AuditLog.action == action)).scalars().all()
# --- folders -------------------------------------------------------------------------------
def test_the_folder_list_counts_each_folders_done_files_and_is_the_callers_own(
db, make_user, as_user
):
alice, bob = make_user(), make_user()
_file(db, alice.id, size=10)
_file(db, alice.id, size=5)
_file(db, alice.id, kind="url", ref="https://vimeo.com/1", size=7)
_file(db, alice.id, status=JobStatus.RUNNING) # not in the library yet
_file(db, bob.id)
got = {
(f["kind"], f["source_key"]): (f["count"], f["size_bytes"])
for f in as_user(alice).get("/api/files/folders").json()
}
assert got == {("source", "youtube"): (2, 15), ("source", "site:vimeo.com"): (1, 7)}
def test_an_own_folder_is_made_once_per_name_whatever_its_case(db, make_user, as_user):
alice = make_user()
c = as_user(alice)
made = c.post("/api/files/folders", json={"name": "Nyaralás"})
assert made.status_code == 200, made.text
assert (made.json()["kind"], made.json()["name"]) == ("own", "Nyaralás")
again = c.post("/api/files/folders", json={"name": "NYARALÁS"})
assert again.status_code == 409
assert again.json()["detail"]["reason"] == "file_folder_name_taken"
blank = c.post("/api/files/folders", json={"name": " "})
assert blank.json()["detail"]["reason"] == "file_folder_name_required"
# Another user may have a folder of the same name.
assert as_user(make_user()).post("/api/files/folders", json={"name": "Nyaralás"}).is_success
assert len(_audits(db, "file_folder_create")) == 2
def test_only_an_own_folder_is_renamed_and_only_by_its_user(db, make_user, as_user):
alice = make_user()
c = as_user(alice)
src = db.get(FileFolder, _file(db, alice.id).folder_id)
own = _own(db, alice.id, "Régi")
taken = _own(db, alice.id, "Foglalt")
ok = c.patch(f"/api/files/folders/{own.id}", json={"name": "Új"})
assert ok.json()["name"] == "Új"
clash = c.patch(f"/api/files/folders/{own.id}", json={"name": taken.name.lower()})
assert clash.json()["detail"]["reason"] == "file_folder_name_taken"
no = c.patch(f"/api/files/folders/{src.id}", json={"name": "YouTube 2"})
assert no.status_code == 409 and no.json()["detail"]["reason"] == "file_folder_not_own"
stranger = as_user(make_user()).patch(f"/api/files/folders/{own.id}", json={"name": "X"})
assert stranger.status_code == 404
db.refresh(own)
assert own.name == "Új"
def test_a_shorter_retention_moves_each_files_expiry_but_never_inside_the_warning_window(
db, make_user, as_user
):
sysconfig.set_value(db, "download_gc_grace_days", 3)
db.commit()
alice = make_user()
fresh = _file(db, alice.id) # expires in 30 days
old = _file(db, alice.id)
old.expires_at = NOW + timedelta(days=10) # 20 days old already
kept = _file(db, alice.id, kept=True)
db.commit()
folder_id = fresh.folder_id
r = as_user(alice).patch(f"/api/files/folders/{folder_id}", json={"retention_days": 20})
assert r.json()["retention_days"] == 20
for j in (fresh, old, kept):
db.refresh(j)
# 30 → 20 days: each loses 10 — the one with 10 left would be due now, so it gets the window.
assert abs(fresh.expires_at - (NOW + timedelta(days=20))) < timedelta(minutes=5)
assert abs(old.expires_at - (NOW + timedelta(days=3))) < timedelta(minutes=5)
assert kept.expires_at is None
forever = as_user(alice).patch(f"/api/files/folders/{folder_id}", json={"retention_days": None})
assert forever.json()["retention_days"] is None
db.refresh(fresh)
assert fresh.expires_at is None
rows = _audits(db, "file_folder_retention")
assert [(a.before, a.after) for a in rows] == [
({"retention_days": 30}, {"retention_days": 20}),
({"retention_days": 20}, {"retention_days": None}),
]
def test_a_retention_set_on_a_folder_that_kept_forever_starts_its_files_clock_now(
db, make_user, as_user
):
alice = make_user()
up = _file(db, alice.id, kind="upload", ref="u1")
assert up.expires_at is None
r = as_user(alice).patch(f"/api/files/folders/{up.folder_id}", json={"retention_days": 7})
assert r.status_code == 200, r.text
db.refresh(up)
assert abs(up.expires_at - (NOW + timedelta(days=7))) < timedelta(minutes=5)
def test_deleting_an_own_folder_sends_its_files_home_and_a_source_folder_cannot_be_deleted(
db, make_user, as_user
):
alice = make_user()
c = as_user(alice)
yt = _file(db, alice.id)
site = _file(db, alice.id, kind="url", ref="https://vimeo.com/2")
home = {yt.id: yt.folder_id, site.id: site.folder_id}
own = _own(db, alice.id)
for j in (yt, site):
assert c.patch(f"/api/files/{j.id}", json={"folder_id": own.id}).is_success
r = c.delete(f"/api/files/folders/{own.id}")
assert r.json() == {"ok": True, "deleted": own.id, "files_moved": 2}
assert db.get(FileFolder, own.id) is None
for j in (yt, site):
db.refresh(j)
assert j.folder_id == home[j.id]
# Back in a source folder, its clock runs again from now.
assert abs(j.expires_at - (NOW + timedelta(days=30))) < timedelta(minutes=5)
src = c.delete(f"/api/files/folders/{yt.folder_id}")
assert src.status_code == 409 and src.json()["detail"]["reason"] == "file_folder_not_own"
other = _own(db, alice.id, "Más")
assert as_user(make_user()).delete(f"/api/files/folders/{other.id}").status_code == 404
assert len(_audits(db, "file_folder_delete")) == 1
# --- the file list -------------------------------------------------------------------------
def test_the_list_holds_the_callers_done_files_by_folder(db, make_user, as_user):
alice, bob = make_user(), make_user()
yt = _file(db, alice.id)
site = _file(db, alice.id, kind="url", ref="https://vimeo.com/3")
_file(db, alice.id, status=JobStatus.QUEUED)
_file(db, bob.id)
c = as_user(alice)
assert {f["id"] for f in c.get("/api/files").json()} == {yt.id, site.id}
only = c.get("/api/files", params={"folder_id": site.folder_id}).json()
assert [f["id"] for f in only] == [site.id]
assert only[0]["folder_id"] == site.folder_id and only[0]["shared"] is False
# Bob's folder id names nothing of Alice's.
bobs = db.execute(select(FileFolder.id).where(FileFolder.user_id == bob.id)).scalar()
assert c.get("/api/files", params={"folder_id": bobs}).json() == []
def test_the_list_is_searched_by_name_and_channel_with_wildcards_taken_literally(
db, make_user, as_user
):
alice = make_user()
hit = _file(db, alice.id, title="Balaton 100% nyár")
by_channel = _file(db, alice.id, title="Más", display_uploader="Balatoni csatorna")
_file(db, alice.id, title="Balaton 1000 nyár")
c = as_user(alice)
assert [f["id"] for f in c.get("/api/files", params={"q": "100%"}).json()] == [hit.id]
found = {f["id"] for f in c.get("/api/files", params={"q": "balatoni"}).json()}
assert found == {by_channel.id}
def test_the_list_sorts_by_name_size_or_when_added(db, make_user, as_user):
alice = make_user()
b = _file(db, alice.id, title="Béka", size=30)
a = _file(db, alice.id, title="alma", size=10)
z = _file(db, alice.id, title="Zebra", size=20)
c = as_user(alice)
def ids(**params):
return [f["id"] for f in c.get("/api/files", params=params).json()]
assert ids() == [z.id, a.id, b.id] # newest first
assert ids(sort="name", order="asc") == [a.id, b.id, z.id] # the reader's alphabet, any case
assert ids(sort="size", order="desc") == [b.id, z.id, a.id]
def test_shared_with_me_lists_what_others_shared_and_nothing_else(db, make_user, as_user):
alice, bob, carol = make_user(), make_user(), make_user()
theirs = _file(db, bob.id)
_file(db, bob.id) # not shared
for_carol = _file(db, bob.id)
db.add(DownloadShare(job_id=theirs.id, from_user_id=bob.id, to_user_id=alice.id))
db.add(DownloadShare(job_id=for_carol.id, from_user_id=bob.id, to_user_id=carol.id))
db.commit()
got = as_user(alice).get("/api/files", params={"scope": "shared"}).json()
assert [(f["id"], f["shared"]) for f in got] == [(theirs.id, True)]
# --- one file ------------------------------------------------------------------------------
def test_a_file_is_renamed_kept_and_given_back_to_its_folders_retention(db, make_user, as_user):
alice = make_user()
job = _file(db, alice.id)
c = as_user(alice)
assert c.patch(f"/api/files/{job.id}", json={"name": "Új név"}).json()["display_name"] == (
"Új név"
)
kept = c.patch(f"/api/files/{job.id}", json={"kept": True}).json()
assert (kept["kept"], kept["expires_at"]) == (True, None)
back = c.patch(f"/api/files/{job.id}", json={"kept": False}).json()
assert back["kept"] is False and back["expires_at"] is not None
# Setting what it already is records nothing.
assert c.patch(f"/api/files/{job.id}", json={"kept": False}).status_code == 200
assert len(_audits(db, "file_keep")) == 1 and len(_audits(db, "file_unkeep")) == 1
assert as_user(make_user()).patch(f"/api/files/{job.id}", json={"kept": True}).status_code == (
404
)
def test_a_file_moves_to_an_own_folder_or_home_and_nowhere_else(db, make_user, as_user):
alice, bob = make_user(), make_user()
job = _file(db, alice.id)
home = job.folder_id
vimeo = db.get(FileFolder, _file(db, alice.id, kind="url", ref="https://vimeo.com/4").folder_id)
own = _own(db, alice.id)
bobs = _own(db, bob.id, "Bobé")
c = as_user(alice)
moved = c.patch(f"/api/files/{job.id}", json={"folder_id": own.id}).json()
assert (moved["folder_id"], moved["expires_at"]) == (own.id, None) # an own folder keeps it
wrong = c.patch(f"/api/files/{job.id}", json={"folder_id": vimeo.id})
assert wrong.json()["detail"]["reason"] == "file_folder_not_its_source"
foreign = c.patch(f"/api/files/{job.id}", json={"folder_id": bobs.id})
assert foreign.json()["detail"]["reason"] == "file_folder_gone"
home_again = c.patch(f"/api/files/{job.id}", json={"folder_id": home}).json()
assert home_again["folder_id"] == home and home_again["expires_at"] is not None
assert [(a.before, a.after) for a in _audits(db, "file_move")] == [
({"folder_id": home}, {"folder_id": own.id}),
({"folder_id": own.id}, {"folder_id": home}),
]
def test_a_deleted_file_leaves_the_library_and_the_log_says_so(db, make_user, as_user, _root):
alice = make_user()
job = _file(db, alice.id)
assert as_user(make_user()).delete(f"/api/files/{job.id}").status_code == 404
r = as_user(alice).delete(f"/api/files/{job.id}")
assert r.json() == {"ok": True, "deleted": job.id}
db.expire_all()
assert db.get(DownloadJob, job.id) is None
[row] = _audits(db, "file_delete")
assert (row.actor_id, row.target_id) == (alice.id, str(job.id))
# --- upload, the file, its hash ------------------------------------------------------------
@pytest.fixture
def _no_probe(monkeypatch):
# A file ffprobe cannot read: a generic file, done at once (no conversion).
monkeypatch.setattr(uploadmod.probe, "probe_json", lambda _p: None)
def test_an_upload_lands_in_the_folder_it_asked_for_with_its_hash(
db, make_user, as_user, _root, _no_probe, upload_headers
):
alice = make_user()
own = _own(db, alice.id)
body = b"hello files"
c = as_user(alice)
r = c.post("/api/files/upload", content=body, headers=upload_headers("a.txt", folder=own.id))
assert r.status_code == 200, r.text
job = db.get(DownloadJob, r.json()["id"])
assert job.folder_id == own.id and job.expires_at is None
asset = db.get(MediaAsset, job.asset_id)
assert asset.sha256 == hashlib.sha256(body).hexdigest()
plain = c.post("/api/files/upload", content=b"x", headers=upload_headers("b.txt")).json()
assert db.get(FileFolder, plain["folder_id"]).source_key == "upload"
assert len(_audits(db, "file_upload")) == 2
@pytest.mark.parametrize(
("which", "reason"),
[
("site", "file_folder_not_its_source"), # YouTube's folder: no upload came from there
("foreign", "file_folder_gone"),
("garbage", "file_folder_gone"),
],
)
def test_an_upload_to_a_folder_it_may_not_use_is_refused_before_its_body_is_read(
db, make_user, as_user, _root, _no_probe, upload_headers, monkeypatch, which, reason
):
from app.routes import files as files_routes
alice, bob = make_user(), make_user()
folder = {
"site": lambda: _file(db, alice.id).folder_id,
"foreign": lambda: _own(db, bob.id).id,
"garbage": lambda: "12abc",
}[which]()
streamed = []
async def spy(*args, **kwargs):
streamed.append(True)
raise AssertionError("the body was read")
monkeypatch.setattr(files_routes, "capped_stream_to", spy)
before = db.execute(select(DownloadJob.id)).all()
r = as_user(alice).post(
"/api/files/upload", content=b"data", headers=upload_headers("a.txt", folder=folder)
)
assert (r.status_code, r.json()["detail"]["reason"]) == (409, reason)
assert streamed == []
assert db.execute(select(DownloadJob.id)).all() == before
def test_a_folder_deleted_while_its_upload_streams_refuses_the_upload(
db, make_user, as_user, _root, _no_probe, upload_headers, monkeypatch
):
from app.routes import files as files_routes
alice = make_user()
own = _own(db, alice.id)
real = files_routes.capped_stream_to
async def deleting_meanwhile(*args, **kwargs):
size = await real(*args, **kwargs)
db.delete(db.get(FileFolder, own.id))
db.commit()
return size
monkeypatch.setattr(files_routes, "capped_stream_to", deleting_meanwhile)
r = as_user(alice).post(
"/api/files/upload", content=b"data", headers=upload_headers("a.txt", folder=own.id)
)
assert (r.status_code, r.json()["detail"]["reason"]) == (409, "file_folder_gone")
assert db.execute(select(DownloadJob.id).where(DownloadJob.user_id == alice.id)).all() == []
assert not (_root / ".uploads").exists() # checked before the file was placed
def test_the_file_carries_its_hash_once_known_and_the_hash_is_taken_on_first_ask(
db, make_user, as_user, _root
):
alice, friend = make_user(), make_user()
body = b"\x00\x01video bytes"
job = _file(db, alice.id, rel="v.mp4")
(_root / "v.mp4").write_bytes(body)
c = as_user(alice)
first = c.get(f"/api/files/{job.id}/file")
assert first.status_code == 200 and first.content == body
assert "repr-digest" not in first.headers # not hashed yet: nothing claimed
want = hashlib.sha256(body).hexdigest()
assert c.get(f"/api/files/{job.id}/sha256").json() == {"sha256": want}
assert db.get(MediaAsset, job.asset_id).sha256 == want
ranged = c.get(f"/api/files/{job.id}/file", headers={"Range": "bytes=0-3"})
assert ranged.status_code == 206
digest = base64.b64encode(bytes.fromhex(want)).decode()
assert ranged.headers["repr-digest"] == f"sha-256=:{digest}:"
# A friend it was shared with may read it too; a stranger is told it does not exist.
db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id))
db.commit()
assert as_user(friend).get(f"/api/files/{job.id}/sha256").json() == {"sha256": want}
assert as_user(make_user()).get(f"/api/files/{job.id}/sha256").status_code == 404
def test_a_new_file_at_an_assets_path_forgets_the_old_hash(db, make_user):
job = _file(db, make_user().id)
asset = db.get(MediaAsset, job.asset_id)
asset.sha256 = "ab" * 32
asset.rel_path = "converted.mp4"
assert asset.sha256 is None
backend/tests/test_migration_0103.py
+62
View File
@@ -0,0 +1,62 @@
"""Migration 0103's data step, DB-lane: the fileless "expired" rows of the old retention go, every
other job stays.
An expired file now leaves the library (`downloads.gc`); before, it stayed as a done job with no
asset. Left in place, those rows would sit unseen in no listing forever.
"""
from alembic import command
from alembic.config import Config
from sqlalchemy import text
from app.db import engine
def test_the_fileless_expired_rows_go_and_every_other_job_stays(db, make_user):
uid = make_user().id
cfg = Config("alembic.ini")
command.downgrade(cfg, "0102_file_folders")
try:
with engine.begin() as conn:
folder = conn.execute(
text(
"INSERT INTO file_folders (user_id, kind, source_key, retention_days)"
" VALUES (:u, 'source', 'youtube', 30) RETURNING id"
),
{"u": uid},
).scalar_one()
asset = conn.execute(
text(
"INSERT INTO media_assets (source_kind, source_ref, format_sig, status, ref_count)"
" VALUES ('youtube', 'abcdefghijk', 'sig', 'ready', 1) RETURNING id"
)
).scalar_one()
def job(status, asset_id):
return conn.execute(
text(
"INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref,"
" profile_snapshot, status, folder_id) VALUES (:u, :a, 'youtube',"
" 'abcdefghijk', '{}', :s, :f) RETURNING id"
),
{"u": uid, "a": asset_id, "s": status, "f": folder},
).scalar_one()
expired = job("done", None)
alive = job("done", asset)
failed = job("error", None) # a failed job has no file either, and is not expired
queued = job("queued", None)
finally:
command.upgrade(cfg, "head")
with engine.connect() as conn:
left = set(conn.execute(text("SELECT id FROM download_jobs")).scalars())
has_hash = conn.execute(
text(
"SELECT count(*) FROM information_schema.columns"
" WHERE table_name = 'media_assets' AND column_name = 'sha256'"
)
).scalar()
assert left == {alive, failed, queued}
assert expired not in left
assert has_hash == 1
backend/tests/test_route_conventions.py
+3 -1
View File
@@ -46,7 +46,9 @@ SERIALIZERS = {
"_collection_card",
"_playlist_card",
"_serialize",
"_serialize_job",
"_serialize_folder",
"serialize",
"serialize_job",
"_serialize_profile",
"_serialize_user",
"_summary",
backend/tests/test_upload.py
+9 -8
View File
@@ -584,7 +584,7 @@ def _display_write_sites() -> dict[tuple[str, str], set[str]]:
# one value that needs no bound, and a clear is still a write.
# service.enqueue / enqueue_edit / `display_name` is already bounded by its caller; the fallback
# enqueue_clip slices the source's title to the same width.
# downloads.upload_download passes through `spool_upload`'s value, bounded there.
# files.ingest_spooled passes through `spool_upload`'s value, bounded there.
# routes enqueue_clip the `display_name=` keyword passes on the name bound above it.
# pipeline._download / _process_clip / worker._finish_from_cache
# slice a fetched or stored title to the column's width.
@@ -614,8 +614,9 @@ DISPLAY_WRITE_SITES = {
"display_text(body.display_name)",
"display_text(body.display_uploader)",
},
("routes/downloads.py", "upload_download"): {"upload.display_name"},
("routes/downloads.py", "spool_upload"): {
("routes/files.py", "ingest_spooled"): {"upload.display_name"},
("routes/files.py", "update_file"): {"display_text(body.name)"},
("routes/files.py", "spool_upload"): {
"display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER))"
},
}
@@ -694,12 +695,12 @@ def test_the_upload_holds_no_db_transaction_while_the_body_streams(
the checkout is its consequence (measured), and the pool counter is shared with whatever else
the test process is holding.
"""
from app.routes import downloads as downloads_routes
from app.routes import files as files_routes
_probe(monkeypatch, (None, None, None, None, None))
seen: dict[str, bool] = {}
real_stream = downloads_routes.capped_stream_to
real_release = downloads_routes.release_read_connection
real_stream = files_routes.capped_stream_to
real_release = files_routes.release_read_connection
def spy_release(session):
# The PRECONDITION, so this test cannot pass vacuously. Asserting only "no transaction
@@ -714,8 +715,8 @@ def test_the_upload_holds_no_db_transaction_while_the_body_streams(
seen["while_streaming"] = bool(db.in_transaction())
return await real_stream(request, out, limit, refusal)
monkeypatch.setattr(downloads_routes, "release_read_connection", spy_release)
monkeypatch.setattr(downloads_routes, "capped_stream_to", spy_stream)
monkeypatch.setattr(files_routes, "release_read_connection", spy_release)
monkeypatch.setattr(files_routes, "capped_stream_to", spy_stream)
c = as_user(make_user())
r = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("held.bin"))
assert r.status_code == 200, r.text
frontend/src/i18n/backendIds.contract.test.ts
+6 -1
View File
@@ -141,7 +141,7 @@ const goneWhats = [
];
// Every generic refusal → `errors.refused.<reason>`: each `refused(<status>, "<reason>")` anywhere in
// the backend (not `_refused`, a router's own family), plus the values of the settings map, which
// `setting_refused` picks from at runtime.
// `setting_refused` picks from at runtime, and every `FolderError` reason.
const appSources = (dir = ""): string[] =>
readdirSync(new URL(`../../../backend/app/${dir}`, import.meta.url), {
withFileTypes: true,
@@ -157,6 +157,11 @@ const refusedReasons = [
...appSources().flatMap((src) =>
[...src.matchAll(/(?<![\w.])refused\(\s*\d+,\s*"(\w+)"/g)].map((m) => m[1]!),
),
// A folder change the rules refuse travels as `FolderError("<reason>")` and is answered with
// `refused(409, e.reason)`, which the pattern above cannot read.
...appSources().flatMap((src) =>
[...src.matchAll(/FolderError\("(\w+)"\)/g)].map((m) => m[1]!),
),
...[
...(backend("refusal.py").split("_SETTING_REASONS = {")[1]?.split("}")[0] ?? "").matchAll(
/:\s*"(\w+)"/g,
frontend/src/i18n/locales/en/auditActions.json
+10 -1
View File
@@ -22,5 +22,14 @@
"download_credential_delete": "Download cookies deleted",
"invite_delete": "Whitelist entry deleted",
"invite_send": "Invitation sent",
"issue_create": "Bug report sent"
"issue_create": "Bug report sent",
"file_upload": "File uploaded",
"file_delete": "File deleted",
"file_keep": "File kept",
"file_unkeep": "File no longer kept",
"file_move": "File moved",
"file_folder_create": "Folder created",
"file_folder_rename": "Folder renamed",
"file_folder_retention": "Folder retention changed",
"file_folder_delete": "Folder deleted"
}
frontend/src/i18n/locales/en/errors.json
+8 -2
View File
@@ -37,7 +37,8 @@
"poster": "There's no poster for this",
"profile": "This download profile doesn't exist any more",
"season": "This season isn't in the library any more",
"show": "This show isn't in the library any more"
"show": "This show isn't in the library any more",
"folder": "That folder no longer exists"
},
"admin": {
"not_allowed": "Their address isn't approved any more. Add it to the whitelist first.",
@@ -219,6 +220,11 @@
"screenshot_storage_full": "You've reached the {{limit}} MB screenshot allowance. Remove images from older sessions, or wait until they expire.",
"issue_sending": "The report is being sent right now. Wait for it to finish.",
"issue_rate_limited": "At most {{limit}} send attempts an hour. Try again later.",
"preview_rate_limited": "Too many previews in a row. Wait a minute."
"preview_rate_limited": "Too many previews in a row. Wait a minute.",
"file_folder_name_required": "Give the folder a name",
"file_folder_name_taken": "You already have a folder with that name",
"file_folder_not_own": "Only a folder you made can be renamed or deleted",
"file_folder_gone": "That folder no longer exists",
"file_folder_not_its_source": "A file can only go into one of your own folders or back to the folder it came from"
}
}
frontend/src/i18n/locales/hu/auditActions.json
+10 -1
View File
@@ -22,5 +22,14 @@
"download_credential_delete": "Letöltési cookie-k törölve",
"invite_delete": "Fehérlista-bejegyzés törölve",
"invite_send": "Meghívó elküldve",
"issue_create": "Hibajegy elküldve"
"issue_create": "Hibajegy elküldve",
"file_upload": "Fájl feltöltve",
"file_delete": "Fájl törölve",
"file_keep": "Fájl megtartva",
"file_unkeep": "Fájl megtartása visszavonva",
"file_move": "Fájl áthelyezve",
"file_folder_create": "Mappa létrehozva",
"file_folder_rename": "Mappa átnevezve",
"file_folder_retention": "Mappa megőrzési ideje módosítva",
"file_folder_delete": "Mappa törölve"
}
frontend/src/i18n/locales/hu/errors.json
+8 -2
View File
@@ -37,7 +37,8 @@
"poster": "Ehhez nincs poszter",
"profile": "Ez a letöltési profil már nem létezik",
"season": "Ez az évad már nincs a könyvtárban",
"show": "Ez a sorozat már nincs a könyvtárban"
"show": "Ez a sorozat már nincs a könyvtárban",
"folder": "Ez a mappa már nem létezik"
},
"admin": {
"not_allowed": "A címe már nincs jóváhagyva. Előbb vedd fel a fehérlistára.",
@@ -219,6 +220,11 @@
"screenshot_storage_full": "Elérted a képernyőképek {{limit}} MB-os keretét. Törölj régebbi munkamenetekből képeket, vagy várd meg, amíg lejárnak.",
"issue_sending": "A hibajegy küldése épp folyamatban van. Várd meg, amíg befejeződik.",
"issue_rate_limited": "Óránként legfeljebb {{limit}} küldési kísérlet. Próbáld újra később.",
"preview_rate_limited": "Túl sok előnézet egymás után. Várj egy percet."
"preview_rate_limited": "Túl sok előnézet egymás után. Várj egy percet.",
"file_folder_name_required": "Adj nevet a mappának",
"file_folder_name_taken": "Már van ilyen nevű mappád",
"file_folder_not_own": "Csak a saját mappádat nevezheted át vagy törölheted",
"file_folder_gone": "Ez a mappa már nem létezik",
"file_folder_not_its_source": "Egy fájl csak a saját mappáidba vagy vissza az eredeti mappájába kerülhet"
}
}