peter
b2d34e6e6c
A user can make a token for a machine of theirs (`/api/tokens`, signed-in only): `sfl_` + 32 random bytes + a checksum, only its SHA-256 kept, shown once. Sent as a Bearer header it is honoured on six `/api/files` routes only — folders, list, file, hash, upload, delete — and there only in the folders its grants name, with the rights they give (list / read / write / delete). One check, `FilesCaller.may`, in every route; a folder or file out of reach answers like a missing one; the files shared with the user are never in reach. A token beats a cookie on the same request. The Hub preset makes the user's one Hub folder (a new folder kind) and grants it every right. Refused tokens are slowed down per client, a runaway machine per token; last use is stamped at most once a minute; making and revoking a token is audited, and a token's uploads and deletes name it in their audit rows.
930 lines
38 KiB
Python
930 lines
38 KiB
Python
"""The Files library HTTP API — a user's files, the folders they sit in, and the file itself.
|
|
|
|
A file in the library is a user's DONE job (`DownloadJob`) over its shared `MediaAsset`: the job is
|
|
the per-user layer (name, folder, kept, expiry, shares and links all hang off it), the asset the
|
|
bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk.
|
|
|
|
In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above
|
|
the routes are shared with that module — serialization, the ownership locks, the upload spool —
|
|
because a download and a library file are one row at two moments.
|
|
|
|
A machine with an API token (`app.apitokens`) may use the routes that take `files_caller`: list the
|
|
folders and files, fetch a file and its hash, upload, delete — each only in a folder its token was
|
|
granted that right in, and never the files shared with the user. A folder or file out of its reach
|
|
answers like a missing one. Everything else here is the signed-in user's alone (`current_user`).
|
|
"""
|
|
|
|
import base64
|
|
import hashlib
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
from collections.abc import AsyncIterator, Sequence
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import BinaryIO, Literal
|
|
from urllib.parse import unquote, urlparse
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
|
from fastapi.responses import FileResponse, Response
|
|
from pydantic import BaseModel, Field
|
|
from sqlalchemy import func, or_, select, update
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import audit, sysconfig
|
|
from app.apitokens import FilesCaller, Right, files_caller
|
|
from app.audit import AuditAction
|
|
from app.auth import current_user
|
|
from app.collation import collated
|
|
from app.config import settings
|
|
from app.db import get_db, release_read_connection
|
|
from app.downloads import folders, preview, quota, service, storage
|
|
from app.downloads import upload as uploadmod
|
|
from app.downloads.status import JobStatus
|
|
from app.mailtext import lang_for_user
|
|
from app.models import (
|
|
DISPLAY_TEXT_MAX,
|
|
DownloadJob,
|
|
DownloadShare,
|
|
FileFolder,
|
|
LibraryState,
|
|
MediaAsset,
|
|
User,
|
|
display_text,
|
|
)
|
|
from app.refusal import refused
|
|
from app.routes._common import capped_stream_to, not_found, owned_or_404
|
|
from app.utils import iso, like_escape
|
|
|
|
router = APIRouter(prefix="/api/files", tags=["files"])
|
|
|
|
# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's
|
|
# own cap; far beyond any real library).
|
|
MAX_LIST = 500
|
|
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
|
|
_MAX_RETENTION_DAYS = 3650
|
|
_HASH_CHUNK = 1 << 20
|
|
# How many files the hash route reads at once: each holds a threadpool thread for the whole read.
|
|
_HASHERS = 2
|
|
_HASHING = threading.BoundedSemaphore(_HASHERS)
|
|
|
|
|
|
# --- shared with routes.downloads -------------------------------------------------------
|
|
|
|
|
|
def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
|
ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
|
|
return {
|
|
"id": job.id,
|
|
"status": job.status,
|
|
"progress": job.progress,
|
|
"speed_bps": job.speed_bps,
|
|
"eta_s": job.eta_s,
|
|
"phase": job.phase,
|
|
"error": job.error,
|
|
"error_code": job.error_code,
|
|
"error_detail": job.error_detail,
|
|
"queue_pos": job.queue_pos,
|
|
"created_at": iso(job.created_at),
|
|
"ready_at": iso(job.ready_at),
|
|
"source_kind": job.source_kind,
|
|
"source_ref": job.source_ref,
|
|
"source_url": service.reference_url(job, asset),
|
|
# A locally-generated poster to fall back to when the source had no thumbnail.
|
|
"poster_url": f"/api/downloads/{job.id}/poster.jpg"
|
|
if (asset is not None and asset.poster_path)
|
|
else None,
|
|
"profile_id": job.profile_id,
|
|
"spec": job.profile_snapshot,
|
|
"display_name": job.display_name,
|
|
"display_uploader": job.display_uploader,
|
|
"display_uploader_url": job.display_uploader_url,
|
|
"extra_links": job.extra_links or [],
|
|
"job_kind": job.job_kind,
|
|
"source_job_id": job.source_job_id,
|
|
"edit_spec": job.edit_spec,
|
|
"can_download": ready and job.status == JobStatus.DONE,
|
|
# The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
|
|
"folder_id": job.folder_id,
|
|
"kept": job.kept,
|
|
"expires_at": iso(job.expires_at),
|
|
"asset": None
|
|
if asset is None
|
|
else {
|
|
"id": asset.id,
|
|
"status": asset.status,
|
|
"title": asset.title,
|
|
"uploader": asset.uploader,
|
|
"upload_date": iso(asset.upload_date),
|
|
"duration_s": asset.duration_s,
|
|
"size_bytes": asset.size_bytes,
|
|
"width": asset.width,
|
|
"height": asset.height,
|
|
"container": asset.container,
|
|
# "video" | "audio" | "file" — the library gates play/edit on this (media only); a
|
|
# generic uploaded file is download-only. Null while still in flight (set at finalize).
|
|
"media_kind": asset.media_kind,
|
|
"uploader_url": asset.uploader_url,
|
|
"thumbnail_url": asset.thumbnail_url,
|
|
# "image" | "pdf" | "text" | null — what `/preview` shows (`downloads/preview.py`).
|
|
"preview": preview.kind(asset.media_kind, asset.container),
|
|
},
|
|
}
|
|
|
|
|
|
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
|
|
ids = {j.asset_id for j in jobs if j.asset_id}
|
|
if not ids:
|
|
return {}
|
|
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
|
|
return {a.id: a for a in rows}
|
|
|
|
|
|
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
|
|
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
|
|
ids = [j.id for j in jobs]
|
|
if not ids:
|
|
return {}
|
|
rows = db.execute(
|
|
select(LibraryState.job_id, LibraryState.position_seconds).where(
|
|
LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
|
|
)
|
|
).all()
|
|
return {job_id: pos for job_id, pos in rows}
|
|
|
|
|
|
def serialize_job(db: Session, job: DownloadJob) -> dict:
|
|
"""Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
|
|
every single-job handler shares."""
|
|
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
|
return serialize(job, asset)
|
|
|
|
|
|
def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
|
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
|
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
|
overlapping requests for one job run one after the other and the second decides on the first's
|
|
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
|
just released back into a holder; either can free a file another job still holds.
|
|
|
|
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
|
(`service.finalize_if_running`), so the two can never deadlock."""
|
|
while True:
|
|
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
|
|
if asset_id is not None:
|
|
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
|
job = db.execute(
|
|
select(DownloadJob)
|
|
.where(DownloadJob.id == job_id)
|
|
.with_for_update()
|
|
.execution_options(populate_existing=True)
|
|
).scalar_one_or_none()
|
|
if job is None:
|
|
raise not_found("download")
|
|
if job.asset_id == asset_id:
|
|
return job
|
|
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
|
|
|
|
|
def quota_detail(e: quota.QuotaExceeded) -> dict:
|
|
"""Structured error the trilingual client localizes (reason key + numbers), instead of a
|
|
pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
|
|
return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current}
|
|
|
|
|
|
# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
|
|
# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
|
|
# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous
|
|
# caller could make the process write the whole per-file allowance to temp, per request (SEC-20
|
|
# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
|
|
# declared AFTER the auth one, is the only shape that closes it — the same one
|
|
# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
|
|
# every route with a raised ceiling rather than leaving it to a comment like this one.
|
|
#
|
|
# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
|
|
# Hungarian filename would otherwise be unsendable.
|
|
UPLOAD_FILENAME_HEADER = "x-upload-filename"
|
|
UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
|
|
# The folder the file goes in (its id); without it, the user's Uploads folder.
|
|
UPLOAD_FOLDER_HEADER = "x-upload-folder"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SpooledUpload:
|
|
"""What the body dependency hands the handler: the bytes already on disk and their hash, plus
|
|
the names and the folder that came alongside them, and who sent them."""
|
|
|
|
path: Path
|
|
size: int
|
|
sha256: str
|
|
user_id: int
|
|
filename: str
|
|
display_name: str | None
|
|
folder_id: int | None
|
|
via: str | None = None
|
|
|
|
|
|
def _header_text(request: Request, name: str) -> str:
|
|
"""One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
|
|
caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
|
|
own fallback names the file."""
|
|
raw = request.headers.get(name, "")
|
|
try:
|
|
return unquote(raw, errors="strict").strip()
|
|
except (UnicodeDecodeError, ValueError):
|
|
# silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
|
|
# cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
|
|
# a label; logging it would hand any client a free log line per request.
|
|
return ""
|
|
|
|
|
|
def _upload_folder(db: Session, caller: FilesCaller, raw: str) -> int | None:
|
|
"""The folder id an upload asked for (`UPLOAD_FOLDER_HEADER`), checked; None when it asked for
|
|
none — which a token may not: it writes only where it was granted to."""
|
|
if not raw.strip():
|
|
if caller.grants is not None:
|
|
raise refused(400, "api_token_folder_required")
|
|
return None
|
|
try:
|
|
folder_id = int(raw)
|
|
except ValueError as e:
|
|
raise refused(409, "file_folder_gone") from e
|
|
if not caller.may(folder_id, Right.WRITE):
|
|
raise not_found("folder")
|
|
try:
|
|
return folders.upload_target(db, caller.user.id, folder_id, hold=False).id
|
|
except folders.FolderError as e:
|
|
raise refused(409, e.reason) from e
|
|
|
|
|
|
class _HashingSink:
|
|
"""The upload's file, hashing the bytes on their way in: the hash costs no second read."""
|
|
|
|
def __init__(self, out: BinaryIO) -> None:
|
|
self.out = out
|
|
self.hash = hashlib.sha256()
|
|
|
|
def write(self, data: bytes, /) -> int:
|
|
self.hash.update(data)
|
|
return self.out.write(data)
|
|
|
|
|
|
async def spool_upload(
|
|
request: Request,
|
|
caller: FilesCaller = Depends(files_caller),
|
|
db: Session = Depends(get_db),
|
|
) -> AsyncIterator[SpooledUpload]:
|
|
"""Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
|
|
|
|
`caller` comes first on purpose. Dependencies resolve in declaration order, so naming
|
|
`files_caller` here makes the auth check a precondition of reading the body at all — a stranger,
|
|
or a token without the right to write in that folder, is refused before a byte moves. The
|
|
`yield` shape owns the temp file's life: ingest MOVES it into place on success (the unlink is
|
|
then a no-op), and on any other exit — cap hit, empty, quota, probe or move error, a client that
|
|
vanished mid-upload — it is removed."""
|
|
cap = sysconfig.get_int(db, "download_upload_max_bytes")
|
|
user_id = caller.user.id
|
|
via = caller.via
|
|
# A folder the file may not go in is refused before the transfer, not after it.
|
|
folder_id = _upload_folder(db, caller, request.headers.get(UPLOAD_FOLDER_HEADER, ""))
|
|
# Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
|
|
# for why, and for the precondition it enforces. Everything this dependency still needs is
|
|
# already an int, a string or a header in hand: touching an ORM attribute below (`caller.user`)
|
|
# would reload it and check a connection straight back out.
|
|
release_read_connection(db)
|
|
staging = Path(settings.files_root) / ".staging" / "uploads"
|
|
staging.mkdir(parents=True, exist_ok=True)
|
|
fd, tmp = tempfile.mkstemp(dir=str(staging))
|
|
tmp_path = Path(tmp)
|
|
try:
|
|
with os.fdopen(fd, "wb") as out:
|
|
sink = _HashingSink(out)
|
|
size = await capped_stream_to(
|
|
request,
|
|
sink,
|
|
cap,
|
|
HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
|
|
)
|
|
yield SpooledUpload(
|
|
path=tmp_path,
|
|
size=size,
|
|
sha256=sink.hash.hexdigest(),
|
|
user_id=user_id,
|
|
filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
|
|
display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
|
|
folder_id=folder_id,
|
|
via=via,
|
|
)
|
|
finally:
|
|
tmp_path.unlink(missing_ok=True)
|
|
|
|
|
|
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
|
"""A file the caller may read: their own, or one shared with them."""
|
|
job = db.get(DownloadJob, job_id)
|
|
if job is None:
|
|
raise not_found("download")
|
|
if job.user_id == user.id:
|
|
return job
|
|
shared = db.execute(
|
|
select(DownloadShare.id).where(
|
|
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
|
|
)
|
|
).first()
|
|
if shared is None:
|
|
raise not_found("download")
|
|
return job
|
|
|
|
|
|
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob:
|
|
"""A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's
|
|
own in a folder it may read."""
|
|
if caller.grants is None:
|
|
return accessible_job(db, caller.user, job_id)
|
|
job = db.get(DownloadJob, job_id)
|
|
if job is None or job.user_id != caller.user.id or not caller.may(job.folder_id, Right.READ):
|
|
raise not_found("download")
|
|
return job
|
|
|
|
|
|
# --- folders -------------------------------------------------------------------------------
|
|
|
|
|
|
def _serialize_folder(folder: FileFolder, count: int, size: int) -> dict:
|
|
return {
|
|
"id": folder.id,
|
|
"kind": folder.kind,
|
|
# A source folder's name is the client's to render from its key, in the reader's language.
|
|
"source_key": folder.source_key,
|
|
"name": folder.name,
|
|
"retention_days": folder.retention_days,
|
|
"count": count,
|
|
"size_bytes": size,
|
|
"created_at": iso(folder.created_at),
|
|
}
|
|
|
|
|
|
@router.get("/folders")
|
|
def list_folders(
|
|
caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
|
) -> list[dict]:
|
|
"""The caller's folders with how many files each holds and their size (a token: the ones it may
|
|
list). A source folder its files have all left stays (count 0): the client decides whether to
|
|
show it."""
|
|
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
|
|
stmt = select(
|
|
FileFolder,
|
|
func.count(DownloadJob.id),
|
|
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
|
|
).where(FileFolder.user_id == caller.user.id)
|
|
listable = caller.folders_with(Right.LIST)
|
|
if listable is not None:
|
|
stmt = stmt.where(FileFolder.id.in_(listable))
|
|
rows = db.execute(
|
|
stmt.outerjoin(DownloadJob, done)
|
|
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
|
.group_by(FileFolder.id)
|
|
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
|
|
).all()
|
|
return [_serialize_folder(f, count, int(size)) for f, count, size in rows]
|
|
|
|
|
|
class FolderIn(BaseModel):
|
|
name: str = Field(min_length=1, max_length=DISPLAY_TEXT_MAX)
|
|
|
|
|
|
def _folder_name(raw: str) -> str:
|
|
name = display_text(raw)
|
|
if not name:
|
|
raise refused(400, "file_folder_name_required")
|
|
return name
|
|
|
|
|
|
@router.post("/folders")
|
|
def create_folder(
|
|
body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
|
) -> dict:
|
|
folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name))
|
|
try:
|
|
with db.begin_nested():
|
|
db.add(folder)
|
|
db.flush()
|
|
except IntegrityError as e: # the unique (user, lower(name)): one folder per name
|
|
raise refused(409, "file_folder_name_taken") from e
|
|
audit.record(
|
|
db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id
|
|
)
|
|
db.commit()
|
|
return _serialize_folder(folder, 0, 0)
|
|
|
|
|
|
class UpdateFolderIn(BaseModel):
|
|
# Presence (model_fields_set) decides what is touched; retention_days null = keep forever.
|
|
name: str | None = Field(default=None, min_length=1, max_length=DISPLAY_TEXT_MAX)
|
|
retention_days: int | None = Field(default=None, ge=1, le=_MAX_RETENTION_DAYS)
|
|
|
|
|
|
@router.patch("/folders/{folder_id}")
|
|
def update_folder(
|
|
folder_id: int,
|
|
body: UpdateFolderIn,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Rename one of the caller's own folders (a source folder's name is its origin's), and/or set
|
|
how long any of their folders keeps a file (`folders.set_retention`)."""
|
|
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
|
sent = body.model_fields_set
|
|
if "name" in sent:
|
|
if folder.kind != folders.FolderKind.OWN:
|
|
raise refused(409, "file_folder_not_own")
|
|
folder.name = _folder_name(body.name or "")
|
|
try:
|
|
with db.begin_nested():
|
|
db.flush()
|
|
except IntegrityError as e:
|
|
raise refused(409, "file_folder_name_taken") from e
|
|
audit.record(
|
|
db,
|
|
user.id,
|
|
AuditAction.FILE_FOLDER_RENAME,
|
|
target_type="file_folder",
|
|
target_id=folder.id,
|
|
)
|
|
if "retention_days" in sent:
|
|
# The value replaced is the one read under the folder's lock, not the one read above it.
|
|
try:
|
|
before = folders.set_retention(db, folder, body.retention_days)
|
|
except folders.FolderError as e:
|
|
raise refused(409, e.reason) from e
|
|
if before != body.retention_days:
|
|
audit.record(
|
|
db,
|
|
user.id,
|
|
AuditAction.FILE_FOLDER_RETENTION,
|
|
target_type="file_folder",
|
|
target_id=folder.id,
|
|
before={"retention_days": before},
|
|
after={"retention_days": body.retention_days},
|
|
)
|
|
db.commit()
|
|
count, size = _folder_totals(db, folder.id)
|
|
return _serialize_folder(folder, count, size)
|
|
|
|
|
|
def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
|
|
count, size = db.execute(
|
|
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
|
|
.select_from(DownloadJob)
|
|
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
|
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
|
|
).one()
|
|
return count, int(size)
|
|
|
|
|
|
@router.delete("/folders/{folder_id}")
|
|
def delete_folder(
|
|
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
|
) -> dict:
|
|
"""Delete one of the caller's own folders; its files go back to their source folders."""
|
|
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
|
try:
|
|
moved = folders.delete_own(db, folder)
|
|
except folders.FolderError as e:
|
|
raise refused(409, e.reason) from e
|
|
audit.record(
|
|
db,
|
|
user.id,
|
|
AuditAction.FILE_FOLDER_DELETE,
|
|
target_type="file_folder",
|
|
target_id=folder_id,
|
|
after={"files_moved": moved},
|
|
)
|
|
db.commit()
|
|
return {"ok": True, "deleted": folder_id, "files_moved": moved}
|
|
|
|
|
|
@router.get("/usage")
|
|
def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
|
"""The caller's storage quota, plus the per-file upload cap.
|
|
|
|
The cap rides here rather than in `quota.usage` because it is not a quota value — it is a
|
|
`sysconfig` knob about one route — and this is the route layer, where composing the two is the
|
|
job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side
|
|
caps answer too late to be pleasant (the edge ceiling refuses the request outright, see
|
|
`bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the
|
|
limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the
|
|
boundary: the caps stay where they are and still refuse a client that ignores this.
|
|
"""
|
|
return {
|
|
**quota.usage(db, user.id),
|
|
"upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"),
|
|
}
|
|
|
|
|
|
@router.delete("/shared/{job_id}")
|
|
def remove_shared_with_me(
|
|
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
|
) -> dict:
|
|
"""Remove a file that was shared WITH this user from their 'Shared with me' list. Deletes
|
|
only the recipient's share grant — the owner's job and the physical file are untouched (this is
|
|
a per-user dismissal, not a delete)."""
|
|
row = db.execute(
|
|
select(DownloadShare).where(
|
|
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
|
|
)
|
|
).scalar_one_or_none()
|
|
if row is not None:
|
|
db.delete(row)
|
|
db.commit()
|
|
return {"ok": True, "removed": job_id}
|
|
|
|
|
|
# --- files ---------------------------------------------------------------------------------
|
|
|
|
|
|
@router.get("")
|
|
def list_files(
|
|
folder_id: int | None = None,
|
|
q: str = Query("", max_length=200),
|
|
sort: Literal["added", "name", "size"] = "added",
|
|
order: Literal["asc", "desc"] = "desc",
|
|
scope: Literal["mine", "shared"] = "mine",
|
|
caller: FilesCaller = Depends(files_caller),
|
|
db: Session = Depends(get_db),
|
|
) -> list[dict]:
|
|
"""The caller's done files (`scope=mine`, optionally one folder's), or the ones shared with
|
|
them (`scope=shared`); searched by name and channel, sorted by when they were added, by name
|
|
(in the reader's alphabet) or by size. A token lists only in the folders it may list, and never
|
|
the shared ones."""
|
|
user = caller.user
|
|
listable = caller.folders_with(Right.LIST)
|
|
if listable is not None:
|
|
if scope == "shared":
|
|
raise refused(403, "api_token_scope")
|
|
if folder_id is not None and folder_id not in listable:
|
|
raise not_found("folder")
|
|
stmt = (
|
|
select(DownloadJob)
|
|
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
|
.where(DownloadJob.status == JobStatus.DONE)
|
|
)
|
|
shared = scope == "shared"
|
|
if shared:
|
|
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
|
|
DownloadShare.to_user_id == user.id
|
|
)
|
|
else:
|
|
stmt = stmt.where(DownloadJob.user_id == user.id)
|
|
if folder_id is not None:
|
|
stmt = stmt.where(DownloadJob.folder_id == folder_id)
|
|
elif listable is not None:
|
|
stmt = stmt.where(DownloadJob.folder_id.in_(listable))
|
|
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
|
|
term = q.strip()
|
|
if term:
|
|
like = f"%{like_escape(term)}%"
|
|
stmt = stmt.where(
|
|
or_(
|
|
name.ilike(like),
|
|
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
|
|
)
|
|
)
|
|
# A file is added to its owner's list when it is done, not when it was asked for (a download that
|
|
# failed for days and then went through is new); one shared with the caller, when it was shared.
|
|
key = {
|
|
"added": DownloadShare.created_at if shared else DownloadJob.ready_at,
|
|
"name": collated(name, lang_for_user(user)),
|
|
"size": MediaAsset.size_bytes,
|
|
}[sort]
|
|
first = key.asc() if order == "asc" else key.desc()
|
|
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
|
|
jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all()
|
|
assets = assets_for(db, jobs)
|
|
positions = positions_for(db, user, jobs)
|
|
shared_at = _shared_at(db, user, jobs) if shared else {}
|
|
sources = source_keys(db, jobs)
|
|
return [
|
|
{
|
|
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
|
|
"position_seconds": positions.get(j.id, 0),
|
|
"shared": shared,
|
|
"shared_at": iso(shared_at.get(j.id)),
|
|
"source_key": sources[j.id],
|
|
}
|
|
for j in jobs
|
|
]
|
|
|
|
|
|
def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
|
|
"""Where each file came from (its source folder's key), for the list's Source column, the
|
|
"back to its source folder" move and the download tray's site: a file in a source folder has its folder's key; one moved into
|
|
an own folder (or someone else's, shared) is traced back by `folders.source_key_for`."""
|
|
ids = {j.folder_id for j in jobs if j.folder_id is not None}
|
|
by_id = (
|
|
{f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
|
|
if ids
|
|
else {}
|
|
)
|
|
out: dict[int, str] = {}
|
|
for j in jobs:
|
|
folder = by_id.get(j.folder_id) if j.folder_id is not None else None
|
|
if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
|
|
out[j.id] = folder.source_key
|
|
else:
|
|
out[j.id] = folders.source_key_for(db, j)
|
|
return out
|
|
|
|
|
|
def _shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
|
|
rows = db.execute(
|
|
select(DownloadShare.job_id, DownloadShare.created_at).where(
|
|
DownloadShare.to_user_id == user.id,
|
|
DownloadShare.job_id.in_([j.id for j in jobs]),
|
|
)
|
|
).all()
|
|
return {job_id: at for job_id, at in rows}
|
|
|
|
|
|
_MAX_EXTRA_LINKS = 8
|
|
|
|
|
|
def _clean_url(raw) -> str | None:
|
|
"""Accept only a plausible http(s) URL (these get rendered as clickable links + embedded in a
|
|
public watch page); anything else is dropped rather than trusted."""
|
|
if not isinstance(raw, str):
|
|
return None
|
|
url = raw.strip()
|
|
if not url:
|
|
return None
|
|
if not url.startswith(("http://", "https://")):
|
|
return None
|
|
# `urlparse` raises on a malformed authority; this function already has a "not a plausible URL"
|
|
# answer, so the typo belongs in it.
|
|
try:
|
|
parsed = urlparse(url)
|
|
# silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL"
|
|
# answer, the caller already has one, and any line here would carry the URL itself.
|
|
except ValueError:
|
|
return None
|
|
if not parsed.netloc:
|
|
return None
|
|
return url[:2048]
|
|
|
|
|
|
class UpdateFileIn(BaseModel):
|
|
# Presence (model_fields_set) decides what is touched; a null/"" text falls back to the system
|
|
# value (the title, the channel). All display-only: the file on disk keeps its own name.
|
|
name: str | None = None
|
|
uploader: str | None = None
|
|
uploader_url: str | None = None
|
|
extra_links: list | None = None
|
|
folder_id: int | None = None
|
|
kept: bool | None = None
|
|
|
|
|
|
@router.patch("/{job_id}")
|
|
def update_file(
|
|
job_id: int,
|
|
body: UpdateFileIn,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Rename a file and its channel, set its reference links (all display-only: the file on disk
|
|
keeps its own name), move it to another folder (`folders.move`), or keep it until deleted /
|
|
hand it back to its folder's retention."""
|
|
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
|
sent = body.model_fields_set
|
|
if "name" in sent:
|
|
job.display_name = display_text(body.name)
|
|
if "uploader" in sent:
|
|
job.display_uploader = display_text(body.uploader)
|
|
if "uploader_url" in sent:
|
|
job.display_uploader_url = _clean_url(body.uploader_url)
|
|
if "extra_links" in sent:
|
|
raw = body.extra_links or []
|
|
cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
|
|
job.extra_links = cleaned or None
|
|
if "folder_id" in sent:
|
|
if body.folder_id is None:
|
|
raise refused(400, "file_folder_gone")
|
|
before = job.folder_id
|
|
try:
|
|
folders.move(db, job, folders.own_folder(db, user.id, body.folder_id))
|
|
except folders.FolderError as e:
|
|
raise refused(409, e.reason) from e
|
|
if job.folder_id != before:
|
|
audit.record(
|
|
db,
|
|
user.id,
|
|
AuditAction.FILE_MOVE,
|
|
target_type="file",
|
|
target_id=job.id,
|
|
before={"folder_id": before},
|
|
after={"folder_id": job.folder_id},
|
|
)
|
|
if "kept" in sent and body.kept is not None and body.kept != job.kept:
|
|
folders.set_kept(db, job, body.kept)
|
|
action = AuditAction.FILE_KEEP if body.kept else AuditAction.FILE_UNKEEP
|
|
audit.record(db, user.id, action, target_type="file", target_id=job.id)
|
|
db.commit()
|
|
return serialize_job(db, job)
|
|
|
|
|
|
@router.delete("/{job_id}")
|
|
def delete_file(
|
|
job_id: int, caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
|
) -> dict:
|
|
"""Delete a file from the caller's library; the bytes go with their last holder. A token
|
|
deletes only in a folder it may delete in — judged on the row as locked, so a file moved out of
|
|
that folder meanwhile is out of its reach."""
|
|
job = own_job_for_update(db, caller.user, job_id)
|
|
if not caller.may(job.folder_id, Right.DELETE):
|
|
db.rollback() # lets go of the locks; nothing was written
|
|
raise not_found("download")
|
|
audit.record(
|
|
db,
|
|
caller.user.id,
|
|
AuditAction.FILE_DELETE,
|
|
target_type="file",
|
|
target_id=job_id,
|
|
summary=caller.via,
|
|
)
|
|
service.delete(db, job) # commits, the audit row with it
|
|
return {"ok": True, "deleted": job_id}
|
|
|
|
|
|
@router.post("/upload")
|
|
def upload_file(
|
|
upload: SpooledUpload = Depends(spool_upload),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Put a local file in the caller's library — in the folder `UPLOAD_FOLDER_HEADER` names, else
|
|
their Uploads folder. Any file type; media gets the player, a generic file is download-only.
|
|
The body IS the file (see the header note above `UPLOAD_FILENAME_HEADER`); `spool_upload` has
|
|
authenticated the caller, checked the folder and streamed it to disk already.
|
|
|
|
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
|
plus sync DB writes — so it must NOT run on the event loop."""
|
|
return serialize_job(db, ingest_spooled(db, upload))
|
|
|
|
|
|
def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
|
|
"""The spooled upload into the library (`upload.ingest_upload`, which commits its own rows),
|
|
its refusals answered as HTTP."""
|
|
if upload.size == 0:
|
|
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
|
try:
|
|
job = uploadmod.ingest_upload(
|
|
db,
|
|
upload.user_id,
|
|
filename=upload.filename,
|
|
src_path=upload.path,
|
|
size=upload.size,
|
|
display_name=upload.display_name,
|
|
sha256=upload.sha256,
|
|
folder_id=upload.folder_id,
|
|
via=upload.via,
|
|
)
|
|
except quota.QuotaExceeded as e:
|
|
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
|
|
except folders.FolderError as e: # the folder went while the file was on its way
|
|
raise refused(409, e.reason) from e
|
|
return job
|
|
|
|
|
|
def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
|
|
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
|
if asset is None or asset.status != "ready" or not asset.rel_path:
|
|
raise refused(409, "download_not_ready")
|
|
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
|
if path is None:
|
|
raise not_found("file")
|
|
return asset, path
|
|
|
|
|
|
def serve_file(db: Session, job: DownloadJob) -> FileResponse:
|
|
"""The file of `job` (the caller may read it), range-aware, named by the user's display name.
|
|
|
|
The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
|
|
(`storage.served_media_type`), NEVER guessed from the file's own extension. A generic 'file' (and
|
|
any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html")
|
|
returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on
|
|
this origin. Recognised media is served with its real type so the in-app player streams it. Both
|
|
are range-aware (FileResponse → 206). A file whose hash is known says so (`Repr-Digest`, RFC
|
|
9530: the digest of the whole file, whichever range is asked for)."""
|
|
asset, path = _ready_file(db, job)
|
|
filename = storage.download_filename(
|
|
job.display_name or asset.title or asset.source_ref, asset.container, path
|
|
)
|
|
headers = {}
|
|
if asset.sha256:
|
|
digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode()
|
|
headers["Repr-Digest"] = f"sha-256=:{digest}:"
|
|
asset.last_access_at = datetime.now(timezone.utc) # the handler commits it
|
|
media_type = storage.served_media_type(asset.media_kind, asset.container)
|
|
if media_type is None:
|
|
return FileResponse(
|
|
path,
|
|
filename=filename,
|
|
media_type="application/octet-stream",
|
|
content_disposition_type="attachment",
|
|
headers=headers,
|
|
)
|
|
return FileResponse(path, filename=filename, media_type=media_type, headers=headers)
|
|
|
|
|
|
@router.get("/{job_id}/file")
|
|
def get_file(
|
|
job_id: int, caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
|
):
|
|
response = serve_file(db, readable_job(db, caller, job_id))
|
|
db.commit()
|
|
return response
|
|
|
|
|
|
# A preview's own headers. `sandbox` leaves the answer no script and an opaque origin, whatever the
|
|
# file turns out to hold; `frame-ancestors 'self'` lets the app's viewer frame a PDF (every other
|
|
# answer keeps the app-wide 'none', main.py), with X-Frame-Options saying the same for old engines.
|
|
PREVIEW_HEADERS = {
|
|
"Content-Security-Policy": "sandbox; frame-ancestors 'self'",
|
|
"X-Frame-Options": "SAMEORIGIN",
|
|
}
|
|
# Set on a text preview cut at `preview.TEXT_PREVIEW_BYTES`, so the viewer can say it shows the start.
|
|
TRUNCATED_HEADER = "X-Preview-Truncated"
|
|
|
|
|
|
@router.get("/{job_id}/preview")
|
|
def preview_file(
|
|
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
|
) -> Response:
|
|
"""The file shown in the library's viewer: an image or a PDF inline, a text format as plain text
|
|
(its first `TEXT_PREVIEW_BYTES`). The type comes from `downloads/preview.py`'s allow-list, so
|
|
an HTML or SVG upload arrives as text, never as a page. Anything else: 415 `file_no_preview`."""
|
|
job = accessible_job(db, user, job_id)
|
|
asset, path = _ready_file(db, job)
|
|
kind = preview.kind(asset.media_kind, asset.container)
|
|
if kind is None:
|
|
raise refused(415, "file_no_preview")
|
|
headers = dict(PREVIEW_HEADERS)
|
|
if kind == "text":
|
|
with path.open("rb") as f:
|
|
head = f.read(preview.TEXT_PREVIEW_BYTES + 1)
|
|
if len(head) > preview.TEXT_PREVIEW_BYTES:
|
|
head = head[: preview.TEXT_PREVIEW_BYTES]
|
|
headers[TRUNCATED_HEADER] = "1"
|
|
text = preview.text_utf8(head)
|
|
if text is None:
|
|
raise refused(415, "file_no_preview")
|
|
asset.last_access_at = datetime.now(timezone.utc)
|
|
db.commit()
|
|
return Response(text, media_type=preview.media_type(kind, asset.container), headers=headers)
|
|
filename = storage.download_filename(
|
|
job.display_name or asset.title or asset.source_ref, asset.container, path
|
|
)
|
|
asset.last_access_at = datetime.now(timezone.utc)
|
|
db.commit()
|
|
return FileResponse(
|
|
path,
|
|
filename=filename,
|
|
media_type=preview.media_type(kind, asset.container),
|
|
content_disposition_type="inline",
|
|
headers=headers,
|
|
)
|
|
|
|
|
|
@router.get("/{job_id}/sha256")
|
|
def get_sha256(
|
|
job_id: int, caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
|
) -> dict:
|
|
"""The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder.
|
|
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool — and
|
|
every other sync route shares that pool, so only `_HASHERS` files are hashed at a time (a caller
|
|
past that is told to come back), and none of them holds a database connection while it reads."""
|
|
asset, path = _ready_file(db, readable_job(db, caller, job_id))
|
|
if asset.sha256:
|
|
return {"sha256": asset.sha256}
|
|
asset_id = asset.id
|
|
if not _HASHING.acquire(blocking=False):
|
|
raise refused(429, "file_hash_busy")
|
|
try:
|
|
release_read_connection(db)
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as f:
|
|
while chunk := f.read(_HASH_CHUNK):
|
|
h.update(chunk)
|
|
except OSError as e:
|
|
raise not_found("file") from e
|
|
finally:
|
|
_HASHING.release()
|
|
digest = h.hexdigest()
|
|
db.execute(
|
|
update(MediaAsset)
|
|
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
|
|
.values(sha256=digest)
|
|
)
|
|
db.commit()
|
|
return {"sha256": digest}
|