feat(downloads): local file upload ingest — media library backend (E7 S5a)
Add the non-yt-dlp ingest path so a user-uploaded file joins the same download
library and inherits its function set. Any file type is accepted: media
(video/audio) gets the full set (metadata, share, play/edit), a generic file
(pdf/zip/docx/…) is download-only.
- MediaAsset.media_kind ("video"|"audio"|"file") — the stored discriminator that
gates library actions. Set at every finalize (download probe, editor carries the
source's kind, upload probe); consolidates the ad-hoc audio-detect that lived only
in links.py into one invariant (downloads.mediakind.derive). Migration 0076 adds
the column + backfills existing rows.
- downloads/upload.py: ingest_upload probes the file (ffprobe → codecs/dims/duration),
places it in a per-user .uploads tree, and creates an already-ready asset + done job
(the same cache-hit shortcut enqueue uses). Uploads never expire (irreplaceable).
- POST /api/downloads/upload: streaming multipart with the per-file cap
(download_upload_max_bytes, default 500 MiB) enforced on actual bytes read; quota
charges the known size up front (quota.check_upload).
- Security: a generic file is always served force-download (attachment +
application/octet-stream) on both the in-app and public endpoints, so uploaded
HTML/SVG can't execute inline. Edit + redownload reject non-media / upload sources.
- python-multipart added (first multipart endpoint in the app).
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
"""media_assets.media_kind (E7 S5a: the video|audio|file discriminator for the media library)
|
||||
|
||||
Adds a stored media-kind to each asset so the library can gate its actions on it (play/edit for
|
||||
media, download-only for a generic uploaded file) instead of re-deriving from codecs in prose. The
|
||||
column is the single home of the invariant `downloads.mediakind.derive` computes at finalize.
|
||||
|
||||
Backfill for existing rows: a stored video codec -> 'video', an audio-only codec -> 'audio', an
|
||||
editor derivative (source_kind='edit', which carries no codecs but is always a video clip) ->
|
||||
'video'. Rows still pending/errored (no codecs, not an edit) stay NULL — kind is set when they
|
||||
finalize. Empty `media_kind` never reaches the UI: only `ready`/`done` rows render actions, and
|
||||
those are all backfilled here.
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0076_media_asset_kind"
|
||||
down_revision: Union[str, None] = "0075_playlist_sync_baseline"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("media_assets", sa.Column("media_kind", sa.String(length=16), nullable=True))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE media_assets
|
||||
SET media_kind = CASE
|
||||
WHEN vcodec IS NOT NULL THEN 'video'
|
||||
WHEN acodec IS NOT NULL THEN 'audio'
|
||||
WHEN source_kind = 'edit' THEN 'video'
|
||||
ELSE NULL
|
||||
END
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("media_assets", "media_kind")
|
||||
@@ -208,6 +208,10 @@ class Settings(BaseSettings):
|
||||
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
||||
download_default_max_concurrent: int = 2
|
||||
download_default_max_jobs: int = 50
|
||||
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
||||
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
||||
# user's overall byte footprint is still governed by download_default_max_bytes.
|
||||
download_upload_max_bytes: int = 524_288_000 # 500 MiB
|
||||
# Retention: an asset expires this many days after its last access; the GC job then
|
||||
# deletes it (once no job references it) after an additional grace window.
|
||||
download_retention_days: int = 30
|
||||
|
||||
@@ -80,13 +80,13 @@ def public_meta(
|
||||
|
||||
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
||||
auto-extracted values, and surfaces the source + any extra reference links as clickable URLs."""
|
||||
from app.downloads import service
|
||||
from app.downloads import mediakind, service
|
||||
|
||||
source_url = service.reference_url(job, asset)
|
||||
# Audio-only assets (mp3 rips, extracted audio) carry a codec but no video stream. WebKit (all
|
||||
# iOS browsers) refuses to play them inside a <video> element, so the watch page must render a
|
||||
# native <audio> instead — flag it here from the recorded codecs (a real video always has vcodec).
|
||||
is_audio = asset.vcodec is None and asset.acodec is not None
|
||||
# Audio-only assets (mp3 rips, extracted audio) can't play inside a <video> element on WebKit
|
||||
# (all iOS browsers), so the watch page renders a native <audio> instead. A generic uploaded
|
||||
# file (media_kind='file') has no player at all — the watch page shows file info + a download.
|
||||
# Both branches read the stored media_kind (the single source), not a re-derived codec check.
|
||||
return {
|
||||
"title": (job and job.display_name) or asset.title,
|
||||
"uploader": (job and job.display_uploader) or asset.uploader,
|
||||
@@ -96,7 +96,10 @@ def public_meta(
|
||||
"duration_s": asset.duration_s,
|
||||
"width": asset.width,
|
||||
"height": asset.height,
|
||||
"audio": is_audio,
|
||||
"audio": asset.media_kind == mediakind.AUDIO,
|
||||
"media_kind": asset.media_kind,
|
||||
"size_bytes": asset.size_bytes,
|
||||
"container": asset.container,
|
||||
"allow_download": link.allow_download,
|
||||
"file_url": file_url,
|
||||
"poster_url": poster_url,
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""The media-kind discriminator for a library asset — the single home of one invariant.
|
||||
|
||||
`MediaAsset.media_kind` is `"video" | "audio" | "file"`. It decides which library actions apply:
|
||||
media (video/audio) gets play/edit/share; a generic uploaded file gets download + share only, and
|
||||
is never rendered inline (see the force-download serving in the file/watch routes). Every finalize
|
||||
path computes it HERE — the download worker + the upload ingest via `derive`, the editor by
|
||||
carrying the source's kind over — so the rule lives in code, not in a comment that can drift (an
|
||||
earlier ad-hoc `vcodec is None and acodec is not None` audio check lived only in links.py).
|
||||
"""
|
||||
|
||||
VIDEO = "video"
|
||||
AUDIO = "audio"
|
||||
FILE = "file"
|
||||
|
||||
|
||||
def derive(vcodec: str | None, acodec: str | None) -> str:
|
||||
"""Media kind from the probed codecs: a video stream -> video, audio-only -> audio, neither
|
||||
(a non-media upload: pdf/zip/docx/…) -> file."""
|
||||
if vcodec:
|
||||
return VIDEO
|
||||
if acodec:
|
||||
return AUDIO
|
||||
return FILE
|
||||
@@ -15,7 +15,7 @@ from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import adapters, credentials, formats, service, storage
|
||||
from app.downloads import adapters, credentials, formats, mediakind, service, storage
|
||||
from app.downloads.jobdb import (
|
||||
_edit_staging_dir,
|
||||
_job_status,
|
||||
@@ -484,6 +484,7 @@ def _download(
|
||||
asset.acodec = (
|
||||
info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
)
|
||||
asset.media_kind = mediakind.derive(asset.vcodec, asset.acodec)
|
||||
asset.title = meta.title
|
||||
asset.uploader = meta.uploader
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
|
||||
@@ -122,6 +122,22 @@ def check_enqueue(db: Session, user_id: int) -> None:
|
||||
raise QuotaExceeded("max_bytes", lim.max_bytes, used)
|
||||
|
||||
|
||||
def check_upload(db: Session, user_id: int, add_bytes: int) -> None:
|
||||
"""Like check_enqueue, but for an upload whose size is KNOWN at ingest: also reject when adding
|
||||
this file would push the footprint over max_bytes. A download can't pre-check its size (unknown
|
||||
until the worker finishes) so check_enqueue only compares the current footprint; an upload can,
|
||||
so it's charged against the cap before it lands. Raises QuotaExceeded(max_jobs|max_bytes)."""
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return
|
||||
n = active_jobs(db, user_id)
|
||||
if n >= lim.max_jobs:
|
||||
raise QuotaExceeded("max_jobs", lim.max_jobs, n)
|
||||
projected = footprint(db, user_id) + add_bytes
|
||||
if projected > lim.max_bytes:
|
||||
raise QuotaExceeded("max_bytes", lim.max_bytes, projected)
|
||||
|
||||
|
||||
def usage(db: Session, user_id: int) -> dict:
|
||||
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
||||
lim = resolve(db, user_id)
|
||||
|
||||
@@ -17,7 +17,7 @@ from app.config import settings
|
||||
from app.downloads import adapters
|
||||
from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, quota, storage
|
||||
from app.downloads import formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
|
||||
@@ -216,6 +216,10 @@ def enqueue_edit(
|
||||
src_asset = db.get(MediaAsset, source_job.asset_id) if source_job.asset_id else None
|
||||
if src_asset is None or src_asset.status != "ready" or not src_asset.rel_path:
|
||||
raise EditError("source_not_ready")
|
||||
# A generic uploaded file (media_kind='file') has no timeline to trim/crop — the editor path is
|
||||
# media-only. Reject here so an upload's file never reaches the ffmpeg edit worker.
|
||||
if src_asset.media_kind == mediakind.FILE:
|
||||
raise EditError("not_media")
|
||||
|
||||
spec = editmod.normalize_edit_spec(edit_spec)
|
||||
if not spec:
|
||||
@@ -232,6 +236,9 @@ def enqueue_edit(
|
||||
asset.thumbnail_url = src_asset.thumbnail_url
|
||||
asset.upload_date = src_asset.upload_date
|
||||
asset.duration_s = editmod.clip_duration(spec, src_asset.duration_s)
|
||||
# A clip inherits its source's kind (a video edit stays video, an audio trim stays audio);
|
||||
# the ffmpeg edit worker doesn't re-probe codecs, so set it here where the source is known.
|
||||
asset.media_kind = src_asset.media_kind or mediakind.VIDEO
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
|
||||
@@ -147,6 +147,28 @@ def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
|
||||
return f".edits/{user_id}/clip_{asset_id}.{ext}"
|
||||
|
||||
|
||||
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
|
||||
"""On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
|
||||
|
||||
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
|
||||
episode sidecars). The name is system-decided and bound to the asset id (`_a{id}` keeps two
|
||||
same-named uploads apart); the original filename is display metadata only. The extension is
|
||||
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
|
||||
suffix; an extensionless upload stays extensionless."""
|
||||
stem = sanitize(Path(filename).stem, 80)
|
||||
ext = _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
|
||||
suffix = f".{ext}" if ext else ""
|
||||
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
|
||||
|
||||
|
||||
def upload_ext(filename: str) -> str | None:
|
||||
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
|
||||
the upload has no extension (a bare name)."""
|
||||
ext = _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
|
||||
return ext or None
|
||||
|
||||
|
||||
def download_filename(display_name: str, container: str | None, path: Path) -> str:
|
||||
"""The Content-Disposition filename for a served download: the cleaned display name carrying a single
|
||||
correct extension — the asset container, else the file's own suffix — without doubling it when the
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Local file ingest for the download center (E7 S5) — the non-yt-dlp path into the media library.
|
||||
|
||||
A user-uploaded file joins the SAME library as a downloaded one, so it inherits the whole function
|
||||
set already built (metadata chips, share links, and — for media — in-app play/edit). Unlike a
|
||||
download it bypasses yt-dlp AND the worker entirely: the file is already on the host, so ingest
|
||||
probes it, places it on disk, and creates an already-`ready` MediaAsset + `done` DownloadJob in one
|
||||
step — the same "the file exists, mark it done now" shortcut `service.enqueue` takes on a cache hit.
|
||||
|
||||
Kinds (the invariant lives in `mediakind`): a probe with a real video stream -> video, audio-only
|
||||
-> audio, neither -> a generic 'file' (pdf/zip/docx/…) that is download-only and never rendered
|
||||
inline (see the force-download serving in the file/watch routes). This module is the ingest seam a
|
||||
future sub-function (e.g. convert) plugs a new adapter into, mirroring how it sidesteps the worker.
|
||||
|
||||
An upload does NOT get an `expires_at`: it's user-provided and may be irreplaceable (there's no
|
||||
source to re-download), so it doesn't age out on the retention clock the way a re-fetchable download
|
||||
does — it lives until the user deletes it.
|
||||
"""
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def _probe(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
||||
or the file isn't media (a file we can't probe is treated as a generic non-media upload).
|
||||
|
||||
A video stream flagged `attached_pic` (embedded cover art in an mp3/m4a) is skipped — it's a
|
||||
thumbnail, not a real video track, so an audio file with album art still classifies as audio."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_streams",
|
||||
"-show_format",
|
||||
"-of",
|
||||
"json",
|
||||
str(path),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=True,
|
||||
).stdout
|
||||
data = json.loads(out)
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||
return None, None, None, None, None
|
||||
|
||||
vcodec = acodec = None
|
||||
width = height = None
|
||||
for stream in data.get("streams", []):
|
||||
kind = stream.get("codec_type")
|
||||
if kind == "video" and vcodec is None:
|
||||
if (stream.get("disposition") or {}).get("attached_pic"):
|
||||
continue # cover art, not a video track
|
||||
vcodec = stream.get("codec_name")
|
||||
width = stream.get("width")
|
||||
height = stream.get("height")
|
||||
elif kind == "audio" and acodec is None:
|
||||
acodec = stream.get("codec_name")
|
||||
|
||||
duration = None
|
||||
try:
|
||||
raw = (data.get("format") or {}).get("duration")
|
||||
if raw is not None:
|
||||
duration = int(float(raw))
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
return vcodec, acodec, width, height, duration
|
||||
|
||||
|
||||
def _title_from_filename(filename: str) -> str:
|
||||
"""A human display title from the original upload name: the stem with separators tidied to
|
||||
spaces (kept as-is otherwise — this is display metadata, not the shell-safe on-disk name)."""
|
||||
stem = Path(filename).stem.replace("_", " ").replace(".", " ").strip()
|
||||
return (stem or "Upload")[:255]
|
||||
|
||||
|
||||
def ingest_upload(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
*,
|
||||
filename: str,
|
||||
src_path: Path,
|
||||
size: int,
|
||||
display_name: str | None = None,
|
||||
) -> DownloadJob:
|
||||
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return the `done`
|
||||
job. `src_path` is MOVED into place on success (the caller stages it to a temp file first).
|
||||
|
||||
Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a
|
||||
rejected upload leaves nothing behind and the caller just cleans up its temp file."""
|
||||
quota.check_upload(db, user_id, size) # the file's size is known → charge it up front
|
||||
|
||||
vcodec, acodec, width, height, duration = _probe(src_path)
|
||||
kind = mediakind.derive(vcodec, acodec)
|
||||
|
||||
# An upload never dedups (each file is its own asset): a random ref makes the cache key unique,
|
||||
# so `format_sig` is a constant marker rather than a format hash.
|
||||
ref = f"{user_id}:{secrets.token_hex(8)}"[:512]
|
||||
asset = MediaAsset(
|
||||
source_kind="upload", source_ref=ref, format_sig="upload", status="pending", ref_count=0
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush() # need asset.id for the on-disk path
|
||||
|
||||
rel = storage.upload_rel_path(user_id, asset.id, filename)
|
||||
storage.place_file(src_path, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
actual_size = final.stat().st_size if final.exists() else size
|
||||
|
||||
asset.status = "ready"
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = actual_size
|
||||
asset.container = storage.upload_ext(filename)
|
||||
asset.vcodec = vcodec
|
||||
asset.acodec = acodec
|
||||
asset.width = width
|
||||
asset.height = height
|
||||
asset.duration_s = duration
|
||||
asset.media_kind = kind
|
||||
asset.title = (display_name or "").strip() or _title_from_filename(filename)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
# No expires_at — see the module docstring (an upload is irreplaceable, so it doesn't age out).
|
||||
if kind == mediakind.VIDEO:
|
||||
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has no
|
||||
# frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes server
|
||||
# side even for a codec the browser can't play).
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind="upload",
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=(display_name or "").strip() or None,
|
||||
status="done",
|
||||
progress=100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
service.incr_ref_count(db, asset.id)
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
return job
|
||||
@@ -708,6 +708,12 @@ class MediaAsset(Base, TimestampMixin):
|
||||
container: Mapped[str | None] = mapped_column(String(16))
|
||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
|
||||
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
|
||||
# still pending (kind is known once the file is probed at finalize). It is the single stored home
|
||||
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
|
||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
|
||||
gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
@@ -6,11 +6,14 @@ worker is a separate process); this module never downloads — it enqueues, mana
|
||||
serves finished files (range-aware, with the user's custom display name), and shares them.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
@@ -23,7 +26,9 @@ from app.db import get_db
|
||||
from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import mediakind
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
@@ -123,6 +128,9 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"width": asset.width,
|
||||
"height": asset.height,
|
||||
"container": asset.container,
|
||||
# "video" | "audio" | "file" — the library gates play/edit on this (media only); a
|
||||
# generic uploaded file is download-only. Null while still in flight (set at finalize).
|
||||
"media_kind": asset.media_kind,
|
||||
"uploader_url": asset.uploader_url,
|
||||
"thumbnail_url": asset.thumbnail_url,
|
||||
"expires_at": iso(asset.expires_at),
|
||||
@@ -354,6 +362,61 @@ def enqueue_download(
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
# Read the upload in 1 MiB chunks so a large file never sits fully in memory, and the size cap is
|
||||
# enforced on the ACTUAL bytes read (a client-set Content-Length can't be trusted to gate it).
|
||||
_UPLOAD_CHUNK = 1024 * 1024
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
def upload_download(
|
||||
file: UploadFile = File(...),
|
||||
display_name: str | None = Form(None),
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The file is streamed
|
||||
to a temp path under the download volume (enforcing the per-file cap as it goes), then handed to
|
||||
the non-yt-dlp ingest which probes + places it and returns an already-`done` job.
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. `file.file` is the sync file object
|
||||
Starlette spools the multipart body into."""
|
||||
cap = sysconfig.get_int(db, "download_upload_max_bytes")
|
||||
staging = Path(settings.download_root) / ".staging" / "uploads"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=str(staging))
|
||||
tmp_path = Path(tmp)
|
||||
try:
|
||||
size = 0
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
while chunk := file.file.read(_UPLOAD_CHUNK):
|
||||
size += len(chunk)
|
||||
if cap and size > cap:
|
||||
raise HTTPException(
|
||||
status_code=422, detail={"code": "upload_too_large", "limit": cap}
|
||||
)
|
||||
out.write(chunk)
|
||||
if size == 0:
|
||||
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||
try:
|
||||
job = uploadmod.ingest_upload(
|
||||
db,
|
||||
user.id,
|
||||
filename=file.filename or "upload",
|
||||
src_path=tmp_path,
|
||||
size=size,
|
||||
display_name=(display_name or "").strip() or None,
|
||||
)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||
return _serialize_job(db, job)
|
||||
finally:
|
||||
# Ingest MOVES the temp file into place on success (unlink is then a no-op); on any early exit
|
||||
# — cap hit, empty, quota, probe/move error — this removes the abandoned staging file.
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class EnqueueEditIn(BaseModel):
|
||||
source_job_id: int = 0
|
||||
edit_spec: dict
|
||||
@@ -544,8 +607,10 @@ def redownload_download(
|
||||
never deletes a file another job/user still shares: the old asset is only removed if THIS job
|
||||
was its last holder."""
|
||||
job = _own_job(db, user, job_id)
|
||||
if job.job_kind == "edit":
|
||||
raise HTTPException(status_code=409, detail="Editor clips can't be re-downloaded.")
|
||||
if job.job_kind in ("edit", "upload"):
|
||||
# Neither has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source. Re-running yt-dlp against source_kind='upload'/'edit' would just fail.
|
||||
raise HTTPException(status_code=409, detail="This item can't be re-downloaded.")
|
||||
if job.status in ("queued", "running", "paused"):
|
||||
raise HTTPException(status_code=409, detail="This download is still in progress.")
|
||||
service.redownload(db, job)
|
||||
@@ -610,7 +675,16 @@ def download_file(
|
||||
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
# Starlette's FileResponse honours the Range header (206 partial content) for seek/resume.
|
||||
# A generic uploaded file is served as a neutral, force-download attachment — never with a
|
||||
# sniffable content-type — so uploaded HTML/SVG can't execute inline on this origin. Media keeps
|
||||
# its guessed type so the in-app player streams it. Both are range-aware (FileResponse → 206).
|
||||
if asset.media_kind == mediakind.FILE:
|
||||
return FileResponse(
|
||||
path,
|
||||
filename=filename,
|
||||
media_type="application/octet-stream",
|
||||
content_disposition_type="attachment",
|
||||
)
|
||||
return FileResponse(path, filename=filename)
|
||||
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.downloads import mediakind, storage
|
||||
from app.models import DownloadJob, DownloadLink, MediaAsset
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.security import verify_password
|
||||
@@ -122,8 +122,18 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
||||
raise HTTPException(status_code=404, detail="This video is no longer available.")
|
||||
|
||||
filename = storage.download_filename(asset.title or "video", asset.container, path)
|
||||
# A generic uploaded file (media_kind='file') is ALWAYS a force-download attachment with a
|
||||
# neutral type — never served inline — so arbitrary uploaded HTML/SVG can't execute on this
|
||||
# login-free public origin. Media follows the link's own inline/attachment choice so it can play
|
||||
# in the watch page. Both are range-aware (FileResponse → 206) for seeking/resume.
|
||||
if asset.media_kind == mediakind.FILE:
|
||||
return FileResponse(
|
||||
path,
|
||||
filename=filename,
|
||||
media_type="application/octet-stream",
|
||||
content_disposition_type="attachment",
|
||||
)
|
||||
disposition = "attachment" if link.allow_download else "inline"
|
||||
# Starlette's FileResponse honours the Range header (206) for seeking/scrubbing.
|
||||
return FileResponse(path, filename=filename, content_disposition_type=disposition)
|
||||
|
||||
|
||||
|
||||
@@ -127,6 +127,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
min=1,
|
||||
max=100_000,
|
||||
),
|
||||
ConfigSpec("download_upload_max_bytes", "int", "downloads", "download_upload_max_bytes", min=0),
|
||||
ConfigSpec(
|
||||
"download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650
|
||||
),
|
||||
|
||||
@@ -43,6 +43,7 @@ pydantic-settings==2.15.0
|
||||
pydantic==2.13.4
|
||||
pydantic_core==2.46.4
|
||||
python-dotenv==1.2.3
|
||||
python-multipart==0.0.32
|
||||
requests==2.34.2
|
||||
starlette==1.6.0
|
||||
typing-inspection==0.4.4
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
fastapi>=0.115,<1.0
|
||||
# Multipart form parsing — required by Starlette/FastAPI to accept file uploads (UploadFile/Form).
|
||||
# The download center's local-file Upload (E7) is the only multipart endpoint in the app.
|
||||
python-multipart>=0.0.9,<1.0
|
||||
uvicorn[standard]>=0.30,<1.0
|
||||
sqlalchemy>=2.0,<2.1
|
||||
psycopg[binary]>=3.2,<4.0
|
||||
|
||||
@@ -65,6 +65,9 @@ def _asset(**over):
|
||||
height=720,
|
||||
vcodec="h264",
|
||||
acodec="aac",
|
||||
media_kind="video",
|
||||
size_bytes=1000,
|
||||
container="mp4",
|
||||
source_webpage_url=None,
|
||||
source_kind="url",
|
||||
)
|
||||
@@ -74,12 +77,13 @@ def _asset(**over):
|
||||
|
||||
class TestPublicMetaAudioFlag:
|
||||
"""The watch page picks <audio> vs <video> from this flag; WebKit refuses an audio-only mp3 in a
|
||||
<video>, so an audio-only asset (codec present, no video stream) must report audio=True."""
|
||||
<video>, so an audio-only asset (media_kind='audio') must report audio=True. The flag now reads
|
||||
the stored media_kind (the single source), not a re-derived codec check."""
|
||||
|
||||
_link = SimpleNamespace(allow_download=True)
|
||||
|
||||
def test_audio_only_asset_is_flagged_audio(self):
|
||||
asset = _asset(vcodec=None, acodec="mp3", width=None, height=None)
|
||||
asset = _asset(media_kind="audio", vcodec=None, acodec="mp3", width=None, height=None)
|
||||
meta = public_meta(self._link, None, asset, "/f.mp3")
|
||||
assert meta["audio"] is True
|
||||
|
||||
@@ -87,10 +91,11 @@ class TestPublicMetaAudioFlag:
|
||||
meta = public_meta(self._link, None, _asset(), "/f.mp4")
|
||||
assert meta["audio"] is False
|
||||
|
||||
def test_a_silent_video_is_not_audio(self):
|
||||
# Video present but no audio track — still a video player, not the audio bar.
|
||||
meta = public_meta(self._link, None, _asset(acodec=None), "/f.mp4")
|
||||
def test_a_generic_file_is_not_audio(self):
|
||||
# A non-media upload (media_kind='file') has no player at all — never the audio bar.
|
||||
meta = public_meta(self._link, None, _asset(media_kind="file"), "/f.bin")
|
||||
assert meta["audio"] is False
|
||||
assert meta["media_kind"] == "file"
|
||||
|
||||
|
||||
class TestCreateLinkDefault:
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
"""E7 S5a — local file upload ingest (the non-yt-dlp path into the media library).
|
||||
|
||||
Covers the ingest spine: media vs generic-file classification, the download-only serving of a
|
||||
generic file (force-download so uploaded HTML/SVG can't execute inline), quota accounting with the
|
||||
size known up front, the per-file cap enforced while streaming, filename sanitization, and the two
|
||||
guards that keep an upload out of paths that assume a re-fetchable source (edit / redownload).
|
||||
|
||||
DB-backed (real Postgres) — skipped when no database is reachable. `settings.download_root` is
|
||||
pointed at a per-test tmp dir so ingest's on-disk moves are isolated; `_probe` is monkeypatched so
|
||||
the tests don't depend on ffprobe (its real behaviour is exercised by the download/edit suites).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import mediakind, quota, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.models import DownloadQuota, MediaAsset
|
||||
|
||||
|
||||
# --- pure: the kind invariant ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_derive_media_kind():
|
||||
assert uploadmod.mediakind.derive("h264", "aac") == mediakind.VIDEO
|
||||
assert uploadmod.mediakind.derive("vp9", None) == mediakind.VIDEO
|
||||
assert uploadmod.mediakind.derive(None, "aac") == mediakind.AUDIO
|
||||
assert uploadmod.mediakind.derive(None, None) == mediakind.FILE
|
||||
|
||||
|
||||
# --- ingest (direct) ------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _root(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _probe(monkeypatch, result):
|
||||
monkeypatch.setattr(uploadmod, "_probe", lambda _p: result)
|
||||
|
||||
|
||||
def test_ingest_generic_file_is_download_only(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
u = make_user()
|
||||
src = _root / "src.bin"
|
||||
src.write_bytes(b"%PDF-1.4 hello")
|
||||
job = uploadmod.ingest_upload(
|
||||
db, u.id, filename="My Notes.pdf", src_path=src, size=src.stat().st_size
|
||||
)
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
assert asset.media_kind == mediakind.FILE
|
||||
assert asset.status == "ready" and job.status == "done" and job.progress == 100
|
||||
assert asset.container == "pdf"
|
||||
assert asset.poster_path is None # a generic file has no player and no poster
|
||||
assert asset.title == "My Notes" # display title from the original name
|
||||
assert job.job_kind == "upload" and job.source_kind == "upload"
|
||||
# The file was MOVED into the per-user uploads tree, inside the root, and the temp is gone.
|
||||
assert not src.exists()
|
||||
assert asset.rel_path.startswith(".uploads/")
|
||||
assert storage.safe_abs_path(str(_root), asset.rel_path) is not None
|
||||
|
||||
|
||||
def test_ingest_audio_is_media_no_poster(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, (None, "aac", None, None, 200))
|
||||
u = make_user()
|
||||
src = _root / "song.m4a"
|
||||
src.write_bytes(b"fake-audio")
|
||||
job = uploadmod.ingest_upload(db, u.id, filename="song.m4a", src_path=src, size=10)
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
assert asset.media_kind == mediakind.AUDIO
|
||||
assert asset.duration_s == 200
|
||||
assert asset.poster_path is None # audio has no frame to poster
|
||||
|
||||
|
||||
def test_ingest_video_records_dims_and_poster(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, ("h264", "aac", 640, 480, 60))
|
||||
# Stub the frame-cut so the test doesn't shell out to ffmpeg.
|
||||
monkeypatch.setattr(uploadmod.editmod, "ensure_poster", lambda _a, _root: "p/x.jpg")
|
||||
u = make_user()
|
||||
src = _root / "clip.mp4"
|
||||
src.write_bytes(b"fake-video")
|
||||
job = uploadmod.ingest_upload(db, u.id, filename="clip.mp4", src_path=src, size=10)
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
assert asset.media_kind == mediakind.VIDEO
|
||||
assert (asset.width, asset.height) == (640, 480)
|
||||
assert asset.poster_path == "p/x.jpg"
|
||||
|
||||
|
||||
def test_ingest_filename_is_sanitized_no_traversal(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
u = make_user()
|
||||
src = _root / "src"
|
||||
src.write_bytes(b"x")
|
||||
job = uploadmod.ingest_upload(db, u.id, filename="../../etc/passwd", src_path=src, size=1)
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
assert ".." not in asset.rel_path
|
||||
assert asset.rel_path.startswith(f".uploads/{u.id}/")
|
||||
assert storage.safe_abs_path(str(_root), asset.rel_path) is not None
|
||||
|
||||
|
||||
def test_ingest_rejects_over_footprint_cap_before_writing(db, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
u = make_user()
|
||||
db.add(
|
||||
DownloadQuota(user_id=u.id, max_bytes=1000, max_concurrent=2, max_jobs=50, unlimited=False)
|
||||
)
|
||||
db.commit()
|
||||
src = _root / "big.bin"
|
||||
src.write_bytes(b"x" * 10)
|
||||
with pytest.raises(quota.QuotaExceeded) as e:
|
||||
uploadmod.ingest_upload(db, u.id, filename="big.bin", src_path=src, size=2000)
|
||||
assert e.value.reason == "max_bytes"
|
||||
assert src.exists() # rejected before any disk move — nothing landed
|
||||
assert db.query(MediaAsset).count() == 0 # and no orphan asset row
|
||||
|
||||
|
||||
# --- route ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_upload_route_generic_forces_download(as_user, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
c = as_user(make_user())
|
||||
r = c.post(
|
||||
"/api/downloads/upload",
|
||||
files={"file": ("report.html", b"<script>alert(1)</script>", "text/html")},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
job = r.json()
|
||||
assert job["asset"]["media_kind"] == "file"
|
||||
assert job["can_download"] is True
|
||||
# The served file must be a neutral force-download attachment — never inline text/html.
|
||||
fr = c.get(f"/api/downloads/{job['id']}/file")
|
||||
assert fr.status_code == 200
|
||||
assert fr.headers["content-type"].startswith("application/octet-stream")
|
||||
assert "attachment" in fr.headers["content-disposition"].lower()
|
||||
|
||||
|
||||
def test_upload_route_enforces_per_file_cap(as_user, make_user, db, _root, monkeypatch):
|
||||
from app import sysconfig
|
||||
|
||||
sysconfig.set_value(db, "download_upload_max_bytes", 5)
|
||||
db.commit()
|
||||
c = as_user(make_user())
|
||||
r = c.post(
|
||||
"/api/downloads/upload",
|
||||
files={"file": ("big.bin", b"way over the cap", "application/octet-stream")},
|
||||
)
|
||||
assert r.status_code == 422
|
||||
assert r.json()["detail"]["code"] == "upload_too_large"
|
||||
|
||||
|
||||
def test_upload_route_rejects_empty_file(as_user, make_user, _root, monkeypatch):
|
||||
c = as_user(make_user())
|
||||
r = c.post(
|
||||
"/api/downloads/upload", files={"file": ("empty.bin", b"", "application/octet-stream")}
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert r.json()["detail"]["code"] == "upload_empty"
|
||||
|
||||
|
||||
def test_redownload_rejects_an_upload(as_user, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
c = as_user(make_user())
|
||||
job = c.post(
|
||||
"/api/downloads/upload", files={"file": ("a.bin", b"data", "application/octet-stream")}
|
||||
).json()
|
||||
r = c.post(f"/api/downloads/{job['id']}/redownload")
|
||||
assert r.status_code == 409
|
||||
|
||||
|
||||
def test_edit_rejects_a_generic_upload_source(as_user, make_user, _root, monkeypatch):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
c = as_user(make_user())
|
||||
job = c.post(
|
||||
"/api/downloads/upload", files={"file": ("a.bin", b"data", "application/octet-stream")}
|
||||
).json()
|
||||
r = c.post(
|
||||
"/api/downloads/edit",
|
||||
json={"source_job_id": job["id"], "edit_spec": {"trim": {"start_s": 0, "end_s": 1}}},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert r.json()["detail"]["reason"] == "not_media"
|
||||
Reference in New Issue
Block a user