fix(downloads): harden upload serving + ingest against stored-XSS and leaks (E7 review)

Deep-review remediation for the E7 upload path, all findings adversarially verified:

- Serve every library file with a Content-Type chosen from the ffprobe-verified media_kind
  + a strict container allow-list (storage.served_media_type), never guessed from the
  user-controlled file extension. Closes a stored-XSS vector where a media/HTML polyglot
  uploaded as "evil.html" was served inline as text/html on the login-free /watch surface.
  Applied to both the authenticated /file and the public watch_file endpoints.
- Classify uploaded still images (jpg/png/gif/webp/…) as media_kind='file' so they are
  force-downloaded and don't get a broken in-app video player.
- Roll back the on-disk side of an upload when the ingest transaction fails after place_file
  (unlink the placed file + its deterministic poster path), so a failed commit can't orphan
  bytes in the GC-skipped .uploads tree.
- Floor download_upload_max_bytes at min=1 so a stored 0 can't silently disable the per-file cap.
- Dedupe the ffprobe-JSON invocation into edit.run_ffprobe_json (shared by probe_codecs and
  upload._probe) and the upload extension into storage._clean_upload_ext; drop the now-unused
  parse_probe_streams.

Tests: public + authenticated force-download and the polyglot-not-served-as-HTML guard, the
real _probe classifier (image/cover-art/dims/duration), the ingest rollback/cleanup path, the
max_jobs branch, the cap-floor invariant, and run_ffprobe_json error handling.
This commit is contained in:
2026-08-29 21:52:02 +02:00
parent 104bd1e3a6
commit 614c5528ad
8 changed files with 475 additions and 128 deletions
+38 -30
View File
@@ -438,18 +438,42 @@ _BROWSER_SAFE_VCODEC = ("h264",)
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
"""(video_codec, audio_codec) out of ffprobe's JSON stream list, lowercased.
JSON, not the flat `default=nw=1` output: there the fields arrive in ffprobe's own struct
order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line "remember the last
codec_type" parser attributes every name to the PREVIOUS stream. Measured on real downloads:
an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
believed the video stream was AAC and re-encoded a perfectly fine file on every download."""
def run_ffprobe_json(
path: Path,
*,
show_streams: bool = False,
show_format: bool = False,
entries: str | None = None,
) -> dict | None:
"""One ffprobe call → its parsed JSON dict, or None on any failure (timeout, non-zero exit,
missing binary, unparseable output). The SINGLE ffprobe-JSON-invocation point in the downloads
package: the browser-playability codec probe here and the upload-ingest probe (`upload._probe`)
both go through it, so the subprocess flags/timeout/error handling live in one place instead of
two drifting copies."""
args = ["ffprobe", "-v", "error"]
if show_streams:
args.append("-show_streams")
if show_format:
args.append("-show_format")
if entries:
args += ["-show_entries", entries]
args += ["-of", "json", str(path)]
try:
streams = json.loads(payload or "{}").get("streams") or []
except ValueError:
return None, None
out = subprocess.run(args, capture_output=True, text=True, timeout=30, check=True).stdout
return json.loads(out)
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
return None
def _codecs_from_streams(streams: list) -> tuple[str | None, str | None]:
"""(video_codec, audio_codec) lowercased from an ffprobe JSON stream list — the first of each kind.
Parsed from the JSON `streams` list, NOT ffprobe's flat `default=nw=1` output: there the fields
arrive in ffprobe's own struct order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line
"remember the last codec_type" parser attributes every name to the PREVIOUS stream. Measured on real
downloads: an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
believed the video stream was AAC and re-encoded a perfectly fine file on every download. Reading each
stream dict by its own `codec_type` is immune to that ordering."""
vcodec = acodec = None
for s in streams:
name = (s.get("codec_name") or "").lower() or None
@@ -464,26 +488,10 @@ def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
try:
out = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-show_entries",
"stream=codec_type,codec_name",
"-of",
"json",
str(path),
],
capture_output=True,
text=True,
timeout=30,
check=True,
).stdout
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
data = run_ffprobe_json(path, entries="stream=codec_type,codec_name")
if data is None:
return None, None
return parse_probe_streams(out)
return _codecs_from_streams(data.get("streams") or [])
def probe_duration(path: Path) -> int:
+59 -4
View File
@@ -19,6 +19,8 @@ from datetime import date
from pathlib import Path
from xml.sax.saxutils import escape
from app.downloads import mediakind
# Reserve room for the "_id.ext" suffix + parent dirs within the 255-char component limit.
_MAX_TITLE = 120
# User-facing (Content-Disposition) names can be longer than an on-disk component.
@@ -147,6 +149,14 @@ def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
return f".edits/{user_id}/clip_{asset_id}.{ext}"
def _clean_upload_ext(filename: str) -> str:
"""The upload's extension reduced to a shell-safe, lowercased, bounded token (``""`` when the
upload has no extension). The SINGLE definition feeding both the on-disk suffix and the stored
`container` — they must agree (the served Content-Disposition suffix has to match the container),
so the sanitize/truncation rule lives in one place, not two drifting copies."""
return _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
"""On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
@@ -157,16 +167,61 @@ def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
suffix; an extensionless upload stays extensionless."""
stem = sanitize(Path(filename).stem, 80)
ext = _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
ext = _clean_upload_ext(filename)
suffix = f".{ext}" if ext else ""
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
def upload_ext(filename: str) -> str | None:
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
the upload has no extension (a bare name)."""
ext = _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
return ext or None
the upload has no extension (a bare name). Shares `_clean_upload_ext` with `upload_rel_path` so the
stored container and the on-disk suffix can never diverge."""
return _clean_upload_ext(filename) or None
# The Content-Type used to serve an inline-playable media asset, keyed by its stored `container`.
# DELIBERATELY a strict allow-list of known audio/video containers: the served type is chosen from
# the asset's ffprobe-verified media_kind + this table, NEVER guessed from the raw (user-controlled)
# upload extension. Without this, an uploaded media/HTML polyglot named "evil.html" — a real MP3/MP4
# that ffprobe classifies as media, so it dodges the generic-file force-download — would be served
# with a Content-Type Starlette guesses from ".html" (text/html) and execute as stored XSS on the
# serving origin. A media asset whose container isn't in this table falls back to a neutral
# force-download attachment (see `served_media_type`), so nothing script-executable is ever inline.
_MEDIA_MIME: dict[str, str] = {
# video containers
"mp4": "video/mp4",
"m4v": "video/mp4",
"mov": "video/quicktime",
"webm": "video/webm",
"mkv": "video/x-matroska",
"ogv": "video/ogg",
"avi": "video/x-msvideo",
# audio containers
"m4a": "audio/mp4",
"mp3": "audio/mpeg",
"aac": "audio/aac",
"ogg": "audio/ogg",
"oga": "audio/ogg",
"opus": "audio/ogg",
"weba": "audio/webm",
"wav": "audio/wav",
"flac": "audio/flac",
}
def served_media_type(media_kind: str | None, container: str | None) -> str | None:
"""The Content-Type a library file MUST be served with — a real audio/video MIME for a recognised
media container, else ``None`` meaning "serve as a neutral force-download attachment".
Returns None for a generic ``file`` kind AND for any media whose container we don't positively
recognise as safe. Callers serve None as ``application/octet-stream`` + ``attachment`` so the
served Content-Type is NEVER derived from the (attacker-controlled) file extension — the guard
that stops uploaded HTML/SVG or a media/HTML polyglot from executing inline on the serving
origin. A ``None`` media_kind (a legacy asset before the discriminator was backfilled) is treated
as media so an old download keeps its inline player."""
if media_kind == mediakind.FILE:
return None
return _MEDIA_MIME.get((container or "").lower())
def download_filename(display_name: str, container: str | None, path: Path) -> str:
+83 -58
View File
@@ -16,9 +16,7 @@ source to re-download), so it doesn't age out on the retention clock the way a r
does — it lives until the user deletes it.
"""
import json
import secrets
import subprocess
from datetime import datetime, timezone
from pathlib import Path
@@ -29,32 +27,43 @@ from app.downloads import edit as editmod
from app.downloads import mediakind, quota, service, storage
from app.models import DownloadJob, MediaAsset
# ffprobe container formats that mean "this is a still image / image sequence", NOT playable media.
# A standalone jpg/png/gif/webp/bmp reports a real video stream (codec mjpeg/png/…) with no
# `attached_pic` flag, so without this guard it would classify as media_kind='video' and get a Play
# button + a broken in-app <video> player (and inline serving on the public share). Keyed on the
# container format so an animated gif/webp also lands as a generic 'file' (no browser plays those in
# a <video> either). SVG and anything else ffprobe can't decode already fails the probe → 'file'.
_IMAGE_FORMATS = {
"image2",
"image2pipe",
"png_pipe",
"mjpeg",
"mjpeg_pipe",
"jpeg_pipe",
"gif",
"webp_pipe",
"bmp_pipe",
"tiff_pipe",
"ico",
"svg_pipe",
}
def _probe(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
or the file isn't media (a file we can't probe is treated as a generic non-media upload).
or the file isn't playable media (an unprobeable file — or a still image — is treated as a generic
non-media upload, so it becomes a download-only 'file' rather than broken 'video').
A video stream flagged `attached_pic` (embedded cover art in an mp3/m4a) is skipped — it's a
thumbnail, not a real video track, so an audio file with album art still classifies as audio."""
try:
out = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-show_streams",
"-show_format",
"-of",
"json",
str(path),
],
capture_output=True,
text=True,
timeout=30,
check=True,
).stdout
data = json.loads(out)
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
if data is None:
return None, None, None, None, None
fmt = data.get("format") or {}
fmt_names = {n.strip() for n in (fmt.get("format_name") or "").split(",")}
if fmt_names & _IMAGE_FORMATS:
# A still image / image sequence is a generic file, not video — no vcodec means media_kind='file'.
return None, None, None, None, None
vcodec = acodec = None
@@ -72,7 +81,7 @@ def _probe(path: Path) -> tuple[str | None, str | None, int | None, int | None,
duration = None
try:
raw = (data.get("format") or {}).get("duration")
raw = fmt.get("duration")
if raw is not None:
duration = int(float(raw))
except (TypeError, ValueError):
@@ -117,42 +126,58 @@ def ingest_upload(
rel = storage.upload_rel_path(user_id, asset.id, filename)
storage.place_file(src_path, settings.download_root, rel)
final = storage.abs_path(settings.download_root, rel)
actual_size = final.stat().st_size if final.exists() else size
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
# asset/job rows roll back but the placed file would be ORPHANED — nothing references it, the
# per-user delete path keys off asset.rel_path (which is gone), and the GC orphan sweep skips the
# dot-prefixed `.uploads/` tree. So roll back the on-disk side too on any failure.
try:
final = storage.abs_path(settings.download_root, rel)
actual_size = final.stat().st_size if final.exists() else size
asset.status = "ready"
asset.rel_path = rel
asset.size_bytes = actual_size
asset.container = storage.upload_ext(filename)
asset.vcodec = vcodec
asset.acodec = acodec
asset.width = width
asset.height = height
asset.duration_s = duration
asset.media_kind = kind
asset.title = (display_name or "").strip() or _title_from_filename(filename)
asset.last_access_at = datetime.now(timezone.utc)
# No expires_at — see the module docstring (an upload is irreplaceable, so it doesn't age out).
if kind == mediakind.VIDEO:
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has no
# frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes server
# side even for a codec the browser can't play).
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
asset.status = "ready"
asset.rel_path = rel
asset.size_bytes = actual_size
asset.container = storage.upload_ext(filename)
asset.vcodec = vcodec
asset.acodec = acodec
asset.width = width
asset.height = height
asset.duration_s = duration
asset.media_kind = kind
asset.title = (display_name or "").strip() or _title_from_filename(filename)
asset.last_access_at = datetime.now(timezone.utc)
# No expires_at — see the module docstring (an upload is irreplaceable, so it doesn't age out).
if kind == mediakind.VIDEO:
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
# server side even for a codec the browser can't play).
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
job = DownloadJob(
user_id=user_id,
asset_id=asset.id,
job_kind="upload",
source_kind="upload",
source_ref=ref,
profile_snapshot={},
display_name=(display_name or "").strip() or None,
status="done",
progress=100,
queue_pos=service.next_queue_pos(db),
)
db.add(job)
service.incr_ref_count(db, asset.id)
db.commit()
job = DownloadJob(
user_id=user_id,
asset_id=asset.id,
job_kind="upload",
source_kind="upload",
source_ref=ref,
profile_snapshot={},
display_name=(display_name or "").strip() or None,
status="done",
progress=100,
queue_pos=service.next_queue_pos(db),
)
db.add(job)
service.incr_ref_count(db, asset.id)
db.commit()
except Exception:
db.rollback()
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
# ffmpeg failure, so a partial poster can be on disk while the return is None — gating on the
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
# when none was generated (audio/file, or a video whose poster never started).
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
raise
db.refresh(job)
return job
+9 -6
View File
@@ -26,7 +26,6 @@ from app.db import get_db
from app.downloads import credentials as dlcreds
from app.downloads import edit as editmod
from app.downloads import links as linksmod
from app.downloads import mediakind
from app.downloads import quota, service, storage
from app.downloads import upload as uploadmod
from app.models import (
@@ -675,17 +674,21 @@ def download_file(
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
# A generic uploaded file is served as a neutral, force-download attachment — never with a
# sniffable content-type — so uploaded HTML/SVG can't execute inline on this origin. Media keeps
# its guessed type so the in-app player streams it. Both are range-aware (FileResponse → 206).
if asset.media_kind == mediakind.FILE:
# The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
# (storage.served_media_type), NEVER guessed from the file's own extension. A generic 'file' (and
# any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html")
# returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on
# this origin. Recognised media is served with its real type so the in-app player streams it. Both
# are range-aware (FileResponse → 206).
media_type = storage.served_media_type(asset.media_kind, asset.container)
if media_type is None:
return FileResponse(
path,
filename=filename,
media_type="application/octet-stream",
content_disposition_type="attachment",
)
return FileResponse(path, filename=filename)
return FileResponse(path, filename=filename, media_type=media_type)
# --- editor filmstrip (scrub track) --------------------------------------------------------
+12 -6
View File
@@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
from app.config import settings
from app.db import get_db
from app.downloads import links as linksmod
from app.downloads import mediakind, storage
from app.downloads import storage
from app.models import DownloadJob, DownloadLink, MediaAsset
from app.ratelimit import RateLimiter
from app.security import verify_password
@@ -122,11 +122,15 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
raise HTTPException(status_code=404, detail="This video is no longer available.")
filename = storage.download_filename(asset.title or "video", asset.container, path)
# A generic uploaded file (media_kind='file') is ALWAYS a force-download attachment with a
# neutral type — never served inline — so arbitrary uploaded HTML/SVG can't execute on this
# login-free public origin. Media follows the link's own inline/attachment choice so it can play
# The served Content-Type comes from the asset's ffprobe-verified media_kind + a container
# allow-list (storage.served_media_type) — NEVER guessed from the file's own extension. A generic
# 'file' (and any media whose container we don't positively recognise, e.g. a media/HTML polyglot
# named "evil.html") returns None → a neutral force-download attachment, so arbitrary uploaded
# HTML/SVG can't execute inline on this login-free public origin. A recognised media container is
# served with its real audio/video type, inline unless the link forces a download so it can play
# in the watch page. Both are range-aware (FileResponse → 206) for seeking/resume.
if asset.media_kind == mediakind.FILE:
media_type = storage.served_media_type(asset.media_kind, asset.container)
if media_type is None:
return FileResponse(
path,
filename=filename,
@@ -134,7 +138,9 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
content_disposition_type="attachment",
)
disposition = "attachment" if link.allow_download else "inline"
return FileResponse(path, filename=filename, content_disposition_type=disposition)
return FileResponse(
path, filename=filename, media_type=media_type, content_disposition_type=disposition
)
@router.get("/watch/{token}/poster.jpg")
+4 -1
View File
@@ -127,7 +127,10 @@ SPECS: tuple[ConfigSpec, ...] = (
min=1,
max=100_000,
),
ConfigSpec("download_upload_max_bytes", "int", "downloads", "download_upload_max_bytes", min=0),
# min=1, NOT 0: the per-file cap is enforced as `if cap and size > cap`, so a stored 0 would read
# as falsy and SILENTLY disable the limit (an unbounded streamed write to the volume). Floor it at
# 1 so the cap is always active; the overall footprint is separately bounded by the byte quota.
ConfigSpec("download_upload_max_bytes", "int", "downloads", "download_upload_max_bytes", min=1),
ConfigSpec(
"download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650
),
+58 -21
View File
@@ -6,7 +6,7 @@ with "PosixPath('.') has an empty name" instead of downloading."""
import pytest
from app.downloads.edit import _concat_quote, parse_probe_streams
from app.downloads.edit import _codecs_from_streams, _concat_quote, run_ffprobe_json
from app.downloads.pipeline import _media_info, _produced_file
from pathlib import Path, PurePosixPath
@@ -105,39 +105,76 @@ class TestMediaInfo:
assert _media_info(info)["id"] == "vid"
class TestParseProbeStreams:
class TestCodecsFromStreams:
# ffprobe emits codec_name BEFORE codec_type, so the old line parser shifted every name onto
# the previous stream and an H.264+AAC mp4 came back as (aac, None) → a needless re-encode.
# _codecs_from_streams reads each stream dict by its own codec_type, immune to that ordering.
def test_reads_each_stream_by_its_own_type(self):
payload = '{"streams":[{"codec_name":"h264","codec_type":"video"},{"codec_name":"aac","codec_type":"audio"}]}'
assert parse_probe_streams(payload) == ("h264", "aac")
streams = [
{"codec_name": "h264", "codec_type": "video"},
{"codec_name": "aac", "codec_type": "audio"},
]
assert _codecs_from_streams(streams) == ("h264", "aac")
def test_audio_first_ordering_is_handled(self):
payload = '{"streams":[{"codec_name":"AAC","codec_type":"audio"},{"codec_name":"H264","codec_type":"video"}]}'
assert parse_probe_streams(payload) == ("h264", "aac")
streams = [
{"codec_name": "AAC", "codec_type": "audio"},
{"codec_name": "H264", "codec_type": "video"},
]
assert _codecs_from_streams(streams) == ("h264", "aac")
def test_extra_streams_do_not_win_over_the_first_of_their_kind(self):
payload = (
'{"streams":[{"codec_name":"h264","codec_type":"video"},'
'{"codec_name":"vp9","codec_type":"video"},'
'{"codec_name":"mov_text","codec_type":"subtitle"},'
'{"codec_name":"opus","codec_type":"audio"}]}'
)
assert parse_probe_streams(payload) == ("h264", "opus")
streams = [
{"codec_name": "h264", "codec_type": "video"},
{"codec_name": "vp9", "codec_type": "video"},
{"codec_name": "mov_text", "codec_type": "subtitle"},
{"codec_name": "opus", "codec_type": "audio"},
]
assert _codecs_from_streams(streams) == ("h264", "opus")
def test_audio_only_and_video_only(self):
assert parse_probe_streams('{"streams":[{"codec_name":"opus","codec_type":"audio"}]}') == (
assert _codecs_from_streams([{"codec_name": "opus", "codec_type": "audio"}]) == (
None,
"opus",
)
assert parse_probe_streams('{"streams":[{"codec_name":"av1","codec_type":"video"}]}') == (
"av1",
None,
)
assert _codecs_from_streams([{"codec_name": "av1", "codec_type": "video"}]) == ("av1", None)
def test_garbage_is_not_a_crash(self):
assert parse_probe_streams("") == (None, None)
assert parse_probe_streams("not json") == (None, None)
def test_empty_stream_list(self):
assert _codecs_from_streams([]) == (None, None)
class TestRunFfprobeJson:
"""The single ffprobe-JSON invocation point (shared by probe_codecs + upload._probe). A failed
probe or unparseable output must degrade to None, not raise — the malformed-input safety the old
parse_probe_streams string parser used to own now lives here."""
def test_unparseable_output_returns_none(self, monkeypatch):
import subprocess
class _R:
stdout = "not json"
monkeypatch.setattr(subprocess, "run", lambda *a, **k: _R())
assert run_ffprobe_json(Path("x")) is None
def test_ffprobe_failure_returns_none(self, monkeypatch):
import subprocess
def _boom(*a, **k):
raise subprocess.CalledProcessError(1, "ffprobe")
monkeypatch.setattr(subprocess, "run", _boom)
assert run_ffprobe_json(Path("x")) is None
def test_parses_streams_from_valid_json(self, monkeypatch):
import subprocess
class _R:
stdout = '{"streams":[{"codec_name":"h264","codec_type":"video"}]}'
monkeypatch.setattr(subprocess, "run", lambda *a, **k: _R())
data = run_ffprobe_json(Path("x"), entries="stream=codec_type,codec_name")
assert _codecs_from_streams(data["streams"]) == ("h264", None)
class TestConcatQuote:
+212 -2
View File
@@ -6,10 +6,14 @@ size known up front, the per-file cap enforced while streaming, filename sanitiz
guards that keep an upload out of paths that assume a re-fetchable source (edit / redownload).
DB-backed (real Postgres) — skipped when no database is reachable. `settings.download_root` is
pointed at a per-test tmp dir so ingest's on-disk moves are isolated; `_probe` is monkeypatched so
the tests don't depend on ffprobe (its real behaviour is exercised by the download/edit suites).
pointed at a per-test tmp dir so ingest's on-disk moves are isolated. The ingest/route tests
monkeypatch `_probe` so they don't shell out to ffprobe; the classifier's OWN parsing (image →
file, cover-art skip, dims, duration) is covered directly by `TestProbeClassifier` below, which
stubs one level down (`run_ffprobe_json`) and exercises the real `_probe`.
"""
from pathlib import Path
import pytest
from app.config import settings
@@ -137,6 +141,77 @@ def test_upload_route_generic_forces_download(as_user, make_user, _root, monkeyp
assert "attachment" in fr.headers["content-disposition"].lower()
def test_upload_route_media_html_extension_not_served_as_html(
as_user, make_user, _root, monkeypatch
):
"""The XSS guard: a MEDIA-classified upload (real codecs) named with an executable extension must
NOT be served with a guessed text/html type. The served Content-Type comes from the verified
media_kind + a container allow-list ('html' isn't one), so it force-downloads as octet-stream —
on both the authenticated file endpoint AND the login-free public watch endpoint. Without the fix
the public inline branch let a media/HTML polyglot named evil.html run as stored XSS."""
_probe(monkeypatch, ("h264", "aac", 320, 240, 5)) # ffprobe says it's video…
c = as_user(make_user())
r = c.post(
"/api/downloads/upload",
files={"file": ("evil.html", b"<script>alert(1)</script>", "text/html")},
)
assert r.status_code == 200, r.text
job = r.json()
assert job["asset"]["media_kind"] == "video" and job["asset"]["container"] == "html"
# Authenticated endpoint: never text/html.
fr = c.get(f"/api/downloads/{job['id']}/file")
assert fr.status_code == 200
assert fr.headers["content-type"].startswith("application/octet-stream")
assert "attachment" in fr.headers["content-disposition"].lower()
# Public login-free endpoint via a STREAM-ONLY link (the inline branch) — still not text/html.
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json()
pr = c.get(f"/api/public/watch/{link['token']}/file")
assert pr.status_code == 200
assert not pr.headers["content-type"].startswith("text/html")
assert pr.headers["content-type"].startswith("application/octet-stream")
assert "attachment" in pr.headers["content-disposition"].lower()
def test_public_watch_file_forces_download_for_generic_file(as_user, make_user, _root, monkeypatch):
"""The login-free public serve of a generic upload (media_kind='file') must be a neutral
force-download attachment — the branch that stops uploaded HTML/SVG executing on the public
origin. This is a SEPARATE code path from the authenticated /file route, so it needs its own
route-level assertion (previously only the authenticated branch was tested)."""
_probe(monkeypatch, (None, None, None, None, None))
c = as_user(make_user())
job = c.post(
"/api/downloads/upload",
files={"file": ("notes.html", b"<script>alert(1)</script>", "text/html")},
).json()
assert job["asset"]["media_kind"] == "file"
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": True}).json()
pr = c.get(f"/api/public/watch/{link['token']}/file")
assert pr.status_code == 200
assert pr.headers["content-type"].startswith("application/octet-stream")
assert "attachment" in pr.headers["content-disposition"].lower()
def test_public_watch_file_serves_media_inline_with_real_type(
as_user, make_user, _root, monkeypatch
):
"""A recognised media container IS still served inline with its real audio/video type (so it
plays in the watch page) — the fix must not break legitimate playback. A stream-only link →
inline; the type is video/mp4 (from the container allow-list), never text/* and never a guess."""
_probe(monkeypatch, ("h264", "aac", 640, 360, 12))
c = as_user(make_user())
job = c.post(
"/api/downloads/upload",
files={"file": ("clip.mp4", b"fake-mp4-bytes", "video/mp4")},
).json()
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json()
pr = c.get(f"/api/public/watch/{link['token']}/file")
assert pr.status_code == 200
assert pr.headers["content-type"].startswith("video/mp4")
assert "inline" in pr.headers["content-disposition"].lower()
def test_upload_route_enforces_per_file_cap(as_user, make_user, db, _root, monkeypatch):
from app import sysconfig
@@ -151,6 +226,16 @@ def test_upload_route_enforces_per_file_cap(as_user, make_user, db, _root, monke
assert r.json()["detail"]["code"] == "upload_too_large"
def test_upload_cap_cannot_be_disabled_with_zero(db):
"""download_upload_max_bytes floors at 1: a stored 0 would read falsy in the route's
`if cap and size > cap` guard and SILENTLY disable the per-file cap, so set_value must reject it.
Pins the min=1 invariant so a revert to min=0 can't pass the gate."""
from app import sysconfig
with pytest.raises(ValueError):
sysconfig.set_value(db, "download_upload_max_bytes", 0)
def test_upload_route_rejects_empty_file(as_user, make_user, _root, monkeypatch):
c = as_user(make_user())
r = c.post(
@@ -182,3 +267,128 @@ def test_edit_rejects_a_generic_upload_source(as_user, make_user, _root, monkeyp
)
assert r.status_code == 400
assert r.json()["detail"]["reason"] == "not_media"
def test_ingest_rejects_over_max_jobs(db, make_user, _root, monkeypatch):
"""The other check_upload rejection branch (holdings count, not bytes): once the user is at
max_jobs, a further upload is refused before any disk write."""
_probe(monkeypatch, (None, None, None, None, None))
u = make_user()
db.add(
DownloadQuota(user_id=u.id, max_bytes=10**12, max_concurrent=2, max_jobs=1, unlimited=False)
)
db.commit()
src1 = _root / "a.bin"
src1.write_bytes(b"a")
uploadmod.ingest_upload(db, u.id, filename="a.bin", src_path=src1, size=1) # now 1 holding job
src2 = _root / "b.bin"
src2.write_bytes(b"b")
with pytest.raises(quota.QuotaExceeded) as e:
uploadmod.ingest_upload(db, u.id, filename="b.bin", src_path=src2, size=1)
assert e.value.reason == "max_jobs"
assert src2.exists() # rejected before the move — nothing landed
def test_ingest_cleans_up_placed_files_on_commit_failure(db, make_user, _root, monkeypatch):
"""A failure AFTER storage.place_file has moved the file onto disk must roll back AND unlink the
placed file + any (even partial) poster — otherwise the orphan is permanent (the per-user delete
keys off the rolled-back asset.rel_path, and the GC orphan sweep skips the dot-prefixed .uploads/
tree). Exercises the rollback/cleanup branch that IS the leak remediation, incl. the poster case
where ensure_poster left a partial file but returned None."""
_probe(monkeypatch, ("h264", "aac", 320, 240, 5)) # video → the poster path runs
u = make_user()
# Simulate ffmpeg leaving a partial poster on disk, then failing (ensure_poster returns None).
def _partial_poster(asset, root):
storage.abs_path(root, str(Path(asset.rel_path).with_suffix(".jpg"))).write_bytes(
b"partial"
)
return None
monkeypatch.setattr(uploadmod.editmod, "ensure_poster", _partial_poster)
# Force the commit to fail after the file (and partial poster) are on disk.
def _boom():
raise RuntimeError("db down")
monkeypatch.setattr(db, "commit", _boom)
src = _root / "clip.mp4"
src.write_bytes(b"fake-video-bytes")
with pytest.raises(RuntimeError):
uploadmod.ingest_upload(db, u.id, filename="clip.mp4", src_path=src, size=16)
db.rollback() # drop the monkeypatched-commit session state before querying
uploads = _root / ".uploads"
leftover = [p for p in uploads.rglob("*") if p.is_file()] if uploads.exists() else []
assert leftover == [], f"orphaned files left on disk: {leftover}"
assert db.query(MediaAsset).count() == 0 # the flushed asset rolled back — no dangling row
# --- classifier: the REAL _probe parsing (stubs ffprobe one level down) ----------------------
# The ingest/route tests above stub _probe wholesale; these exercise _probe's OWN logic — image
# detection, the cover-art skip, dimension/duration parsing — by faking only run_ffprobe_json, so a
# regression in the classifier the force-download guarantee leans on is actually caught.
class TestProbeClassifier:
def _probe_with(self, monkeypatch, data):
monkeypatch.setattr(uploadmod.editmod, "run_ffprobe_json", lambda *a, **k: data)
return uploadmod._probe(Path("x"))
def test_probe_failure_is_a_generic_file(self, monkeypatch):
# run_ffprobe_json returns None (ffprobe failed / not media) → all None → derive() == FILE.
v, a, w, h, d = self._probe_with(monkeypatch, None)
assert (v, a, w, h, d) == (None, None, None, None, None)
assert mediakind.derive(v, a) == mediakind.FILE
@pytest.mark.parametrize("fmt", ["image2", "png_pipe", "jpeg_pipe", "webp_pipe", "gif"])
def test_still_image_classifies_as_file_not_video(self, monkeypatch, fmt):
# A standalone image exposes a real video stream, but its container format is an image demuxer
# — it must NOT become media_kind='video' (broken player + inline serving). Stays a file.
data = {
"format": {"format_name": fmt},
"streams": [{"codec_type": "video", "codec_name": "png", "width": 8, "height": 8}],
}
v, a, w, h, d = self._probe_with(monkeypatch, data)
assert v is None and mediakind.derive(v, a) == mediakind.FILE
def test_cover_art_video_stream_is_skipped_audio_stays_audio(self, monkeypatch):
# Album art embedded in an audio file is a video stream flagged attached_pic — skip it so the
# asset classifies as audio, not video.
data = {
"format": {"format_name": "mp3", "duration": "212.5"},
"streams": [
{
"codec_type": "video",
"codec_name": "mjpeg",
"disposition": {"attached_pic": 1},
"width": 600,
"height": 600,
},
{"codec_type": "audio", "codec_name": "mp3"},
],
}
v, a, w, h, d = self._probe_with(monkeypatch, data)
assert (v, a, d) == (None, "mp3", 212)
assert mediakind.derive(v, a) == mediakind.AUDIO
def test_real_video_records_codecs_dims_duration(self, monkeypatch):
data = {
"format": {"format_name": "mov,mp4,m4a", "duration": "60.0"},
"streams": [
{"codec_type": "video", "codec_name": "h264", "width": 1920, "height": 1080},
{"codec_type": "audio", "codec_name": "aac"},
],
}
v, a, w, h, d = self._probe_with(monkeypatch, data)
assert (v, a, w, h, d) == ("h264", "aac", 1920, 1080, 60)
assert mediakind.derive(v, a) == mediakind.VIDEO
def test_unparseable_duration_is_none(self, monkeypatch):
data = {
"format": {"format_name": "wav", "duration": "N/A"},
"streams": [{"codec_type": "audio", "codec_name": "pcm_s16le"}],
}
v, a, w, h, d = self._probe_with(monkeypatch, data)
assert d is None and mediakind.derive(v, a) == mediakind.AUDIO