fix(downloads): harden upload serving + ingest against stored-XSS and leaks (E7 review)

Deep-review remediation for the E7 upload path, all findings adversarially verified:

- Serve every library file with a Content-Type chosen from the ffprobe-verified media_kind
  + a strict container allow-list (storage.served_media_type), never guessed from the
  user-controlled file extension. Closes a stored-XSS vector where a media/HTML polyglot
  uploaded as "evil.html" was served inline as text/html on the login-free /watch surface.
  Applied to both the authenticated /file and the public watch_file endpoints.
- Classify uploaded still images (jpg/png/gif/webp/…) as media_kind='file' so they are
  force-downloaded and don't get a broken in-app video player.
- Roll back the on-disk side of an upload when the ingest transaction fails after place_file
  (unlink the placed file + its deterministic poster path), so a failed commit can't orphan
  bytes in the GC-skipped .uploads tree.
- Floor download_upload_max_bytes at min=1 so a stored 0 can't silently disable the per-file cap.
- Dedupe the ffprobe-JSON invocation into edit.run_ffprobe_json (shared by probe_codecs and
  upload._probe) and the upload extension into storage._clean_upload_ext; drop the now-unused
  parse_probe_streams.

Tests: public + authenticated force-download and the polyglot-not-served-as-HTML guard, the
real _probe classifier (image/cover-art/dims/duration), the ingest rollback/cleanup path, the
max_jobs branch, the cap-floor invariant, and run_ffprobe_json error handling.
This commit is contained in:
2026-08-29 21:52:02 +02:00
parent 104bd1e3a6
commit 614c5528ad
8 changed files with 475 additions and 128 deletions
+38 -30
View File
@@ -438,18 +438,42 @@ _BROWSER_SAFE_VCODEC = ("h264",)
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
"""(video_codec, audio_codec) out of ffprobe's JSON stream list, lowercased.
JSON, not the flat `default=nw=1` output: there the fields arrive in ffprobe's own struct
order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line "remember the last
codec_type" parser attributes every name to the PREVIOUS stream. Measured on real downloads:
an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
believed the video stream was AAC and re-encoded a perfectly fine file on every download."""
def run_ffprobe_json(
path: Path,
*,
show_streams: bool = False,
show_format: bool = False,
entries: str | None = None,
) -> dict | None:
"""One ffprobe call → its parsed JSON dict, or None on any failure (timeout, non-zero exit,
missing binary, unparseable output). The SINGLE ffprobe-JSON-invocation point in the downloads
package: the browser-playability codec probe here and the upload-ingest probe (`upload._probe`)
both go through it, so the subprocess flags/timeout/error handling live in one place instead of
two drifting copies."""
args = ["ffprobe", "-v", "error"]
if show_streams:
args.append("-show_streams")
if show_format:
args.append("-show_format")
if entries:
args += ["-show_entries", entries]
args += ["-of", "json", str(path)]
try:
streams = json.loads(payload or "{}").get("streams") or []
except ValueError:
return None, None
out = subprocess.run(args, capture_output=True, text=True, timeout=30, check=True).stdout
return json.loads(out)
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
return None
def _codecs_from_streams(streams: list) -> tuple[str | None, str | None]:
"""(video_codec, audio_codec) lowercased from an ffprobe JSON stream list — the first of each kind.
Parsed from the JSON `streams` list, NOT ffprobe's flat `default=nw=1` output: there the fields
arrive in ffprobe's own struct order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line
"remember the last codec_type" parser attributes every name to the PREVIOUS stream. Measured on real
downloads: an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
believed the video stream was AAC and re-encoded a perfectly fine file on every download. Reading each
stream dict by its own `codec_type` is immune to that ordering."""
vcodec = acodec = None
for s in streams:
name = (s.get("codec_name") or "").lower() or None
@@ -464,26 +488,10 @@ def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
try:
out = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-show_entries",
"stream=codec_type,codec_name",
"-of",
"json",
str(path),
],
capture_output=True,
text=True,
timeout=30,
check=True,
).stdout
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
data = run_ffprobe_json(path, entries="stream=codec_type,codec_name")
if data is None:
return None, None
return parse_probe_streams(out)
return _codecs_from_streams(data.get("streams") or [])
def probe_duration(path: Path) -> int:
+59 -4
View File
@@ -19,6 +19,8 @@ from datetime import date
from pathlib import Path
from xml.sax.saxutils import escape
from app.downloads import mediakind
# Reserve room for the "_id.ext" suffix + parent dirs within the 255-char component limit.
_MAX_TITLE = 120
# User-facing (Content-Disposition) names can be longer than an on-disk component.
@@ -147,6 +149,14 @@ def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
return f".edits/{user_id}/clip_{asset_id}.{ext}"
def _clean_upload_ext(filename: str) -> str:
"""The upload's extension reduced to a shell-safe, lowercased, bounded token (``""`` when the
upload has no extension). The SINGLE definition feeding both the on-disk suffix and the stored
`container` — they must agree (the served Content-Disposition suffix has to match the container),
so the sanitize/truncation rule lives in one place, not two drifting copies."""
return _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
"""On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
@@ -157,16 +167,61 @@ def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
suffix; an extensionless upload stays extensionless."""
stem = sanitize(Path(filename).stem, 80)
ext = _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
ext = _clean_upload_ext(filename)
suffix = f".{ext}" if ext else ""
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
def upload_ext(filename: str) -> str | None:
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
the upload has no extension (a bare name)."""
ext = _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
return ext or None
the upload has no extension (a bare name). Shares `_clean_upload_ext` with `upload_rel_path` so the
stored container and the on-disk suffix can never diverge."""
return _clean_upload_ext(filename) or None
# The Content-Type used to serve an inline-playable media asset, keyed by its stored `container`.
# DELIBERATELY a strict allow-list of known audio/video containers: the served type is chosen from
# the asset's ffprobe-verified media_kind + this table, NEVER guessed from the raw (user-controlled)
# upload extension. Without this, an uploaded media/HTML polyglot named "evil.html" — a real MP3/MP4
# that ffprobe classifies as media, so it dodges the generic-file force-download — would be served
# with a Content-Type Starlette guesses from ".html" (text/html) and execute as stored XSS on the
# serving origin. A media asset whose container isn't in this table falls back to a neutral
# force-download attachment (see `served_media_type`), so nothing script-executable is ever inline.
_MEDIA_MIME: dict[str, str] = {
# video containers
"mp4": "video/mp4",
"m4v": "video/mp4",
"mov": "video/quicktime",
"webm": "video/webm",
"mkv": "video/x-matroska",
"ogv": "video/ogg",
"avi": "video/x-msvideo",
# audio containers
"m4a": "audio/mp4",
"mp3": "audio/mpeg",
"aac": "audio/aac",
"ogg": "audio/ogg",
"oga": "audio/ogg",
"opus": "audio/ogg",
"weba": "audio/webm",
"wav": "audio/wav",
"flac": "audio/flac",
}
def served_media_type(media_kind: str | None, container: str | None) -> str | None:
"""The Content-Type a library file MUST be served with — a real audio/video MIME for a recognised
media container, else ``None`` meaning "serve as a neutral force-download attachment".
Returns None for a generic ``file`` kind AND for any media whose container we don't positively
recognise as safe. Callers serve None as ``application/octet-stream`` + ``attachment`` so the
served Content-Type is NEVER derived from the (attacker-controlled) file extension — the guard
that stops uploaded HTML/SVG or a media/HTML polyglot from executing inline on the serving
origin. A ``None`` media_kind (a legacy asset before the discriminator was backfilled) is treated
as media so an old download keeps its inline player."""
if media_kind == mediakind.FILE:
return None
return _MEDIA_MIME.get((container or "").lower())
def download_filename(display_name: str, container: str | None, path: Path) -> str:
+83 -58
View File
@@ -16,9 +16,7 @@ source to re-download), so it doesn't age out on the retention clock the way a r
does — it lives until the user deletes it.
"""
import json
import secrets
import subprocess
from datetime import datetime, timezone
from pathlib import Path
@@ -29,32 +27,43 @@ from app.downloads import edit as editmod
from app.downloads import mediakind, quota, service, storage
from app.models import DownloadJob, MediaAsset
# ffprobe container formats that mean "this is a still image / image sequence", NOT playable media.
# A standalone jpg/png/gif/webp/bmp reports a real video stream (codec mjpeg/png/…) with no
# `attached_pic` flag, so without this guard it would classify as media_kind='video' and get a Play
# button + a broken in-app <video> player (and inline serving on the public share). Keyed on the
# container format so an animated gif/webp also lands as a generic 'file' (no browser plays those in
# a <video> either). SVG and anything else ffprobe can't decode already fails the probe → 'file'.
_IMAGE_FORMATS = {
"image2",
"image2pipe",
"png_pipe",
"mjpeg",
"mjpeg_pipe",
"jpeg_pipe",
"gif",
"webp_pipe",
"bmp_pipe",
"tiff_pipe",
"ico",
"svg_pipe",
}
def _probe(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
or the file isn't media (a file we can't probe is treated as a generic non-media upload).
or the file isn't playable media (an unprobeable file — or a still image — is treated as a generic
non-media upload, so it becomes a download-only 'file' rather than broken 'video').
A video stream flagged `attached_pic` (embedded cover art in an mp3/m4a) is skipped — it's a
thumbnail, not a real video track, so an audio file with album art still classifies as audio."""
try:
out = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-show_streams",
"-show_format",
"-of",
"json",
str(path),
],
capture_output=True,
text=True,
timeout=30,
check=True,
).stdout
data = json.loads(out)
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
if data is None:
return None, None, None, None, None
fmt = data.get("format") or {}
fmt_names = {n.strip() for n in (fmt.get("format_name") or "").split(",")}
if fmt_names & _IMAGE_FORMATS:
# A still image / image sequence is a generic file, not video — no vcodec means media_kind='file'.
return None, None, None, None, None
vcodec = acodec = None
@@ -72,7 +81,7 @@ def _probe(path: Path) -> tuple[str | None, str | None, int | None, int | None,
duration = None
try:
raw = (data.get("format") or {}).get("duration")
raw = fmt.get("duration")
if raw is not None:
duration = int(float(raw))
except (TypeError, ValueError):
@@ -117,42 +126,58 @@ def ingest_upload(
rel = storage.upload_rel_path(user_id, asset.id, filename)
storage.place_file(src_path, settings.download_root, rel)
final = storage.abs_path(settings.download_root, rel)
actual_size = final.stat().st_size if final.exists() else size
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
# asset/job rows roll back but the placed file would be ORPHANED — nothing references it, the
# per-user delete path keys off asset.rel_path (which is gone), and the GC orphan sweep skips the
# dot-prefixed `.uploads/` tree. So roll back the on-disk side too on any failure.
try:
final = storage.abs_path(settings.download_root, rel)
actual_size = final.stat().st_size if final.exists() else size
asset.status = "ready"
asset.rel_path = rel
asset.size_bytes = actual_size
asset.container = storage.upload_ext(filename)
asset.vcodec = vcodec
asset.acodec = acodec
asset.width = width
asset.height = height
asset.duration_s = duration
asset.media_kind = kind
asset.title = (display_name or "").strip() or _title_from_filename(filename)
asset.last_access_at = datetime.now(timezone.utc)
# No expires_at — see the module docstring (an upload is irreplaceable, so it doesn't age out).
if kind == mediakind.VIDEO:
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has no
# frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes server
# side even for a codec the browser can't play).
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
asset.status = "ready"
asset.rel_path = rel
asset.size_bytes = actual_size
asset.container = storage.upload_ext(filename)
asset.vcodec = vcodec
asset.acodec = acodec
asset.width = width
asset.height = height
asset.duration_s = duration
asset.media_kind = kind
asset.title = (display_name or "").strip() or _title_from_filename(filename)
asset.last_access_at = datetime.now(timezone.utc)
# No expires_at — see the module docstring (an upload is irreplaceable, so it doesn't age out).
if kind == mediakind.VIDEO:
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
# server side even for a codec the browser can't play).
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
job = DownloadJob(
user_id=user_id,
asset_id=asset.id,
job_kind="upload",
source_kind="upload",
source_ref=ref,
profile_snapshot={},
display_name=(display_name or "").strip() or None,
status="done",
progress=100,
queue_pos=service.next_queue_pos(db),
)
db.add(job)
service.incr_ref_count(db, asset.id)
db.commit()
job = DownloadJob(
user_id=user_id,
asset_id=asset.id,
job_kind="upload",
source_kind="upload",
source_ref=ref,
profile_snapshot={},
display_name=(display_name or "").strip() or None,
status="done",
progress=100,
queue_pos=service.next_queue_pos(db),
)
db.add(job)
service.incr_ref_count(db, asset.id)
db.commit()
except Exception:
db.rollback()
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
# ffmpeg failure, so a partial poster can be on disk while the return is None — gating on the
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
# when none was generated (audio/file, or a video whose poster never started).
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
raise
db.refresh(job)
return job