fix(downloads): harden upload serving + ingest against stored-XSS and leaks (E7 review)
Deep-review remediation for the E7 upload path, all findings adversarially verified: - Serve every library file with a Content-Type chosen from the ffprobe-verified media_kind + a strict container allow-list (storage.served_media_type), never guessed from the user-controlled file extension. Closes a stored-XSS vector where a media/HTML polyglot uploaded as "evil.html" was served inline as text/html on the login-free /watch surface. Applied to both the authenticated /file and the public watch_file endpoints. - Classify uploaded still images (jpg/png/gif/webp/…) as media_kind='file' so they are force-downloaded and don't get a broken in-app video player. - Roll back the on-disk side of an upload when the ingest transaction fails after place_file (unlink the placed file + its deterministic poster path), so a failed commit can't orphan bytes in the GC-skipped .uploads tree. - Floor download_upload_max_bytes at min=1 so a stored 0 can't silently disable the per-file cap. - Dedupe the ffprobe-JSON invocation into edit.run_ffprobe_json (shared by probe_codecs and upload._probe) and the upload extension into storage._clean_upload_ext; drop the now-unused parse_probe_streams. Tests: public + authenticated force-download and the polyglot-not-served-as-HTML guard, the real _probe classifier (image/cover-art/dims/duration), the ingest rollback/cleanup path, the max_jobs branch, the cap-floor invariant, and run_ffprobe_json error handling.
This commit is contained in:
@@ -16,9 +16,7 @@ source to re-download), so it doesn't age out on the retention clock the way a r
|
||||
does — it lives until the user deletes it.
|
||||
"""
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
@@ -29,32 +27,43 @@ from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
# ffprobe container formats that mean "this is a still image / image sequence", NOT playable media.
|
||||
# A standalone jpg/png/gif/webp/bmp reports a real video stream (codec mjpeg/png/…) with no
|
||||
# `attached_pic` flag, so without this guard it would classify as media_kind='video' and get a Play
|
||||
# button + a broken in-app <video> player (and inline serving on the public share). Keyed on the
|
||||
# container format so an animated gif/webp also lands as a generic 'file' (no browser plays those in
|
||||
# a <video> either). SVG and anything else ffprobe can't decode already fails the probe → 'file'.
|
||||
_IMAGE_FORMATS = {
|
||||
"image2",
|
||||
"image2pipe",
|
||||
"png_pipe",
|
||||
"mjpeg",
|
||||
"mjpeg_pipe",
|
||||
"jpeg_pipe",
|
||||
"gif",
|
||||
"webp_pipe",
|
||||
"bmp_pipe",
|
||||
"tiff_pipe",
|
||||
"ico",
|
||||
"svg_pipe",
|
||||
}
|
||||
|
||||
|
||||
def _probe(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
||||
or the file isn't media (a file we can't probe is treated as a generic non-media upload).
|
||||
or the file isn't playable media (an unprobeable file — or a still image — is treated as a generic
|
||||
non-media upload, so it becomes a download-only 'file' rather than broken 'video').
|
||||
|
||||
A video stream flagged `attached_pic` (embedded cover art in an mp3/m4a) is skipped — it's a
|
||||
thumbnail, not a real video track, so an audio file with album art still classifies as audio."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_streams",
|
||||
"-show_format",
|
||||
"-of",
|
||||
"json",
|
||||
str(path),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=True,
|
||||
).stdout
|
||||
data = json.loads(out)
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
if data is None:
|
||||
return None, None, None, None, None
|
||||
|
||||
fmt = data.get("format") or {}
|
||||
fmt_names = {n.strip() for n in (fmt.get("format_name") or "").split(",")}
|
||||
if fmt_names & _IMAGE_FORMATS:
|
||||
# A still image / image sequence is a generic file, not video — no vcodec means media_kind='file'.
|
||||
return None, None, None, None, None
|
||||
|
||||
vcodec = acodec = None
|
||||
@@ -72,7 +81,7 @@ def _probe(path: Path) -> tuple[str | None, str | None, int | None, int | None,
|
||||
|
||||
duration = None
|
||||
try:
|
||||
raw = (data.get("format") or {}).get("duration")
|
||||
raw = fmt.get("duration")
|
||||
if raw is not None:
|
||||
duration = int(float(raw))
|
||||
except (TypeError, ValueError):
|
||||
@@ -117,42 +126,58 @@ def ingest_upload(
|
||||
|
||||
rel = storage.upload_rel_path(user_id, asset.id, filename)
|
||||
storage.place_file(src_path, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
actual_size = final.stat().st_size if final.exists() else size
|
||||
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
|
||||
# asset/job rows roll back but the placed file would be ORPHANED — nothing references it, the
|
||||
# per-user delete path keys off asset.rel_path (which is gone), and the GC orphan sweep skips the
|
||||
# dot-prefixed `.uploads/` tree. So roll back the on-disk side too on any failure.
|
||||
try:
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
actual_size = final.stat().st_size if final.exists() else size
|
||||
|
||||
asset.status = "ready"
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = actual_size
|
||||
asset.container = storage.upload_ext(filename)
|
||||
asset.vcodec = vcodec
|
||||
asset.acodec = acodec
|
||||
asset.width = width
|
||||
asset.height = height
|
||||
asset.duration_s = duration
|
||||
asset.media_kind = kind
|
||||
asset.title = (display_name or "").strip() or _title_from_filename(filename)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
# No expires_at — see the module docstring (an upload is irreplaceable, so it doesn't age out).
|
||||
if kind == mediakind.VIDEO:
|
||||
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has no
|
||||
# frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes server
|
||||
# side even for a codec the browser can't play).
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.status = "ready"
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = actual_size
|
||||
asset.container = storage.upload_ext(filename)
|
||||
asset.vcodec = vcodec
|
||||
asset.acodec = acodec
|
||||
asset.width = width
|
||||
asset.height = height
|
||||
asset.duration_s = duration
|
||||
asset.media_kind = kind
|
||||
asset.title = (display_name or "").strip() or _title_from_filename(filename)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
# No expires_at — see the module docstring (an upload is irreplaceable, so it doesn't age out).
|
||||
if kind == mediakind.VIDEO:
|
||||
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
|
||||
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
|
||||
# server side even for a codec the browser can't play).
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind="upload",
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=(display_name or "").strip() or None,
|
||||
status="done",
|
||||
progress=100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
service.incr_ref_count(db, asset.id)
|
||||
db.commit()
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind="upload",
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=(display_name or "").strip() or None,
|
||||
status="done",
|
||||
progress=100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
service.incr_ref_count(db, asset.id)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
|
||||
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
|
||||
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
|
||||
# ffmpeg failure, so a partial poster can be on disk while the return is None — gating on the
|
||||
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
|
||||
# when none was generated (audio/file, or a video whose poster never started).
|
||||
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
|
||||
storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
|
||||
raise
|
||||
db.refresh(job)
|
||||
return job
|
||||
|
||||
Reference in New Issue
Block a user