fix(uploads): read the body after authenticating, not before it
`POST /api/downloads/upload` declared `file: UploadFile = File(...)`, and FastAPI parses and spools a declared body before the first dependency runs -- `require_human` included. An unauthenticated caller could therefore make the process write the whole per-file allowance (500 MiB by default) to temp, once per request. SEC-20's ceiling bounded that number but could not stop the writing, which is why E27 parked it rather than closing it. The route now takes no body parameter. `spool_upload` names `require_human` first and only then streams `request.stream()` to the staging temp file, so a stranger is refused before a byte moves -- the shape the screenshot route has used since E16. Filename and display name travel as percent-encoded headers (latin-1 on the wire, so a Hungarian filename needs the encoding) instead of multipart parts. `capped_stream_to` joins `capped_body` in routes/_common.py: the same counting rule, written to a file descriptor rather than accumulated, because holding 500 MiB would trade a disk problem for a memory one. Its refusal is the caller's, so the upload route keeps answering 422 `upload_too_large` with the admin knob's own number. With no envelope to allow for, the edge is now exactly that knob, and MULTIPART_ENVELOPE_ALLOWANCE is gone. The invariant stops being prose: test_body_limit.py asks FastAPI itself (`route.dependant.body_params`) and fails the gate if any route with a raised ceiling declares a body. A comment could not have prevented the defect it described; that test would have. Both new tests were seen red for the right reason -- before the fix the anonymous probe got its 401 only after the server had read 4236 bytes of a multipart body, and re-declaring the File parameter turns the invariant test red naming the route and the parameter.
This commit is contained in:
@@ -15,6 +15,7 @@ tests that COMMIT (the whole point of this lane — transaction-ownership behavi
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
@@ -248,6 +249,27 @@ def as_user(db, override_dep, _configured):
|
||||
return _as
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upload_headers():
|
||||
"""Build the headers `POST /api/downloads/upload` takes since P1.
|
||||
|
||||
Shared, because two test modules drive that route and a second copy of this is exactly the drift
|
||||
that lets the wire shape change in one place only. The body IS the file now; the names ride as
|
||||
percent-encoded headers (`quote` mirrors the client's `encodeURIComponent`, since headers are
|
||||
latin-1 and a Hungarian filename would not survive raw)."""
|
||||
|
||||
def _headers(filename: str, display_name: str | None = None) -> dict[str, str]:
|
||||
headers = {
|
||||
"content-type": "application/octet-stream",
|
||||
"x-upload-filename": quote(filename),
|
||||
}
|
||||
if display_name is not None:
|
||||
headers["x-upload-display-name"] = quote(display_name)
|
||||
return headers
|
||||
|
||||
return _headers
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_rate_limiters():
|
||||
"""Every RateLimiter is a module-level singleton keyed by client IP, and TestClient always
|
||||
|
||||
@@ -251,14 +251,57 @@ def test_the_upload_ceiling_and_the_admin_knob_cannot_drift_apart():
|
||||
constant. If the knob could be set above the edge, an admin raising it would silently start
|
||||
getting 413s from a layer the admin page never mentions.
|
||||
|
||||
The relation is `<=`, not `==`, and the gap is the point: the edge counts RAW body bytes while
|
||||
the route counts the DECODED file, so the edge has to allow for the multipart envelope or a file
|
||||
of exactly the advertised maximum is refused one layer before the route that would take it.
|
||||
Since P1 the two count the SAME bytes — the body is the file, with no multipart envelope
|
||||
wrapped round it — so the old allowance is gone and the edge is the knob's own maximum,
|
||||
exactly. A gap reappearing means someone re-wrapped the upload in an envelope without saying so.
|
||||
"""
|
||||
spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes")
|
||||
edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/downloads/upload")]
|
||||
assert spec.max is not None and spec.max <= edge
|
||||
assert edge - spec.max == bodylimit.MULTIPART_ENVELOPE_ALLOWANCE
|
||||
assert spec.max == edge
|
||||
|
||||
|
||||
def _route_objects(root):
|
||||
"""`_declared_routes`' sibling, keeping the route OBJECT so its FastAPI dependant can be read."""
|
||||
found: dict[tuple[str, str], object] = {}
|
||||
stack = [root]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
for route in getattr(node, "routes", ()) or ():
|
||||
inner = getattr(route, "original_router", None)
|
||||
if inner is not None:
|
||||
stack.append(inner)
|
||||
continue
|
||||
for method in getattr(route, "methods", None) or ():
|
||||
found[(method, route.path)] = route
|
||||
return found
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("method", "template"), sorted(bodylimit.ROUTE_MAX_BODY_BYTES))
|
||||
def test_no_route_with_a_raised_ceiling_declares_a_body_parameter(method, template):
|
||||
"""P1's invariant, and the reason this is a test rather than a comment.
|
||||
|
||||
A raised ceiling is only honest when the route's own cap applies to the bytes AS THEY ARRIVE.
|
||||
FastAPI reads and spools a DECLARED body (`= File(...)`, `= Form(...)`, a Pydantic model)
|
||||
BEFORE the first dependency resolves — so on such a route this middleware's ceiling is the only
|
||||
thing between an anonymous caller and the whole allowance, `require_human` never gets a say, and
|
||||
the route's in-handler cap measures bytes that have already landed. That is exactly what P1
|
||||
closed on the upload route, which sat in `ROUTE_MAX_BODY_BYTES` with a 500 MiB ceiling and a
|
||||
`file: UploadFile = File(...)` body for one release.
|
||||
|
||||
Asked of FastAPI itself (`route.dependant.body_params`) rather than parsed or grepped: the
|
||||
router already knows which parameters it will read the body for, and a shape a regex or an AST
|
||||
walk guesses wrong is precisely how the next route slips through."""
|
||||
from app.main import app
|
||||
|
||||
route = _route_objects(app).get((method, template))
|
||||
assert route is not None, f"{method} {template} is not a route any more"
|
||||
body_params = [p.name for p in getattr(route.dependant, "body_params", ())]
|
||||
assert not body_params, (
|
||||
f"{method} {template} declares a body parameter ({', '.join(body_params)}), so FastAPI "
|
||||
"spools it before the first dependency runs and the raised ceiling is all that stands "
|
||||
"between an anonymous caller and the whole allowance. Read the body yourself, from a "
|
||||
"dependency declared AFTER the auth one — see routes/downloads.py:spool_upload."
|
||||
)
|
||||
|
||||
|
||||
def test_a_stored_override_outside_the_bounds_falls_back_on_READ(db):
|
||||
|
||||
@@ -18,12 +18,12 @@ _PW = "open sesame"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unlock_url(as_user, make_user, tmp_path, monkeypatch):
|
||||
def unlock_url(as_user, make_user, tmp_path, monkeypatch, upload_headers):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(probemod, "probe_media", lambda _p: ("h264", "aac", 320, 240, 5))
|
||||
owner = as_user(make_user())
|
||||
job = owner.post(
|
||||
"/api/downloads/upload", files={"file": ("clip.mp4", b"fake-video", "video/mp4")}
|
||||
"/api/downloads/upload", content=b"fake-video", headers=upload_headers("clip.mp4")
|
||||
)
|
||||
assert job.status_code == 200, job.text
|
||||
link = owner.post(f"/api/downloads/{job.json()['id']}/links", json={"password": _PW})
|
||||
|
||||
+124
-25
@@ -123,15 +123,106 @@ def test_ingest_rejects_over_footprint_cap_before_writing(db, make_user, _root,
|
||||
assert db.query(MediaAsset).count() == 0 # and no orphan asset row
|
||||
|
||||
|
||||
# --- route: the auth-first body (P1) --------------------------------------------------------
|
||||
|
||||
|
||||
def _counting_body(chunks: int = 4, size: int = 1024):
|
||||
"""A request body that records how much of itself the server actually pulled.
|
||||
|
||||
httpx's ASGI transport feeds the app lazily, one `http.request` message per yielded chunk, so a
|
||||
chunk that was never yielded is a chunk the app never asked for. That is the whole assertion
|
||||
below: it is not "the upload was refused", it is "the refusal happened before the bytes moved".
|
||||
"""
|
||||
pulled: list[int] = []
|
||||
|
||||
def body():
|
||||
for _ in range(chunks):
|
||||
pulled.append(size)
|
||||
yield b"x" * size
|
||||
|
||||
return body(), pulled
|
||||
|
||||
|
||||
_BOUNDARY = "p1boundary"
|
||||
|
||||
|
||||
def _counting_multipart(chunks: int = 4, size: int = 1024):
|
||||
"""The same counter, wrapped in a real multipart envelope.
|
||||
|
||||
The content-type is the load-bearing part. Starlette's `request.form()` returns an EMPTY
|
||||
FormData without touching the stream when the content-type is not multipart/urlencoded — so a
|
||||
raw-body probe cannot tell "the route stopped reading strangers" apart from "there was nothing
|
||||
here to parse". Only a well-formed multipart body makes the old shape actually spool.
|
||||
"""
|
||||
pulled: list[int] = []
|
||||
head = (
|
||||
f"--{_BOUNDARY}\r\n"
|
||||
'Content-Disposition: form-data; name="file"; filename="clip.mp4"\r\n'
|
||||
"Content-Type: application/octet-stream\r\n\r\n"
|
||||
).encode()
|
||||
tail = f"\r\n--{_BOUNDARY}--\r\n".encode()
|
||||
|
||||
def body():
|
||||
pulled.append(len(head))
|
||||
yield head
|
||||
for _ in range(chunks):
|
||||
pulled.append(size)
|
||||
yield b"x" * size
|
||||
pulled.append(len(tail))
|
||||
yield tail
|
||||
|
||||
return body(), pulled
|
||||
|
||||
|
||||
def test_an_anonymous_upload_is_refused_before_a_byte_of_it_is_read(client):
|
||||
"""P1 — the defect E27 bounded but did not close.
|
||||
|
||||
While the route declared `file: UploadFile = File(...)`, FastAPI parsed and spooled the body
|
||||
BEFORE the first dependency ran, so `require_human` could not gate it: an unauthenticated
|
||||
caller could make the server write the whole per-file allowance (500 MiB by default) to temp,
|
||||
per request. SEC-20's ceiling capped the number; it could not stop the writing.
|
||||
|
||||
The route now reads its own body from a dependency declared AFTER the auth one, the shape
|
||||
`POST /api/debug-sessions/{sid}/screenshots` already uses. The claim under test is the strong
|
||||
one — a stranger's bytes are never read WHATEVER they claim to send, so the probe sends the
|
||||
multipart the old route would have spooled, not the raw body the new one expects."""
|
||||
body, pulled = _counting_multipart()
|
||||
r = client.post(
|
||||
"/api/downloads/upload",
|
||||
content=body,
|
||||
headers={"content-type": f"multipart/form-data; boundary={_BOUNDARY}"},
|
||||
)
|
||||
assert r.status_code == 401
|
||||
assert pulled == [], f"the server read {sum(pulled)} bytes from an anonymous caller"
|
||||
|
||||
|
||||
def test_an_authenticated_upload_still_has_its_body_read(as_user, make_user, _root, monkeypatch):
|
||||
"""The mutation check for the test above: a route that read NOBODY's body would pass it just as
|
||||
well as the fixed one. The same counting body, signed in, must be pulled in full."""
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
body, pulled = _counting_body()
|
||||
c = as_user(make_user())
|
||||
r = c.post(
|
||||
"/api/downloads/upload",
|
||||
content=body,
|
||||
headers={"content-type": "application/octet-stream", "x-upload-filename": "clip.bin"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert sum(pulled) == 4096
|
||||
|
||||
|
||||
# --- route ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_upload_route_generic_forces_download(as_user, make_user, _root, monkeypatch):
|
||||
def test_upload_route_generic_forces_download(
|
||||
as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
c = as_user(make_user())
|
||||
r = c.post(
|
||||
"/api/downloads/upload",
|
||||
files={"file": ("report.html", b"<script>alert(1)</script>", "text/html")},
|
||||
content=b"<script>alert(1)</script>",
|
||||
headers=upload_headers("report.html"),
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
job = r.json()
|
||||
@@ -145,7 +236,7 @@ def test_upload_route_generic_forces_download(as_user, make_user, _root, monkeyp
|
||||
|
||||
|
||||
def test_upload_route_media_html_extension_not_served_as_html(
|
||||
as_user, make_user, _root, monkeypatch
|
||||
as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
"""The XSS guard: a MEDIA-classified upload (real codecs) named with an executable extension must
|
||||
NOT be served with a guessed text/html type. The served Content-Type comes from the verified
|
||||
@@ -156,7 +247,8 @@ def test_upload_route_media_html_extension_not_served_as_html(
|
||||
c = as_user(make_user())
|
||||
r = c.post(
|
||||
"/api/downloads/upload",
|
||||
files={"file": ("evil.html", b"<script>alert(1)</script>", "text/html")},
|
||||
content=b"<script>alert(1)</script>",
|
||||
headers=upload_headers("evil.html"),
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
job = r.json()
|
||||
@@ -177,7 +269,9 @@ def test_upload_route_media_html_extension_not_served_as_html(
|
||||
assert "attachment" in pr.headers["content-disposition"].lower()
|
||||
|
||||
|
||||
def test_public_watch_file_forces_download_for_generic_file(as_user, make_user, _root, monkeypatch):
|
||||
def test_public_watch_file_forces_download_for_generic_file(
|
||||
as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
"""The login-free public serve of a generic upload (media_kind='file') must be a neutral
|
||||
force-download attachment — the branch that stops uploaded HTML/SVG executing on the public
|
||||
origin. This is a SEPARATE code path from the authenticated /file route, so it needs its own
|
||||
@@ -186,7 +280,8 @@ def test_public_watch_file_forces_download_for_generic_file(as_user, make_user,
|
||||
c = as_user(make_user())
|
||||
job = c.post(
|
||||
"/api/downloads/upload",
|
||||
files={"file": ("notes.html", b"<script>alert(1)</script>", "text/html")},
|
||||
content=b"<script>alert(1)</script>",
|
||||
headers=upload_headers("notes.html"),
|
||||
).json()
|
||||
assert job["asset"]["media_kind"] == "file"
|
||||
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": True}).json()
|
||||
@@ -197,7 +292,7 @@ def test_public_watch_file_forces_download_for_generic_file(as_user, make_user,
|
||||
|
||||
|
||||
def test_public_watch_file_serves_media_inline_with_real_type(
|
||||
as_user, make_user, _root, monkeypatch
|
||||
as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
"""A recognised media container IS still served inline with its real audio/video type (so it
|
||||
plays in the watch page) — the fix must not break legitimate playback. A stream-only link →
|
||||
@@ -206,7 +301,8 @@ def test_public_watch_file_serves_media_inline_with_real_type(
|
||||
c = as_user(make_user())
|
||||
job = c.post(
|
||||
"/api/downloads/upload",
|
||||
files={"file": ("clip.mp4", b"fake-mp4-bytes", "video/mp4")},
|
||||
content=b"fake-mp4-bytes",
|
||||
headers=upload_headers("clip.mp4"),
|
||||
).json()
|
||||
link = c.post(f"/api/downloads/{job['id']}/links", json={"allow_download": False}).json()
|
||||
pr = c.get(f"/api/public/watch/{link['token']}/file")
|
||||
@@ -215,7 +311,9 @@ def test_public_watch_file_serves_media_inline_with_real_type(
|
||||
assert "inline" in pr.headers["content-disposition"].lower()
|
||||
|
||||
|
||||
def test_upload_route_enforces_per_file_cap(as_user, make_user, db, _root, monkeypatch):
|
||||
def test_upload_route_enforces_per_file_cap(
|
||||
as_user, make_user, db, _root, monkeypatch, upload_headers
|
||||
):
|
||||
from app import sysconfig
|
||||
|
||||
sysconfig.set_value(db, "download_upload_max_bytes", 5)
|
||||
@@ -223,47 +321,48 @@ def test_upload_route_enforces_per_file_cap(as_user, make_user, db, _root, monke
|
||||
c = as_user(make_user())
|
||||
r = c.post(
|
||||
"/api/downloads/upload",
|
||||
files={"file": ("big.bin", b"way over the cap", "application/octet-stream")},
|
||||
content=b"way over the cap",
|
||||
headers=upload_headers("big.bin"),
|
||||
)
|
||||
assert r.status_code == 422
|
||||
assert r.json()["detail"]["code"] == "upload_too_large"
|
||||
|
||||
|
||||
def test_upload_cap_cannot_be_disabled_with_zero(db):
|
||||
"""download_upload_max_bytes floors at 1: a stored 0 would read falsy in the route's
|
||||
`if cap and size > cap` guard and SILENTLY disable the per-file cap, so set_value must reject it.
|
||||
Pins the min=1 invariant so a revert to min=0 can't pass the gate."""
|
||||
"""download_upload_max_bytes floors at 1, and set_value must enforce it.
|
||||
|
||||
The falsy-0 footgun this originally guarded is gone — the route's old `if cap and size > cap`
|
||||
read a stored 0 as "no cap at all", while `capped_stream_to` compares unconditionally and a 0
|
||||
would refuse everything instead. Refusing everything is the safe direction, but it is still
|
||||
wrong, and the knob is also the number the admin page and the 422 `limit` advertise. Pins the
|
||||
min=1 invariant so a revert to min=0 can't pass the gate."""
|
||||
from app import sysconfig
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
sysconfig.set_value(db, "download_upload_max_bytes", 0)
|
||||
|
||||
|
||||
def test_upload_route_rejects_empty_file(as_user, make_user, _root, monkeypatch):
|
||||
def test_upload_route_rejects_empty_file(as_user, make_user, _root, monkeypatch, upload_headers):
|
||||
c = as_user(make_user())
|
||||
r = c.post(
|
||||
"/api/downloads/upload", files={"file": ("empty.bin", b"", "application/octet-stream")}
|
||||
)
|
||||
r = c.post("/api/downloads/upload", content=b"", headers=upload_headers("empty.bin"))
|
||||
assert r.status_code == 400
|
||||
assert r.json()["detail"]["code"] == "upload_empty"
|
||||
|
||||
|
||||
def test_redownload_rejects_an_upload(as_user, make_user, _root, monkeypatch):
|
||||
def test_redownload_rejects_an_upload(as_user, make_user, _root, monkeypatch, upload_headers):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
c = as_user(make_user())
|
||||
job = c.post(
|
||||
"/api/downloads/upload", files={"file": ("a.bin", b"data", "application/octet-stream")}
|
||||
).json()
|
||||
job = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("a.bin")).json()
|
||||
r = c.post(f"/api/downloads/{job['id']}/redownload")
|
||||
assert r.status_code == 409
|
||||
|
||||
|
||||
def test_edit_rejects_a_generic_upload_source(as_user, make_user, _root, monkeypatch):
|
||||
def test_edit_rejects_a_generic_upload_source(
|
||||
as_user, make_user, _root, monkeypatch, upload_headers
|
||||
):
|
||||
_probe(monkeypatch, (None, None, None, None, None))
|
||||
c = as_user(make_user())
|
||||
job = c.post(
|
||||
"/api/downloads/upload", files={"file": ("a.bin", b"data", "application/octet-stream")}
|
||||
).json()
|
||||
job = c.post("/api/downloads/upload", content=b"data", headers=upload_headers("a.bin")).json()
|
||||
r = c.post(
|
||||
"/api/downloads/edit",
|
||||
json={"source_job_id": job["id"], "edit_spec": {"trim": {"start_s": 0, "end_s": 1}}},
|
||||
|
||||
Reference in New Issue
Block a user