feat(files): a file that is not media opens in a viewer — an image, a PDF, text as text
- GET /api/files/{id}/preview: an allow-list on the stored extension (downloads/preview.py) picks
image / PDF / text; served inline under `CSP: sandbox; frame-ancestors 'self'`, the type from the
table; HTML and SVG arrive as plain text; a text is its first 1 MiB (X-Preview-Truncated), sent
as UTF-8 whatever it was written in (BOM, UTF-8, else cp1250); a NUL means binary: 415
- the list says which preview each file gets (asset.preview); FE/BE contract on kinds and header
- the viewer (shared dialog shell): ← / → through the list's such files in its order, Save, open in
a tab, the file's menu; one stage height for every kind; a no-preview card offers the file
- the name, the card's picture and the menu's Preview open it; an image up to 8 MB is its own
thumbnail
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
"""What a library file previews as — the single home of the preview allow-list.
|
||||
|
||||
A generic file (`media_kind == "file"`; video and audio have their players) previews by its stored
|
||||
`container`, the upload's cleaned extension, and ONLY through this table: an image is shown as an
|
||||
image, a PDF in the browser's own viewer, and a text format as plain text. Anything else has no
|
||||
preview. The served Content-Type comes from here too, never from a guess on the name, and HTML / SVG
|
||||
sit in the TEXT list on purpose: rendered on this origin, an uploaded page would run with the
|
||||
reader's session. The route adds `CSP: sandbox` on top, so even a mislabelled file (a "png" that is
|
||||
really a page) gets no script and no origin.
|
||||
|
||||
The serializer hands the kind to the client (`asset.preview`), so the library and the server can
|
||||
never disagree on which files open a preview.
|
||||
"""
|
||||
|
||||
import codecs
|
||||
from typing import Literal
|
||||
|
||||
from app.downloads import mediakind
|
||||
|
||||
PreviewKind = Literal["image", "pdf", "text"]
|
||||
|
||||
IMAGE_TYPES: dict[str, str] = {
|
||||
"png": "image/png",
|
||||
"jpg": "image/jpeg",
|
||||
"jpeg": "image/jpeg",
|
||||
"gif": "image/gif",
|
||||
"webp": "image/webp",
|
||||
"avif": "image/avif",
|
||||
"bmp": "image/bmp",
|
||||
}
|
||||
|
||||
PDF_TYPE = "application/pdf"
|
||||
|
||||
# Read as text: notes, data, subtitles, config and source. HTML and SVG belong here (see above).
|
||||
TEXT_EXTS: frozenset[str] = frozenset(
|
||||
{
|
||||
"txt", "text", "md", "markdown", "log", "csv", "tsv", "json", "jsonl", "ndjson", "xml",
|
||||
"yaml", "yml", "toml", "ini", "conf", "cfg", "properties", "env", "srt", "vtt", "ass", "ssa",
|
||||
"sub", "nfo", "cue", "m3u", "m3u8", "html", "htm", "xhtml", "svg", "css", "scss", "less",
|
||||
"js", "mjs", "cjs", "ts", "tsx", "jsx", "py", "rb", "php", "pl", "lua", "go", "rs", "java",
|
||||
"kt", "kts", "swift", "c", "h", "cc", "cpp", "hpp", "cs", "sh", "bash", "zsh", "fish", "ps1",
|
||||
"bat", "cmd", "sql", "r", "dart", "diff", "patch", "tex", "rst", "adoc", "gradle", "dockerfile",
|
||||
}
|
||||
) # fmt: skip
|
||||
|
||||
TEXT_TYPE = "text/plain; charset=utf-8"
|
||||
|
||||
# A text preview is the file's head: enough to read, never a whole log loaded into a tab.
|
||||
TEXT_PREVIEW_BYTES = 1 << 20
|
||||
# A NUL byte in the first this-many bytes means the "text" is not text.
|
||||
BINARY_SNIFF_BYTES = 8192
|
||||
|
||||
|
||||
def kind(media_kind: str | None, container: str | None) -> PreviewKind | None:
|
||||
"""The preview a library file gets, or None (a media file plays instead; others download)."""
|
||||
if media_kind != mediakind.FILE:
|
||||
return None
|
||||
ext = (container or "").lower()
|
||||
if ext in IMAGE_TYPES:
|
||||
return "image"
|
||||
if ext == "pdf":
|
||||
return "pdf"
|
||||
if ext in TEXT_EXTS:
|
||||
return "text"
|
||||
return None
|
||||
|
||||
|
||||
def media_type(preview: PreviewKind, container: str | None) -> str:
|
||||
"""The Content-Type a preview is served with: from this table, never guessed from the name."""
|
||||
if preview == "image":
|
||||
return IMAGE_TYPES[(container or "").lower()]
|
||||
if preview == "pdf":
|
||||
return PDF_TYPE
|
||||
return TEXT_TYPE
|
||||
|
||||
|
||||
# A text that is not UTF-8 is read as Central European Windows: what an old Hungarian subtitle or
|
||||
# note is written in (its ő and ű are not in Latin-1).
|
||||
FALLBACK_ENCODING = "cp1250"
|
||||
_BOMS = (
|
||||
(codecs.BOM_UTF8, "utf-8-sig"),
|
||||
(codecs.BOM_UTF16_LE, "utf-16"),
|
||||
(codecs.BOM_UTF16_BE, "utf-16"),
|
||||
)
|
||||
|
||||
|
||||
def looks_binary(head: bytes) -> bool:
|
||||
"""Whether the start of a "text" file is binary after all (a NUL in it)."""
|
||||
return b"\x00" in head[:BINARY_SNIFF_BYTES]
|
||||
|
||||
|
||||
def text_utf8(head: bytes) -> bytes | None:
|
||||
"""A text file's head as UTF-8, the one charset the preview is served in; None when it is not text.
|
||||
A byte-order mark names its encoding (UTF-16 is full of NULs, so it is read before the binary
|
||||
check); else UTF-8 is kept as it is — a character cut at the size cap included, the reader shows
|
||||
that one as a replacement mark — and anything else is read as `FALLBACK_ENCODING`."""
|
||||
for bom, encoding in _BOMS:
|
||||
if head.startswith(bom):
|
||||
return head.decode(encoding, errors="replace").encode()
|
||||
if looks_binary(head):
|
||||
return None
|
||||
try:
|
||||
head.decode("utf-8")
|
||||
except UnicodeDecodeError as e: # silent-ok: not UTF-8 is an answer — read it as the fallback
|
||||
if not (e.reason == "unexpected end of data" and e.end == len(head)):
|
||||
return head.decode(FALLBACK_ENCODING, errors="replace").encode()
|
||||
return head
|
||||
+2
-1
@@ -268,7 +268,8 @@ async def static_cache_headers(request, call_next):
|
||||
# Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning:
|
||||
# nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to
|
||||
# external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be
|
||||
# iframed anywhere (we never embed our own pages; we embed OTHERS, which this does not affect), and
|
||||
# iframed anywhere (we embed OTHERS, which this does not affect; the one own answer the app frames, a
|
||||
# library file's preview, sets its own `frame-ancestors 'self'` — these are defaults, `setdefault`), and
|
||||
# Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates
|
||||
# the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and
|
||||
# suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy
|
||||
|
||||
@@ -21,7 +21,7 @@ from typing import BinaryIO, Literal
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -33,7 +33,7 @@ from app.auth import current_user
|
||||
from app.collation import collated
|
||||
from app.config import settings
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.downloads import folders, quota, service, storage
|
||||
from app.downloads import folders, preview, quota, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.status import JobStatus
|
||||
from app.mailtext import lang_for_user
|
||||
@@ -117,6 +117,8 @@ def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"media_kind": asset.media_kind,
|
||||
"uploader_url": asset.uploader_url,
|
||||
"thumbnail_url": asset.thumbnail_url,
|
||||
# "image" | "pdf" | "text" | null — what `/preview` shows (`downloads/preview.py`).
|
||||
"preview": preview.kind(asset.media_kind, asset.container),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -784,6 +786,56 @@ def get_file(job_id: int, user: User = Depends(current_user), db: Session = Depe
|
||||
return response
|
||||
|
||||
|
||||
# A preview's own headers. `sandbox` leaves the answer no script and an opaque origin, whatever the
|
||||
# file turns out to hold; `frame-ancestors 'self'` lets the app's viewer frame a PDF (every other
|
||||
# answer keeps the app-wide 'none', main.py), with X-Frame-Options saying the same for old engines.
|
||||
PREVIEW_HEADERS = {
|
||||
"Content-Security-Policy": "sandbox; frame-ancestors 'self'",
|
||||
"X-Frame-Options": "SAMEORIGIN",
|
||||
}
|
||||
# Set on a text preview cut at `preview.TEXT_PREVIEW_BYTES`, so the viewer can say it shows the start.
|
||||
TRUNCATED_HEADER = "X-Preview-Truncated"
|
||||
|
||||
|
||||
@router.get("/{job_id}/preview")
|
||||
def preview_file(
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
"""The file shown in the library's viewer: an image or a PDF inline, a text format as plain text
|
||||
(its first `TEXT_PREVIEW_BYTES`). The type comes from `downloads/preview.py`'s allow-list, so
|
||||
an HTML or SVG upload arrives as text, never as a page. Anything else: 415 `file_no_preview`."""
|
||||
job = accessible_job(db, user, job_id)
|
||||
asset, path = _ready_file(db, job)
|
||||
kind = preview.kind(asset.media_kind, asset.container)
|
||||
if kind is None:
|
||||
raise refused(415, "file_no_preview")
|
||||
headers = dict(PREVIEW_HEADERS)
|
||||
if kind == "text":
|
||||
with path.open("rb") as f:
|
||||
head = f.read(preview.TEXT_PREVIEW_BYTES + 1)
|
||||
if len(head) > preview.TEXT_PREVIEW_BYTES:
|
||||
head = head[: preview.TEXT_PREVIEW_BYTES]
|
||||
headers[TRUNCATED_HEADER] = "1"
|
||||
text = preview.text_utf8(head)
|
||||
if text is None:
|
||||
raise refused(415, "file_no_preview")
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return Response(text, media_type=preview.media_type(kind, asset.container), headers=headers)
|
||||
filename = storage.download_filename(
|
||||
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||
)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return FileResponse(
|
||||
path,
|
||||
filename=filename,
|
||||
media_type=preview.media_type(kind, asset.container),
|
||||
content_disposition_type="inline",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/sha256")
|
||||
def get_sha256(
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
"""A library file's preview (`GET /api/files/{id}/preview`, `downloads/preview.py`), DB-lane.
|
||||
|
||||
What it may show is an allow-list on the stored extension: an image or a PDF inline, a text format
|
||||
(HTML and SVG among them) as plain text, its head only. Every answer is sandboxed and may be framed
|
||||
by the app alone; anything else, a binary "text" included, has no preview.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import folders, preview, service
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadShare, MediaAsset
|
||||
from app.routes.files import TRUNCATED_HEADER
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _root(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _upload(db, root, user_id, ext, body, *, media_kind="file", title="notes"):
|
||||
"""A done uploaded file named `title.ext` holding `body`."""
|
||||
n = len(db.execute(select(MediaAsset.id)).all())
|
||||
rel = f"up{n}.{ext}" if ext else f"up{n}"
|
||||
(root / rel).write_bytes(body)
|
||||
asset = MediaAsset(
|
||||
source_kind="upload",
|
||||
source_ref=f"up{n}",
|
||||
format_sig=f"sig-up{n}",
|
||||
status="ready",
|
||||
rel_path=rel,
|
||||
size_bytes=len(body),
|
||||
ref_count=0,
|
||||
title=title,
|
||||
media_kind=media_kind,
|
||||
container=ext,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
source_kind="upload",
|
||||
source_ref=f"up{n}",
|
||||
profile_snapshot={},
|
||||
status=JobStatus.DONE,
|
||||
)
|
||||
db.add(job)
|
||||
db.flush()
|
||||
folders.place(db, job)
|
||||
service.incr_ref_count(db, asset.id)
|
||||
db.commit()
|
||||
return job
|
||||
|
||||
|
||||
def _assert_sandboxed(r):
|
||||
assert r.headers["content-security-policy"] == "sandbox; frame-ancestors 'self'"
|
||||
assert r.headers["x-frame-options"] == "SAMEORIGIN"
|
||||
assert r.headers["x-content-type-options"] == "nosniff"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ext,ctype",
|
||||
[
|
||||
("png", "image/png"),
|
||||
("JPG", "image/jpeg"),
|
||||
("webp", "image/webp"),
|
||||
("pdf", "application/pdf"),
|
||||
],
|
||||
)
|
||||
def test_an_image_or_a_pdf_is_shown_inline_with_its_listed_type_in_a_sandbox(
|
||||
db, make_user, as_user, _root, ext, ctype
|
||||
):
|
||||
alice = make_user()
|
||||
job = _upload(db, _root, alice.id, ext, b"bytes of it")
|
||||
r = as_user(alice).get(f"/api/files/{job.id}/preview")
|
||||
assert r.status_code == 200 and r.content == b"bytes of it"
|
||||
assert r.headers["content-type"] == ctype
|
||||
assert r.headers["content-disposition"].startswith("inline;")
|
||||
assert f'filename="notes.{ext}"' in r.headers["content-disposition"]
|
||||
_assert_sandboxed(r)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ext", ["html", "svg", "txt", "py"])
|
||||
def test_html_svg_and_code_arrive_as_plain_text_never_as_a_page(db, make_user, as_user, _root, ext):
|
||||
alice = make_user()
|
||||
body = b"<svg onload=x><script>y</script></svg>"
|
||||
job = _upload(db, _root, alice.id, ext, body)
|
||||
r = as_user(alice).get(f"/api/files/{job.id}/preview")
|
||||
assert r.status_code == 200 and r.content == body
|
||||
assert r.headers["content-type"] == "text/plain; charset=utf-8"
|
||||
assert TRUNCATED_HEADER.lower() not in r.headers
|
||||
_assert_sandboxed(r)
|
||||
|
||||
|
||||
def test_a_long_text_is_cut_to_its_head_and_says_so(db, make_user, as_user, _root):
|
||||
alice = make_user()
|
||||
body = b"a" * preview.TEXT_PREVIEW_BYTES + b"tail"
|
||||
job = _upload(db, _root, alice.id, "log", body)
|
||||
r = as_user(alice).get(f"/api/files/{job.id}/preview")
|
||||
assert r.status_code == 200 and len(r.content) == preview.TEXT_PREVIEW_BYTES
|
||||
assert r.headers[TRUNCATED_HEADER] == "1"
|
||||
exact = _upload(db, _root, alice.id, "log", b"a" * preview.TEXT_PREVIEW_BYTES)
|
||||
whole = as_user(alice).get(f"/api/files/{exact.id}/preview")
|
||||
assert len(whole.content) == preview.TEXT_PREVIEW_BYTES
|
||||
assert TRUNCATED_HEADER.lower() not in whole.headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ext,body,media_kind",
|
||||
[
|
||||
("zip", b"PK\x03\x04", "file"), # not on the list
|
||||
("docx", b"PK\x03\x04", "file"),
|
||||
(None, b"plain", "file"), # no extension
|
||||
("txt", b"text\x00with a nul", "file"), # a "text" that is binary
|
||||
("mp4", b"video", "video"), # media plays in the player instead
|
||||
("png", b"png", "video"), # an extension alone never makes media a preview
|
||||
],
|
||||
)
|
||||
def test_anything_else_has_no_preview(db, make_user, as_user, _root, ext, body, media_kind):
|
||||
alice = make_user()
|
||||
job = _upload(db, _root, alice.id, ext, body, media_kind=media_kind)
|
||||
r = as_user(alice).get(f"/api/files/{job.id}/preview")
|
||||
assert r.status_code == 415
|
||||
assert r.json()["detail"] == {"code": "refused", "reason": "file_no_preview"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
"Árvíztűrő tükörfúrógép".encode("cp1250"), # an old Hungarian subtitle
|
||||
"Árvíztűrő tükörfúrógép".encode("utf-8-sig"),
|
||||
"Árvíztűrő tükörfúrógép".encode("utf-16"), # a BOM, then NULs: text all the same
|
||||
"Árvíztűrő tükörfúrógép".encode("utf-8"),
|
||||
],
|
||||
)
|
||||
def test_a_text_arrives_as_utf8_whatever_it_was_written_in(db, make_user, as_user, _root, body):
|
||||
alice = make_user()
|
||||
job = _upload(db, _root, alice.id, "srt", body)
|
||||
r = as_user(alice).get(f"/api/files/{job.id}/preview")
|
||||
assert r.status_code == 200
|
||||
assert r.content.decode("utf-8") == "Árvíztűrő tükörfúrógép"
|
||||
|
||||
|
||||
def test_a_character_cut_at_the_cap_keeps_the_text_utf8(db, make_user, as_user, _root):
|
||||
alice = make_user()
|
||||
body = b"a" * (preview.TEXT_PREVIEW_BYTES - 1) + "ő".encode() # its 2nd byte past the cap
|
||||
job = _upload(db, _root, alice.id, "txt", body)
|
||||
r = as_user(alice).get(f"/api/files/{job.id}/preview")
|
||||
assert r.content == body[: preview.TEXT_PREVIEW_BYTES] # not read as cp1250
|
||||
|
||||
|
||||
def test_the_list_says_which_preview_each_file_gets(db, make_user, as_user, _root):
|
||||
alice = make_user()
|
||||
for ext in ["png", "pdf", "html", "zip"]:
|
||||
_upload(db, _root, alice.id, ext, b"x", title=ext)
|
||||
_upload(db, _root, alice.id, "mp4", b"x", media_kind="video", title="mp4")
|
||||
got = {
|
||||
f["asset"]["container"]: f["asset"]["preview"]
|
||||
for f in as_user(alice).get("/api/files").json()
|
||||
}
|
||||
assert got == {"png": "image", "pdf": "pdf", "html": "text", "zip": None, "mp4": None}
|
||||
|
||||
|
||||
def test_a_preview_is_the_holders_or_a_friends_it_was_shared_with(db, make_user, as_user, _root):
|
||||
alice, friend = make_user(), make_user()
|
||||
job = _upload(db, _root, alice.id, "png", b"img")
|
||||
assert as_user(make_user()).get(f"/api/files/{job.id}/preview").status_code == 404
|
||||
db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id))
|
||||
db.commit()
|
||||
assert as_user(friend).get(f"/api/files/{job.id}/preview").status_code == 200
|
||||
Reference in New Issue
Block a user