Files
siftlode/backend/tests/test_files_api.py
T
peter 85aa917b38 feat(files): the Files page — the library in one aligned list, its folders in a rail
Files is the one library: every done file, the ones shared with me, or one
folder's, picked in a side rail (sources, own folders, the storage meter) and
mirrored to ?folder=. One table with every kind of value in its own column;
Copy link, Share and Save in sight, every action on a right click: play,
cut, rename in the row, move to an own folder or back home, keep, details,
re-download, delete.

The Downloads page keeps only the queue until the tray replaces it. The old
/api/downloads library routes are gone: list shared, upload, the details
PATCH, delete, the file and the usage now live under /api/files, and a
listed file says which source it came from. The job's "expired" flag is gone
with the state it described.
2026-10-11 05:34:25 +02:00

509 lines
21 KiB
Python

"""The Files library API (`routes.files`), DB-lane: folders (list, own create/rename/delete, a
folder's retention), the file list (done only, by folder, searched, sorted, shared with me), a
file's rename / move / keep / delete, the upload into a folder, and the file's hash.
Every route is also held to the caller's own rows: another user's folder or file answers like a
missing one.
"""
import base64
import hashlib
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import select
from app import sysconfig
from app.config import settings
from app.downloads import folders, service
from app.downloads import upload as uploadmod
from app.downloads.status import JobStatus
from app.models import AuditLog, DownloadJob, DownloadShare, FileFolder, MediaAsset
NOW = datetime.now(timezone.utc)
@pytest.fixture
def _root(tmp_path, monkeypatch):
monkeypatch.setattr(settings, "files_root", str(tmp_path))
return tmp_path
def _file(db, user_id, *, kind="youtube", ref=None, title="A title", size=10, rel=None, **kw):
"""A done library file: an asset with bytes on disk-path terms, and the user's job over it."""
n = db.execute(select(MediaAsset.id)).all()
ref = ref or f"vid{len(n):08d}"
asset = MediaAsset(
source_kind=kind,
source_ref=ref,
format_sig=f"sig-{ref}",
status="ready",
rel_path=rel or f"{ref}.mp4",
size_bytes=size,
ref_count=0,
title=title,
media_kind="video",
container="mp4",
)
db.add(asset)
db.flush()
job = DownloadJob(
user_id=user_id,
asset_id=asset.id,
source_kind=kind,
source_ref=ref,
profile_snapshot={},
status=kw.pop("status", JobStatus.DONE),
**kw,
)
db.add(job)
db.flush()
folders.place(db, job)
folders.stamp_expiry(db, job)
service.incr_ref_count(db, asset.id)
db.commit()
return job
def _own(db, user_id, name="Saját"):
folder = FileFolder(user_id=user_id, kind=folders.FolderKind.OWN, name=name)
db.add(folder)
db.commit()
return folder
def _audits(db, action):
return db.execute(select(AuditLog).where(AuditLog.action == action)).scalars().all()
# --- folders -------------------------------------------------------------------------------
def test_the_folder_list_counts_each_folders_done_files_and_is_the_callers_own(
db, make_user, as_user
):
alice, bob = make_user(), make_user()
_file(db, alice.id, size=10)
_file(db, alice.id, size=5)
_file(db, alice.id, kind="url", ref="https://vimeo.com/1", size=7)
_file(db, alice.id, status=JobStatus.RUNNING) # not in the library yet
_file(db, bob.id)
got = {
(f["kind"], f["source_key"]): (f["count"], f["size_bytes"])
for f in as_user(alice).get("/api/files/folders").json()
}
assert got == {("source", "youtube"): (2, 15), ("source", "site:vimeo.com"): (1, 7)}
def test_an_own_folder_is_made_once_per_name_whatever_its_case(db, make_user, as_user):
alice = make_user()
c = as_user(alice)
made = c.post("/api/files/folders", json={"name": "Nyaralás"})
assert made.status_code == 200, made.text
assert (made.json()["kind"], made.json()["name"]) == ("own", "Nyaralás")
again = c.post("/api/files/folders", json={"name": "NYARALÁS"})
assert again.status_code == 409
assert again.json()["detail"]["reason"] == "file_folder_name_taken"
blank = c.post("/api/files/folders", json={"name": " "})
assert blank.json()["detail"]["reason"] == "file_folder_name_required"
# Another user may have a folder of the same name.
assert as_user(make_user()).post("/api/files/folders", json={"name": "Nyaralás"}).is_success
assert len(_audits(db, "file_folder_create")) == 2
def test_only_an_own_folder_is_renamed_and_only_by_its_user(db, make_user, as_user):
alice = make_user()
c = as_user(alice)
src = db.get(FileFolder, _file(db, alice.id).folder_id)
own = _own(db, alice.id, "Régi")
taken = _own(db, alice.id, "Foglalt")
ok = c.patch(f"/api/files/folders/{own.id}", json={"name": "Új"})
assert ok.json()["name"] == "Új"
clash = c.patch(f"/api/files/folders/{own.id}", json={"name": taken.name.lower()})
assert clash.json()["detail"]["reason"] == "file_folder_name_taken"
no = c.patch(f"/api/files/folders/{src.id}", json={"name": "YouTube 2"})
assert no.status_code == 409 and no.json()["detail"]["reason"] == "file_folder_not_own"
stranger = as_user(make_user()).patch(f"/api/files/folders/{own.id}", json={"name": "X"})
assert stranger.status_code == 404
db.refresh(own)
assert own.name == "Új"
def test_a_shorter_retention_moves_each_files_expiry_but_never_inside_the_warning_window(
db, make_user, as_user
):
sysconfig.set_value(db, "download_gc_grace_days", 3)
db.commit()
alice = make_user()
fresh = _file(db, alice.id) # expires in 30 days
old = _file(db, alice.id)
old.expires_at = NOW + timedelta(days=10) # 20 days old already
kept = _file(db, alice.id, kept=True)
db.commit()
folder_id = fresh.folder_id
r = as_user(alice).patch(f"/api/files/folders/{folder_id}", json={"retention_days": 20})
assert r.json()["retention_days"] == 20
for j in (fresh, old, kept):
db.refresh(j)
# 30 → 20 days: each loses 10 — the one with 10 left would be due now, so it gets the window.
assert abs(fresh.expires_at - (NOW + timedelta(days=20))) < timedelta(minutes=5)
assert abs(old.expires_at - (NOW + timedelta(days=3))) < timedelta(minutes=5)
assert kept.expires_at is None
forever = as_user(alice).patch(f"/api/files/folders/{folder_id}", json={"retention_days": None})
assert forever.json()["retention_days"] is None
db.refresh(fresh)
assert fresh.expires_at is None
rows = _audits(db, "file_folder_retention")
assert [(a.before, a.after) for a in rows] == [
({"retention_days": 30}, {"retention_days": 20}),
({"retention_days": 20}, {"retention_days": None}),
]
def test_a_retention_set_on_a_folder_that_kept_forever_starts_its_files_clock_now(
db, make_user, as_user
):
alice = make_user()
up = _file(db, alice.id, kind="upload", ref="u1")
assert up.expires_at is None
r = as_user(alice).patch(f"/api/files/folders/{up.folder_id}", json={"retention_days": 7})
assert r.status_code == 200, r.text
db.refresh(up)
assert abs(up.expires_at - (NOW + timedelta(days=7))) < timedelta(minutes=5)
def test_deleting_an_own_folder_sends_its_files_home_and_a_source_folder_cannot_be_deleted(
db, make_user, as_user
):
alice = make_user()
c = as_user(alice)
yt = _file(db, alice.id)
site = _file(db, alice.id, kind="url", ref="https://vimeo.com/2")
home = {yt.id: yt.folder_id, site.id: site.folder_id}
own = _own(db, alice.id)
for j in (yt, site):
assert c.patch(f"/api/files/{j.id}", json={"folder_id": own.id}).is_success
r = c.delete(f"/api/files/folders/{own.id}")
assert r.json() == {"ok": True, "deleted": own.id, "files_moved": 2}
assert db.get(FileFolder, own.id) is None
for j in (yt, site):
db.refresh(j)
assert j.folder_id == home[j.id]
# Back in a source folder, its clock runs again from now.
assert abs(j.expires_at - (NOW + timedelta(days=30))) < timedelta(minutes=5)
src = c.delete(f"/api/files/folders/{yt.folder_id}")
assert src.status_code == 409 and src.json()["detail"]["reason"] == "file_folder_not_own"
other = _own(db, alice.id, "Más")
assert as_user(make_user()).delete(f"/api/files/folders/{other.id}").status_code == 404
assert len(_audits(db, "file_folder_delete")) == 1
# --- the file list -------------------------------------------------------------------------
def test_the_list_holds_the_callers_done_files_by_folder(db, make_user, as_user):
alice, bob = make_user(), make_user()
yt = _file(db, alice.id)
site = _file(db, alice.id, kind="url", ref="https://vimeo.com/3")
_file(db, alice.id, status=JobStatus.QUEUED)
_file(db, bob.id)
c = as_user(alice)
assert {f["id"] for f in c.get("/api/files").json()} == {yt.id, site.id}
only = c.get("/api/files", params={"folder_id": site.folder_id}).json()
assert [f["id"] for f in only] == [site.id]
assert only[0]["folder_id"] == site.folder_id and only[0]["shared"] is False
# Bob's folder id names nothing of Alice's.
bobs = db.execute(select(FileFolder.id).where(FileFolder.user_id == bob.id)).scalar()
assert c.get("/api/files", params={"folder_id": bobs}).json() == []
def test_the_list_is_searched_by_name_and_channel_with_wildcards_taken_literally(
db, make_user, as_user
):
alice = make_user()
hit = _file(db, alice.id, title="Balaton 100% nyár")
by_channel = _file(db, alice.id, title="Más", display_uploader="Balatoni csatorna")
_file(db, alice.id, title="Balaton 1000 nyár")
c = as_user(alice)
assert [f["id"] for f in c.get("/api/files", params={"q": "100%"}).json()] == [hit.id]
found = {f["id"] for f in c.get("/api/files", params={"q": "balatoni"}).json()}
assert found == {by_channel.id}
def test_the_list_sorts_by_name_size_or_when_added(db, make_user, as_user):
alice = make_user()
b = _file(db, alice.id, title="Béka", size=30)
a = _file(db, alice.id, title="alma", size=10)
z = _file(db, alice.id, title="Zebra", size=20)
c = as_user(alice)
def ids(**params):
return [f["id"] for f in c.get("/api/files", params=params).json()]
assert ids() == [z.id, a.id, b.id] # newest first
assert ids(sort="name", order="asc") == [a.id, b.id, z.id] # the reader's alphabet, any case
assert ids(sort="size", order="desc") == [b.id, z.id, a.id]
def test_shared_with_me_lists_what_others_shared_and_nothing_else(db, make_user, as_user):
alice, bob, carol = make_user(), make_user(), make_user()
theirs = _file(db, bob.id)
_file(db, bob.id) # not shared
for_carol = _file(db, bob.id)
db.add(DownloadShare(job_id=theirs.id, from_user_id=bob.id, to_user_id=alice.id))
db.add(DownloadShare(job_id=for_carol.id, from_user_id=bob.id, to_user_id=carol.id))
db.commit()
got = as_user(alice).get("/api/files", params={"scope": "shared"}).json()
assert [(f["id"], f["shared"]) for f in got] == [(theirs.id, True)]
def test_shared_with_me_is_ordered_by_when_it_was_shared_and_can_be_dismissed(
db, make_user, as_user
):
"""A friend's old file shared today is new to me: "added" is the share's date there. Dismissing
one drops only my grant; the owner's file stays."""
alice, bob = make_user(), make_user()
old = _file(db, bob.id, created_at=NOW - timedelta(days=30))
new = _file(db, bob.id, created_at=NOW - timedelta(days=1))
db.add(
DownloadShare(
job_id=new.id,
from_user_id=bob.id,
to_user_id=alice.id,
created_at=NOW - timedelta(hours=2),
)
)
db.add(
DownloadShare(
job_id=old.id,
from_user_id=bob.id,
to_user_id=alice.id,
created_at=NOW - timedelta(hours=1),
)
)
db.commit()
c = as_user(alice)
got = c.get("/api/files", params={"scope": "shared"}).json()
assert [f["id"] for f in got] == [old.id, new.id]
assert got[0]["shared_at"] is not None and got[0]["shared_at"] > got[1]["shared_at"]
r = c.delete(f"/api/files/shared/{old.id}")
assert r.json() == {"ok": True, "removed": old.id}
assert [f["id"] for f in c.get("/api/files", params={"scope": "shared"}).json()] == [new.id]
assert db.get(DownloadJob, old.id) is not None
def test_each_listed_file_says_where_it_came_from_wherever_it_sits(db, make_user, as_user):
"""The Source column: a file's own source folder, also after it moved into an own folder."""
alice = make_user()
yt = _file(db, alice.id)
site = _file(db, alice.id, kind="url", ref="https://www.tiktok.com/@x/video/1")
moved = _file(db, alice.id)
own = _own(db, alice.id)
c = as_user(alice)
assert c.patch(f"/api/files/{moved.id}", json={"folder_id": own.id}).status_code == 200
got = {f["id"]: f["source_key"] for f in c.get("/api/files").json()}
assert got == {yt.id: "youtube", site.id: "site:tiktok.com", moved.id: "youtube"}
# --- one file ------------------------------------------------------------------------------
def test_a_file_is_renamed_kept_and_given_back_to_its_folders_retention(db, make_user, as_user):
alice = make_user()
job = _file(db, alice.id)
c = as_user(alice)
assert c.patch(f"/api/files/{job.id}", json={"name": "Új név"}).json()["display_name"] == (
"Új név"
)
kept = c.patch(f"/api/files/{job.id}", json={"kept": True}).json()
assert (kept["kept"], kept["expires_at"]) == (True, None)
back = c.patch(f"/api/files/{job.id}", json={"kept": False}).json()
assert back["kept"] is False and back["expires_at"] is not None
# Setting what it already is records nothing.
assert c.patch(f"/api/files/{job.id}", json={"kept": False}).status_code == 200
assert len(_audits(db, "file_keep")) == 1 and len(_audits(db, "file_unkeep")) == 1
assert as_user(make_user()).patch(f"/api/files/{job.id}", json={"kept": True}).status_code == (
404
)
def test_a_files_channel_and_links_are_set_and_only_plausible_links_are_kept(
db, make_user, as_user
):
alice = make_user()
job = _file(db, alice.id)
c = as_user(alice)
got = c.patch(
f"/api/files/{job.id}",
json={
"uploader": " Saját csatorna ",
"uploader_url": "mailto:someone@example.com",
"extra_links": ["https://a.example/x", "ftp://b.example", "", 7, "http://[::1/x"]
+ [f"https://c.example/{i}" for i in range(10)],
},
).json()
assert got["display_uploader"] == "Saját csatorna"
assert got["display_uploader_url"] is None
assert got["extra_links"][0] == "https://a.example/x" and len(got["extra_links"]) == 8
cleared = c.patch(f"/api/files/{job.id}", json={"uploader": "", "extra_links": []}).json()
assert (cleared["display_uploader"], cleared["extra_links"]) == (None, [])
def test_the_usage_carries_the_quota_and_the_upload_cap(db, make_user, as_user):
got = as_user(make_user()).get("/api/files/usage").json()
assert got["upload_max_bytes"] == sysconfig.get_int(db, "download_upload_max_bytes")
assert "footprint_bytes" in got and "max_bytes" in got
def test_a_file_moves_to_an_own_folder_or_home_and_nowhere_else(db, make_user, as_user):
alice, bob = make_user(), make_user()
job = _file(db, alice.id)
home = job.folder_id
vimeo = db.get(FileFolder, _file(db, alice.id, kind="url", ref="https://vimeo.com/4").folder_id)
own = _own(db, alice.id)
bobs = _own(db, bob.id, "Bobé")
c = as_user(alice)
moved = c.patch(f"/api/files/{job.id}", json={"folder_id": own.id}).json()
assert (moved["folder_id"], moved["expires_at"]) == (own.id, None) # an own folder keeps it
wrong = c.patch(f"/api/files/{job.id}", json={"folder_id": vimeo.id})
assert wrong.json()["detail"]["reason"] == "file_folder_not_its_source"
foreign = c.patch(f"/api/files/{job.id}", json={"folder_id": bobs.id})
assert foreign.json()["detail"]["reason"] == "file_folder_gone"
home_again = c.patch(f"/api/files/{job.id}", json={"folder_id": home}).json()
assert home_again["folder_id"] == home and home_again["expires_at"] is not None
assert [(a.before, a.after) for a in _audits(db, "file_move")] == [
({"folder_id": home}, {"folder_id": own.id}),
({"folder_id": own.id}, {"folder_id": home}),
]
def test_a_deleted_file_leaves_the_library_and_the_log_says_so(db, make_user, as_user, _root):
alice = make_user()
job = _file(db, alice.id)
assert as_user(make_user()).delete(f"/api/files/{job.id}").status_code == 404
r = as_user(alice).delete(f"/api/files/{job.id}")
assert r.json() == {"ok": True, "deleted": job.id}
db.expire_all()
assert db.get(DownloadJob, job.id) is None
[row] = _audits(db, "file_delete")
assert (row.actor_id, row.target_id) == (alice.id, str(job.id))
# --- upload, the file, its hash ------------------------------------------------------------
@pytest.fixture
def _no_probe(monkeypatch):
# A file ffprobe cannot read: a generic file, done at once (no conversion).
monkeypatch.setattr(uploadmod.probe, "probe_json", lambda _p: None)
def test_an_upload_lands_in_the_folder_it_asked_for_with_its_hash(
db, make_user, as_user, _root, _no_probe, upload_headers
):
alice = make_user()
own = _own(db, alice.id)
body = b"hello files"
c = as_user(alice)
r = c.post("/api/files/upload", content=body, headers=upload_headers("a.txt", folder=own.id))
assert r.status_code == 200, r.text
job = db.get(DownloadJob, r.json()["id"])
assert job.folder_id == own.id and job.expires_at is None
asset = db.get(MediaAsset, job.asset_id)
assert asset.sha256 == hashlib.sha256(body).hexdigest()
plain = c.post("/api/files/upload", content=b"x", headers=upload_headers("b.txt")).json()
assert db.get(FileFolder, plain["folder_id"]).source_key == "upload"
assert len(_audits(db, "file_upload")) == 2
@pytest.mark.parametrize(
("which", "reason"),
[
("site", "file_folder_not_its_source"), # YouTube's folder: no upload came from there
("foreign", "file_folder_gone"),
("garbage", "file_folder_gone"),
],
)
def test_an_upload_to_a_folder_it_may_not_use_is_refused_before_its_body_is_read(
db, make_user, as_user, _root, _no_probe, upload_headers, monkeypatch, which, reason
):
from app.routes import files as files_routes
alice, bob = make_user(), make_user()
folder = {
"site": lambda: _file(db, alice.id).folder_id,
"foreign": lambda: _own(db, bob.id).id,
"garbage": lambda: "12abc",
}[which]()
streamed = []
async def spy(*args, **kwargs):
streamed.append(True)
raise AssertionError("the body was read")
monkeypatch.setattr(files_routes, "capped_stream_to", spy)
before = db.execute(select(DownloadJob.id)).all()
r = as_user(alice).post(
"/api/files/upload", content=b"data", headers=upload_headers("a.txt", folder=folder)
)
assert (r.status_code, r.json()["detail"]["reason"]) == (409, reason)
assert streamed == []
assert db.execute(select(DownloadJob.id)).all() == before
def test_a_folder_deleted_while_its_upload_streams_refuses_the_upload(
db, make_user, as_user, _root, _no_probe, upload_headers, monkeypatch
):
from app.routes import files as files_routes
alice = make_user()
own = _own(db, alice.id)
real = files_routes.capped_stream_to
async def deleting_meanwhile(*args, **kwargs):
size = await real(*args, **kwargs)
db.delete(db.get(FileFolder, own.id))
db.commit()
return size
monkeypatch.setattr(files_routes, "capped_stream_to", deleting_meanwhile)
r = as_user(alice).post(
"/api/files/upload", content=b"data", headers=upload_headers("a.txt", folder=own.id)
)
assert (r.status_code, r.json()["detail"]["reason"]) == (409, "file_folder_gone")
assert db.execute(select(DownloadJob.id).where(DownloadJob.user_id == alice.id)).all() == []
assert not (_root / ".uploads").exists() # checked before the file was placed
def test_the_file_carries_its_hash_once_known_and_the_hash_is_taken_on_first_ask(
db, make_user, as_user, _root
):
alice, friend = make_user(), make_user()
body = b"\x00\x01video bytes"
job = _file(db, alice.id, rel="v.mp4")
(_root / "v.mp4").write_bytes(body)
c = as_user(alice)
first = c.get(f"/api/files/{job.id}/file")
assert first.status_code == 200 and first.content == body
assert "repr-digest" not in first.headers # not hashed yet: nothing claimed
want = hashlib.sha256(body).hexdigest()
assert c.get(f"/api/files/{job.id}/sha256").json() == {"sha256": want}
assert db.get(MediaAsset, job.asset_id).sha256 == want
ranged = c.get(f"/api/files/{job.id}/file", headers={"Range": "bytes=0-3"})
assert ranged.status_code == 206
digest = base64.b64encode(bytes.fromhex(want)).decode()
assert ranged.headers["repr-digest"] == f"sha-256=:{digest}:"
# A friend it was shared with may read it too; a stranger is told it does not exist.
db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id))
db.commit()
assert as_user(friend).get(f"/api/files/{job.id}/sha256").json() == {"sha256": want}
assert as_user(make_user()).get(f"/api/files/{job.id}/sha256").status_code == 404
def test_a_new_file_at_an_assets_path_forgets_the_old_hash(db, make_user):
job = _file(db, make_user().id)
asset = db.get(MediaAsset, job.asset_id)
asset.sha256 = "ab" * 32
asset.rel_path = "converted.mp4"
assert asset.sha256 is None