"""API tokens for the Files library (`app.apitokens`, `routes.tokens`), DB-lane. A token is made, listed and revoked by its signed-in user only; its value is in the answer that made it and nowhere after. With it a machine reaches `/api/files` and nothing else, and there only the folders its grants name, with the rights they give — every right checked in a folder it has and in one it has not. A token that is malformed, unknown, expired, revoked, or whose account is suspended holds nothing, and a client trying tokens is slowed down. """ from datetime import datetime, timedelta, timezone import pytest from fastapi import Depends, Request from fastapi.testclient import TestClient from sqlalchemy import select, update from app import apitokens, audit from app.apitokens import files_caller from app.auth import current_user from app.downloads import folders, service from app.downloads import gc as gcmod from app.downloads import upload as uploadmod from app.downloads.status import JobStatus from app.config import settings from app.db import SessionLocal, get_db from app.main import app from app.notifications import create_notification from app.models import ApiToken, ApiTokenGrant, AuditLog, Notification, User from app.downloads.models import DownloadJob, FileFolder, MediaAsset from app.ratelimit import RateLimiter from app.routes import tokens as tokens_routes from app.routes.auth import bump_session_epoch from app.security import hash_token @pytest.fixture def _root(tmp_path, monkeypatch): monkeypatch.setattr(settings, "files_root", str(tmp_path)) return tmp_path @pytest.fixture def _no_probe(monkeypatch): # A file ffprobe cannot read: a generic file, done at once (no conversion). monkeypatch.setattr(uploadmod.probe, "probe_json", lambda _p: None) def _own(db, user_id, name): folder = FileFolder(user_id=user_id, kind=folders.FolderKind.OWN, name=name) db.add(folder) db.commit() return folder def _file_in(db, root, user_id, folder, name): """A done file of `user_id`'s with bytes on disk, sitting in `folder`.""" asset = MediaAsset( source_kind="upload", source_ref=name, format_sig=f"sig-{name}", status="ready", rel_path=f"{name}.bin", size_bytes=4, ref_count=0, title=name, media_kind="file", container="bin", ) db.add(asset) db.flush() job = DownloadJob( user_id=user_id, asset_id=asset.id, source_kind="upload", source_ref=name, profile_snapshot={}, status=JobStatus.DONE, ) db.add(job) db.flush() folders.place(db, job) job.folder_id = folder.id folders.stamp_done(db, job) service.incr_ref_count(db, asset.id) db.commit() (root / f"{name}.bin").write_bytes(b"data") return job def _bearer(raw: str) -> TestClient: return TestClient(app, headers={"authorization": f"Bearer {raw}"}) def _make(c, **body) -> dict: r = c.post("/api/tokens", json={"name": "a machine", **body}) assert r.status_code == 200, r.text return r.json() def _audits(db, action): return db.execute(select(AuditLog).where(AuditLog.action == action)).scalars().all() # --- the token itself ---------------------------------------------------------------------------- def test_a_token_has_its_shape_and_a_checksum_that_refuses_a_typo(): raw = apitokens.mint() assert raw.startswith("sfl_") and len(raw) == apitokens.TOKEN_LEN assert apitokens.well_formed(raw) assert apitokens.mint() != raw flipped = raw[:10] + ("A" if raw[10] != "A" else "B") + raw[11:] assert not apitokens.well_formed(flipped) assert not apitokens.well_formed(raw[:-1]) assert not apitokens.well_formed("xxx_" + raw[4:]) # A header byte past ASCII arrives as one latin-1 character: refused, never a 500. assert not apitokens.well_formed(raw[:-1] + "é") def test_a_non_ascii_token_is_refused_as_any_bad_one(db, client): # The test client re-sends a high byte as two UTF-8 bytes, which arrive as two latin-1 # characters: one "A" fewer keeps the header a token's length, so it is the checksum that meets # them, not the length check. raw = ("Bearer sfl_" + "A" * 47 + "é").encode("latin-1") r = TestClient(app, headers={"authorization": raw}).get("/api/files/folders") assert r.status_code == 401 and r.json()["detail"]["reason"] == "not_authenticated" def test_the_hub_preset_makes_one_hub_folder_with_every_right_and_shows_the_token_once( db, make_user, as_user ): alice = make_user() c = as_user(alice) made = _make(c, preset="hub") raw = made["token"] assert apitokens.well_formed(raw) and made["prefix"] == raw[: apitokens.SHOWN_LEN] [grant] = made["grants"] assert grant["kind"] == "hub" and grant["rights"] == ["delete", "list", "read", "write"] # Only the hash is kept, and the value is not in the list. stored = db.execute(select(ApiToken)).scalar_one() assert stored.token_hash != raw and raw not in str(c.get("/api/tokens").json()) # A second hub token reuses the one Hub folder. again = _make(c, preset="hub") assert again["grants"][0]["folder_id"] == grant["folder_id"] hubs = db.execute(select(FileFolder).where(FileFolder.kind == "hub")).scalars().all() assert len(hubs) == 1 assert len(_audits(db, "api_token_create")) == 2 def test_a_token_is_granted_only_the_users_own_folders_and_writes_only_where_uploads_land( db, make_user, as_user ): alice, bob = make_user(), make_user() theirs = _own(db, bob.id, "Bob's") c = as_user(alice) r = c.post( "/api/tokens", json={"name": "x", "grants": [{"folder_id": theirs.id, "rights": ["read"]}]}, ) assert r.status_code == 409 and r.json()["detail"]["reason"] == "file_folder_gone" youtube = folders.source_folder(db, alice.id, folders.SourceKey.YOUTUBE) db.commit() r = c.post( "/api/tokens", json={"name": "x", "grants": [{"folder_id": youtube.id, "rights": ["write"]}]}, ) assert r.json()["detail"]["reason"] == "file_folder_not_its_source" assert c.post("/api/tokens", json={"name": "x"}).json()["detail"]["reason"] == ( "api_token_no_grants" ) assert db.execute(select(ApiToken)).all() == [] @pytest.mark.parametrize("right", ["read", "write"]) def test_a_folder_granted_is_held_until_the_token_is_in(db, make_user, right): """A delete of the folder at the same moment waits and takes the new grant with it, rather than failing the grant's foreign key.""" import threading alice = make_user() fid = _own(db, alice.id, "Mine").id body = tokens_routes.TokenIn( name="x", grants=[tokens_routes.GrantIn(folder_id=fid, rights=[right])] ) tokens_routes._wanted_grants(db, alice, body) outcome: list[object] = [] def delete_it(): other = SessionLocal() try: outcome.append(folders.delete_own(other, other.get(FileFolder, fid))) other.commit() except Exception as e: # noqa: BLE001 — the assertion below names what went wrong outcome.append(e) finally: other.close() t = threading.Thread(target=delete_it) t.start() t.join(timeout=1) assert t.is_alive() # the delete waits for the token on its way in db.commit() t.join(timeout=10) assert not isinstance(outcome[0], Exception), outcome def test_a_user_has_at_most_the_ceiling_of_tokens(db, make_user, as_user, monkeypatch): monkeypatch.setattr(tokens_routes, "MAX_TOKENS", 2) c = as_user(make_user()) _make(c, preset="hub") _make(c, preset="hub") r = c.post("/api/tokens", json={"name": "third", "preset": "hub"}) assert r.status_code == 409 and r.json()["detail"]["reason"] == "api_token_limit" # --- what a token reaches ------------------------------------------------------------------------ @pytest.mark.parametrize("right", ["list", "read", "write", "delete"]) def test_each_right_holds_in_its_folder_only( db, make_user, as_user, _root, _no_probe, upload_headers, right ): """One right in folder A; folder B has none. Every Files route a token may use, in both.""" alice = make_user() a, b = _own(db, alice.id, "A"), _own(db, alice.id, "B") in_a, in_b = _file_in(db, _root, alice.id, a, "fa"), _file_in(db, _root, alice.id, b, "fb") raw = _make(as_user(alice), grants=[{"folder_id": a.id, "rights": [right]}])["token"] m = _bearer(raw) folders_seen = [f["id"] for f in m.get("/api/files/folders").json()] assert folders_seen == ([a.id] if right == "list" else []) listed = m.get("/api/files", params={"folder_id": a.id}) assert listed.status_code == (200 if right == "list" else 404) if right == "list": assert [f["id"] for f in listed.json()] == [in_a.id] assert [f["id"] for f in m.get("/api/files").json()] == [in_a.id] else: assert m.get("/api/files").json() == [] assert m.get("/api/files", params={"folder_id": b.id}).status_code == 404 read_a = m.get(f"/api/files/{in_a.id}/file") assert read_a.status_code == (200 if right == "read" else 404) assert m.get(f"/api/files/{in_a.id}/sha256").status_code == (200 if right == "read" else 404) assert m.get(f"/api/files/{in_b.id}/file").status_code == 404 up_a = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt", folder=a.id)) assert up_a.status_code == (200 if right == "write" else 404) if right == "write": assert up_a.json()["folder_id"] == a.id up_b = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt", folder=b.id)) assert up_b.status_code == 404 no_folder = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt")) assert no_folder.status_code == 400 assert no_folder.json()["detail"]["reason"] == "api_token_folder_required" assert m.delete(f"/api/files/{in_b.id}").status_code == 404 assert db.get(DownloadJob, in_b.id) is not None del_a = m.delete(f"/api/files/{in_a.id}") assert del_a.status_code == (200 if right == "delete" else 404) db.expire_all() assert (db.get(DownloadJob, in_a.id) is None) == (right == "delete") def test_a_token_never_reaches_another_users_files_or_the_shared_ones( db, make_user, as_user, _root ): alice, bob = make_user(), make_user() hub_raw = _make(as_user(alice), preset="hub")["token"] bobs = _file_in(db, _root, bob.id, _own(db, bob.id, "B"), "bf") m = _bearer(hub_raw) assert m.get(f"/api/files/{bobs.id}/file").status_code == 404 assert m.delete(f"/api/files/{bobs.id}").status_code == 404 shared = m.get("/api/files", params={"scope": "shared"}) assert shared.status_code == 403 and shared.json()["detail"]["reason"] == "api_token_scope" def test_a_token_is_judged_by_its_grants_even_beside_a_signed_in_cookie(db, make_user, client): """Through the real session stack (`client`, a real sign-in): the `as_user` fixture stands in for the very dependency this is about.""" alice = make_user(email="alice@example.com") a = _own(db, alice.id, "A") c = client signed_in = c.post( "/auth/password-login", json={"email": "alice@example.com", "password": "correct horse battery"}, ) assert signed_in.status_code == 200 raw = _make(c, grants=[{"folder_id": a.id, "rights": ["read"]}])["token"] assert len(c.get("/api/files/folders").json()) == 1 # the user sees every folder both = c.get("/api/files/folders", headers={"authorization": f"Bearer {raw}"}) assert both.json() == [] # the token may not list: its grants decide def _signed_in(client, email="alice@example.com") -> TestClient: r = client.post( "/auth/password-login", json={"email": email, "password": "correct horse battery"} ) assert r.status_code == 200, r.text return client def test_a_token_works_on_the_files_routes_only(db, make_user, client): """Through the real session stack: under `as_user` every route without its header answers 401 whatever `current_user` does with a token, so these refusals would prove nothing there.""" make_user(email="alice@example.com") raw = _make(_signed_in(client), preset="hub")["token"] m = _bearer(raw) assert m.get("/api/files/folders").status_code == 200 # Making a wider token, or reading the account, takes the user's session. assert m.get("/api/me").json() == {"authenticated": False} # the token is no one here assert m.get("/api/tokens").status_code == 401 assert m.post("/api/tokens", json={"name": "wider", "preset": "hub"}).status_code == 401 assert m.get("/api/files/usage").status_code == 401 assert m.patch("/api/files/1", json={"name": "x"}).status_code == 401 def test_only_the_files_routes_take_a_token(): """The route table, not a list kept by hand: every route that resolves `files_caller` is under /api/files, and these are all of them. The walk recurses into included routers (see `test_body_limit._declared_routes` for why `app.routes` alone sees none of ours).""" takers, nodes, routes = set(), [app], [] while nodes: for route in getattr(nodes.pop(), "routes", ()) or (): inner = getattr(route, "original_router", None) if inner is not None: nodes.append(inner) else: routes.append(route) assert len(routes) > 100 # the walk found the app's routes, not just the shell's for route in routes: dependant = getattr(route, "dependant", None) if dependant is None: continue stack, calls = [dependant], set() while stack: d = stack.pop() calls.add(d.call) stack.extend(d.dependencies) if files_caller in calls: for method in route.methods: takers.add((method, route.path)) assert all(path.startswith("/api/files") for _, path in takers) assert takers == { ("GET", "/api/files/folders"), ("GET", "/api/files"), ("GET", "/api/files/{job_id}/file"), ("GET", "/api/files/{job_id}/sha256"), ("POST", "/api/files/upload"), ("DELETE", "/api/files/{job_id}"), } def _reset_password(db, client, alice): from app.routes.auth import _issue_token link = _issue_token(db, alice, "reset", timedelta(hours=1)) r = client.post( "/auth/password-reset/confirm", json={"token": link, "password": "a brand new password"} ) assert r.status_code == 200, r.text def _change_password(db, client, alice): r = client.post( "/auth/set-password", json={"password": "a brand new password", "current_password": "correct horse battery"}, ) assert r.status_code == 200, r.text def _log_out_others(db, client, alice): assert client.post("/auth/logout-others").status_code == 200 def _revoke_untrusted_password(db, client, alice): from app.routes.auth import _revoke_password _revoke_password(db, alice) db.commit() @pytest.mark.parametrize( "end_sessions", [_reset_password, _change_password, _log_out_others, _revoke_untrusted_password], ) def test_whatever_ends_every_session_ends_every_token_too(db, make_user, client, end_sessions): """A token minted from a stolen session must not outlive the owner's answer to the theft.""" alice = make_user(email="alice@example.com") bob = make_user(email="bob@example.com") raw = _make(_signed_in(client), preset="hub")["token"] bobs = apitokens.mint() db.add(ApiToken(user_id=bob.id, name="b", prefix=bobs[:12], token_hash=hash_token(bobs))) db.commit() assert _bearer(raw).get("/api/files/folders").status_code == 200 end_sessions(db, client, alice) assert _bearer(raw).get("/api/files/folders").status_code == 401 assert db.execute(select(ApiToken.user_id)).scalars().all() == [bob.id] # only hers went [revoked] = _audits(db, "api_token_revoke") assert revoked.actor_id == alice.id and "every session" in (revoked.summary or "") def test_a_token_asked_for_as_every_session_ends_is_refused(db, make_user, client, override_dep): """The session was still good when its request came in, and ended before the token was made.""" alice = make_user(email="alice@example.com") c = _signed_in(client) def ended_meanwhile(request: Request, session=Depends(get_db)): user = current_user(request, session) other = SessionLocal() try: bump_session_epoch(other, other.get(User, alice.id)) other.commit() finally: other.close() return user override_dep(current_user, ended_meanwhile) r = c.post("/api/tokens", json={"name": "late", "preset": "hub"}) assert r.status_code == 401 assert db.execute(select(ApiToken.id)).scalars().all() == [] def _meanwhile(work): """Run `work(other_session)` in another thread and session, committed; (thread, outcome).""" import threading outcome: list[object] = [] def run(): other = SessionLocal() try: outcome.append(work(other)) other.commit() except Exception as e: # noqa: BLE001 — the assertion below names what went wrong outcome.append(e) finally: other.close() t = threading.Thread(target=run) t.start() return t, outcome def test_ending_every_session_never_deadlocks_with_a_tokens_expiry_notice(db, make_user): """The notice's sweep holds the token's row and then writes a row naming the user; the sessions' end holds the user and then deletes the token. A lock on the user that a mere reference to it waits on would make that a deadlock — one of the two failing.""" alice = make_user() raw = apitokens.mint() soon = datetime.now(timezone.utc) + timedelta(days=1) tok = ApiToken( user_id=alice.id, name="t", prefix=raw[:12], token_hash=hash_token(raw), expires_at=soon ) db.add(tok) db.commit() # `warn_expiring`, its two halves held apart: the token row taken… db.execute(update(ApiToken).where(ApiToken.id == tok.id).values(expiry_warned=True)) t, outcome = _meanwhile(lambda other: bump_session_epoch(other, other.get(User, alice.id))) t.join(timeout=1) assert t.is_alive() # the sessions' end waits for the token row # …and then its notice, naming the user. create_notification(db, alice.id, "api_token_expiring", title="t", commit=False) db.commit() t.join(timeout=10) assert outcome == [None] db.expire_all() assert db.execute(select(ApiToken.id)).scalars().all() == [] def test_a_token_granted_a_folder_being_deleted_never_deadlocks(db, make_user): """The delete holds the folder and then writes its audit line naming the user; the token holds the user and then waits for the folder.""" alice = make_user() fid = _own(db, alice.id, "Mine").id folders.delete_own(db, db.get(FileFolder, fid)) # the folder held, as the delete route does body = tokens_routes.TokenIn( name="x", grants=[tokens_routes.GrantIn(folder_id=fid, rights=["read"])] ) t, outcome = _meanwhile( lambda other: tokens_routes.create_token(body, other.get(User, alice.id), other) ) t.join(timeout=1) assert t.is_alive() # the token waits for the folder audit.record(db, alice.id, "file_folder_delete", target_type="file_folder", target_id=fid) db.commit() t.join(timeout=10) [refusal] = outcome assert getattr(refusal, "status_code", None) == 409, refusal # the folder went first def test_a_token_made_as_every_session_ends_goes_with_them(db, make_user): """The other order: the token is on its way in when the sessions end.""" import threading alice = make_user(email="alice@example.com") db.execute(select(User.id).where(User.id == alice.id).with_for_update()) # as create_token raw = apitokens.mint() db.add(ApiToken(user_id=alice.id, name="t", prefix=raw[:12], token_hash=hash_token(raw))) db.flush() outcome: list[object] = [] def end_sessions(): other = SessionLocal() try: bump_session_epoch(other, other.get(User, alice.id)) other.commit() outcome.append("done") except Exception as e: # noqa: BLE001 — the assertion below names what went wrong outcome.append(e) finally: other.close() t = threading.Thread(target=end_sessions) t.start() t.join(timeout=1) assert t.is_alive() # it waits for the token being made db.commit() t.join(timeout=10) assert outcome == ["done"] db.expire_all() assert db.execute(select(ApiToken.id)).scalars().all() == [] # --- a token that does not hold ------------------------------------------------------------------ def _refused(m: TestClient) -> int: return m.get("/api/files/folders").status_code def test_a_malformed_unknown_expired_revoked_or_suspended_token_holds_nothing( db, make_user, as_user ): alice = make_user() c = as_user(alice) raw = _make(c, preset="hub")["token"] assert _refused(_bearer(raw)) == 200 assert _refused(_bearer("sfl_short")) == 401 assert _refused(_bearer(apitokens.mint())) == 401 # well formed, never made expiring = _make(c, preset="hub", expires_days=1) token = db.get(ApiToken, expiring["id"]) token.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1) db.commit() assert _refused(_bearer(expiring["token"])) == 401 assert c.get("/api/tokens").json()[1]["expired"] is True assert c.delete(f"/api/tokens/{expiring['id']}").json() == { "ok": True, "deleted": expiring["id"], } assert ( db.execute(select(ApiTokenGrant).where(ApiTokenGrant.token_id == expiring["id"])).all() == [] ) assert len(_audits(db, "api_token_revoke")) == 1 revoked = _make(c, preset="hub") assert c.delete(f"/api/tokens/{revoked['id']}").status_code == 200 assert _refused(_bearer(revoked["token"])) == 401 alice.is_suspended = True db.commit() assert _refused(_bearer(raw)) == 401 def test_a_token_about_to_run_out_warns_its_user_once_and_is_marked_in_the_list( db, make_user, as_user ): alice, bob = make_user(), make_user() c = as_user(alice) soon = _make(c, preset="hub", expires_days=3) _make(c, preset="hub", expires_days=30) # outside the span _make(c, preset="hub") # never runs out gone = _make(c, preset="hub", expires_days=1) db.get(ApiToken, gone["id"]).expires_at = datetime.now(timezone.utc) - timedelta(seconds=1) _make(as_user(bob), preset="hub", expires_days=2) db.commit() assert [t["expiring"] for t in c.get("/api/tokens").json()] == [True, False, False, False] assert apitokens.warn_expiring(db) == 2 # alice's and bob's; the run-out one is not "soon" notes = db.execute(select(Notification).where(Notification.user_id == alice.id)).scalars() [note] = notes.all() assert note.type == "api_token_expiring" assert note.data["token_id"] == soon["id"] and note.data["name"] == "a machine" assert note.data["expires_at"] == soon["expires_at"] assert apitokens.warn_expiring(db) == 0 # once per token def test_the_files_gc_runs_the_token_warning(db, make_user, as_user, _root): _make(as_user(make_user()), preset="hub", expires_days=3) assert gcmod.run_download_gc(db)["tokens_warned"] == 1 def test_another_users_token_cannot_be_revoked(db, make_user, as_user): alice, bob = make_user(), make_user() made = _make(as_user(alice), preset="hub") assert as_user(bob).delete(f"/api/tokens/{made['id']}").status_code == 404 assert db.get(ApiToken, made["id"]) is not None def test_a_client_trying_tokens_is_slowed_down(db, make_user, as_user, monkeypatch): alice = make_user() raw = _make(as_user(alice), preset="hub")["token"] monkeypatch.setattr(apitokens, "_bad", RateLimiter(3, 300)) for _ in range(3): assert _refused(_bearer(apitokens.mint())) == 401 slowed = _bearer(apitokens.mint()).get("/api/files/folders") assert slowed.status_code == 429 assert slowed.json()["detail"]["reason"] == "api_token_rate_limited" # Even a good token from that client waits out the window: a guesser cannot tell a hit. assert _refused(_bearer(raw)) == 429 def test_a_runaway_machine_is_held_back_per_token(db, make_user, as_user, monkeypatch): c = as_user(make_user()) first, second = _make(c, preset="hub")["token"], _make(c, preset="hub")["token"] monkeypatch.setattr(apitokens, "_calls", RateLimiter(2, 60)) assert [_refused(_bearer(first)) for _ in range(3)] == [200, 200, 429] assert _refused(_bearer(second)) == 200 # --- what a token leaves behind ------------------------------------------------------------------ def test_a_tokens_use_is_stamped_and_its_uploads_and_deletes_name_it_in_the_audit( db, make_user, as_user, _root, _no_probe, upload_headers ): alice = make_user() made = _make(as_user(alice), preset="hub") hub = made["grants"][0]["folder_id"] assert db.get(ApiToken, made["id"]).last_used_at is None m = _bearer(made["token"]) up = m.post("/api/files/upload", content=b"hi", headers=upload_headers("h.txt", folder=hub)) assert up.status_code == 200 db.expire_all() assert db.get(ApiToken, made["id"]).last_used_at is not None assert m.delete(f"/api/files/{up.json()['id']}").status_code == 200 via = f"api token {made['prefix']}…" assert [a.summary for a in _audits(db, "file_upload")] == [via] assert [a.summary for a in _audits(db, "file_delete")] == [via]