Files
siftlode/backend/tests/test_api_tokens.py
T

649 lines
25 KiB
Python

"""API tokens for the Files library (`app.apitokens`, `routes.tokens`), DB-lane.
A token is made, listed and revoked by its signed-in user only; its value is in the answer that made
it and nowhere after. With it a machine reaches `/api/files` and nothing else, and there only the
folders its grants name, with the rights they give — every right checked in a folder it has and in
one it has not. A token that is malformed, unknown, expired, revoked, or whose account is suspended
holds nothing, and a client trying tokens is slowed down.
"""
from datetime import datetime, timedelta, timezone
import pytest
from fastapi import Depends, Request
from fastapi.testclient import TestClient
from sqlalchemy import select, update
from app import apitokens, audit
from app.apitokens import files_caller
from app.auth import current_user
from app.downloads import folders, service
from app.downloads import gc as gcmod
from app.downloads import upload as uploadmod
from app.downloads.status import JobStatus
from app.config import settings
from app.db import SessionLocal, get_db
from app.main import app
from app.notifications import create_notification
from app.models import (
ApiToken,
ApiTokenGrant,
AuditLog,
DownloadJob,
FileFolder,
MediaAsset,
Notification,
User,
)
from app.ratelimit import RateLimiter
from app.routes import tokens as tokens_routes
from app.routes.auth import bump_session_epoch
from app.security import hash_token
@pytest.fixture
def _root(tmp_path, monkeypatch):
monkeypatch.setattr(settings, "files_root", str(tmp_path))
return tmp_path
@pytest.fixture
def _no_probe(monkeypatch):
# A file ffprobe cannot read: a generic file, done at once (no conversion).
monkeypatch.setattr(uploadmod.probe, "probe_json", lambda _p: None)
def _own(db, user_id, name):
folder = FileFolder(user_id=user_id, kind=folders.FolderKind.OWN, name=name)
db.add(folder)
db.commit()
return folder
def _file_in(db, root, user_id, folder, name):
"""A done file of `user_id`'s with bytes on disk, sitting in `folder`."""
asset = MediaAsset(
source_kind="upload",
source_ref=name,
format_sig=f"sig-{name}",
status="ready",
rel_path=f"{name}.bin",
size_bytes=4,
ref_count=0,
title=name,
media_kind="file",
container="bin",
)
db.add(asset)
db.flush()
job = DownloadJob(
user_id=user_id,
asset_id=asset.id,
source_kind="upload",
source_ref=name,
profile_snapshot={},
status=JobStatus.DONE,
)
db.add(job)
db.flush()
folders.place(db, job)
job.folder_id = folder.id
folders.stamp_done(db, job)
service.incr_ref_count(db, asset.id)
db.commit()
(root / f"{name}.bin").write_bytes(b"data")
return job
def _bearer(raw: str) -> TestClient:
return TestClient(app, headers={"authorization": f"Bearer {raw}"})
def _make(c, **body) -> dict:
r = c.post("/api/tokens", json={"name": "a machine", **body})
assert r.status_code == 200, r.text
return r.json()
def _audits(db, action):
return db.execute(select(AuditLog).where(AuditLog.action == action)).scalars().all()
# --- the token itself ----------------------------------------------------------------------------
def test_a_token_has_its_shape_and_a_checksum_that_refuses_a_typo():
raw = apitokens.mint()
assert raw.startswith("sfl_") and len(raw) == apitokens.TOKEN_LEN
assert apitokens.well_formed(raw)
assert apitokens.mint() != raw
flipped = raw[:10] + ("A" if raw[10] != "A" else "B") + raw[11:]
assert not apitokens.well_formed(flipped)
assert not apitokens.well_formed(raw[:-1])
assert not apitokens.well_formed("xxx_" + raw[4:])
# A header byte past ASCII arrives as one latin-1 character: refused, never a 500.
assert not apitokens.well_formed(raw[:-1] + "é")
def test_a_non_ascii_token_is_refused_as_any_bad_one(db, client):
# The test client re-sends a high byte as two UTF-8 bytes, which arrive as two latin-1
# characters: one "A" fewer keeps the header a token's length, so it is the checksum that meets
# them, not the length check.
raw = ("Bearer sfl_" + "A" * 47 + "é").encode("latin-1")
r = TestClient(app, headers={"authorization": raw}).get("/api/files/folders")
assert r.status_code == 401 and r.json()["detail"]["reason"] == "not_authenticated"
def test_the_hub_preset_makes_one_hub_folder_with_every_right_and_shows_the_token_once(
db, make_user, as_user
):
alice = make_user()
c = as_user(alice)
made = _make(c, preset="hub")
raw = made["token"]
assert apitokens.well_formed(raw) and made["prefix"] == raw[: apitokens.SHOWN_LEN]
[grant] = made["grants"]
assert grant["kind"] == "hub" and grant["rights"] == ["delete", "list", "read", "write"]
# Only the hash is kept, and the value is not in the list.
stored = db.execute(select(ApiToken)).scalar_one()
assert stored.token_hash != raw and raw not in str(c.get("/api/tokens").json())
# A second hub token reuses the one Hub folder.
again = _make(c, preset="hub")
assert again["grants"][0]["folder_id"] == grant["folder_id"]
hubs = db.execute(select(FileFolder).where(FileFolder.kind == "hub")).scalars().all()
assert len(hubs) == 1
assert len(_audits(db, "api_token_create")) == 2
def test_a_token_is_granted_only_the_users_own_folders_and_writes_only_where_uploads_land(
db, make_user, as_user
):
alice, bob = make_user(), make_user()
theirs = _own(db, bob.id, "Bob's")
c = as_user(alice)
r = c.post(
"/api/tokens",
json={"name": "x", "grants": [{"folder_id": theirs.id, "rights": ["read"]}]},
)
assert r.status_code == 409 and r.json()["detail"]["reason"] == "file_folder_gone"
youtube = folders.source_folder(db, alice.id, folders.SourceKey.YOUTUBE)
db.commit()
r = c.post(
"/api/tokens",
json={"name": "x", "grants": [{"folder_id": youtube.id, "rights": ["write"]}]},
)
assert r.json()["detail"]["reason"] == "file_folder_not_its_source"
assert c.post("/api/tokens", json={"name": "x"}).json()["detail"]["reason"] == (
"api_token_no_grants"
)
assert db.execute(select(ApiToken)).all() == []
@pytest.mark.parametrize("right", ["read", "write"])
def test_a_folder_granted_is_held_until_the_token_is_in(db, make_user, right):
"""A delete of the folder at the same moment waits and takes the new grant with it, rather than
failing the grant's foreign key."""
import threading
alice = make_user()
fid = _own(db, alice.id, "Mine").id
body = tokens_routes.TokenIn(
name="x", grants=[tokens_routes.GrantIn(folder_id=fid, rights=[right])]
)
tokens_routes._wanted_grants(db, alice, body)
outcome: list[object] = []
def delete_it():
other = SessionLocal()
try:
outcome.append(folders.delete_own(other, other.get(FileFolder, fid)))
other.commit()
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
outcome.append(e)
finally:
other.close()
t = threading.Thread(target=delete_it)
t.start()
t.join(timeout=1)
assert t.is_alive() # the delete waits for the token on its way in
db.commit()
t.join(timeout=10)
assert not isinstance(outcome[0], Exception), outcome
def test_a_user_has_at_most_the_ceiling_of_tokens(db, make_user, as_user, monkeypatch):
monkeypatch.setattr(tokens_routes, "MAX_TOKENS", 2)
c = as_user(make_user())
_make(c, preset="hub")
_make(c, preset="hub")
r = c.post("/api/tokens", json={"name": "third", "preset": "hub"})
assert r.status_code == 409 and r.json()["detail"]["reason"] == "api_token_limit"
# --- what a token reaches ------------------------------------------------------------------------
@pytest.mark.parametrize("right", ["list", "read", "write", "delete"])
def test_each_right_holds_in_its_folder_only(
db, make_user, as_user, _root, _no_probe, upload_headers, right
):
"""One right in folder A; folder B has none. Every Files route a token may use, in both."""
alice = make_user()
a, b = _own(db, alice.id, "A"), _own(db, alice.id, "B")
in_a, in_b = _file_in(db, _root, alice.id, a, "fa"), _file_in(db, _root, alice.id, b, "fb")
raw = _make(as_user(alice), grants=[{"folder_id": a.id, "rights": [right]}])["token"]
m = _bearer(raw)
folders_seen = [f["id"] for f in m.get("/api/files/folders").json()]
assert folders_seen == ([a.id] if right == "list" else [])
listed = m.get("/api/files", params={"folder_id": a.id})
assert listed.status_code == (200 if right == "list" else 404)
if right == "list":
assert [f["id"] for f in listed.json()] == [in_a.id]
assert [f["id"] for f in m.get("/api/files").json()] == [in_a.id]
else:
assert m.get("/api/files").json() == []
assert m.get("/api/files", params={"folder_id": b.id}).status_code == 404
read_a = m.get(f"/api/files/{in_a.id}/file")
assert read_a.status_code == (200 if right == "read" else 404)
assert m.get(f"/api/files/{in_a.id}/sha256").status_code == (200 if right == "read" else 404)
assert m.get(f"/api/files/{in_b.id}/file").status_code == 404
up_a = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt", folder=a.id))
assert up_a.status_code == (200 if right == "write" else 404)
if right == "write":
assert up_a.json()["folder_id"] == a.id
up_b = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt", folder=b.id))
assert up_b.status_code == 404
no_folder = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt"))
assert no_folder.status_code == 400
assert no_folder.json()["detail"]["reason"] == "api_token_folder_required"
assert m.delete(f"/api/files/{in_b.id}").status_code == 404
assert db.get(DownloadJob, in_b.id) is not None
del_a = m.delete(f"/api/files/{in_a.id}")
assert del_a.status_code == (200 if right == "delete" else 404)
db.expire_all()
assert (db.get(DownloadJob, in_a.id) is None) == (right == "delete")
def test_a_token_never_reaches_another_users_files_or_the_shared_ones(
db, make_user, as_user, _root
):
alice, bob = make_user(), make_user()
hub_raw = _make(as_user(alice), preset="hub")["token"]
bobs = _file_in(db, _root, bob.id, _own(db, bob.id, "B"), "bf")
m = _bearer(hub_raw)
assert m.get(f"/api/files/{bobs.id}/file").status_code == 404
assert m.delete(f"/api/files/{bobs.id}").status_code == 404
shared = m.get("/api/files", params={"scope": "shared"})
assert shared.status_code == 403 and shared.json()["detail"]["reason"] == "api_token_scope"
def test_a_token_is_judged_by_its_grants_even_beside_a_signed_in_cookie(db, make_user, client):
"""Through the real session stack (`client`, a real sign-in): the `as_user` fixture stands in
for the very dependency this is about."""
alice = make_user(email="alice@example.com")
a = _own(db, alice.id, "A")
c = client
signed_in = c.post(
"/auth/password-login",
json={"email": "alice@example.com", "password": "correct horse battery"},
)
assert signed_in.status_code == 200
raw = _make(c, grants=[{"folder_id": a.id, "rights": ["read"]}])["token"]
assert len(c.get("/api/files/folders").json()) == 1 # the user sees every folder
both = c.get("/api/files/folders", headers={"authorization": f"Bearer {raw}"})
assert both.json() == [] # the token may not list: its grants decide
def _signed_in(client, email="alice@example.com") -> TestClient:
r = client.post(
"/auth/password-login", json={"email": email, "password": "correct horse battery"}
)
assert r.status_code == 200, r.text
return client
def test_a_token_works_on_the_files_routes_only(db, make_user, client):
"""Through the real session stack: under `as_user` every route without its header answers 401
whatever `current_user` does with a token, so these refusals would prove nothing there."""
make_user(email="alice@example.com")
raw = _make(_signed_in(client), preset="hub")["token"]
m = _bearer(raw)
assert m.get("/api/files/folders").status_code == 200
# Making a wider token, or reading the account, takes the user's session.
assert m.get("/api/me").json() == {"authenticated": False} # the token is no one here
assert m.get("/api/tokens").status_code == 401
assert m.post("/api/tokens", json={"name": "wider", "preset": "hub"}).status_code == 401
assert m.get("/api/files/usage").status_code == 401
assert m.patch("/api/files/1", json={"name": "x"}).status_code == 401
def test_only_the_files_routes_take_a_token():
"""The route table, not a list kept by hand: every route that resolves `files_caller` is under
/api/files, and these are all of them. The walk recurses into included routers (see
`test_body_limit._declared_routes` for why `app.routes` alone sees none of ours)."""
takers, nodes, routes = set(), [app], []
while nodes:
for route in getattr(nodes.pop(), "routes", ()) or ():
inner = getattr(route, "original_router", None)
if inner is not None:
nodes.append(inner)
else:
routes.append(route)
assert len(routes) > 100 # the walk found the app's routes, not just the shell's
for route in routes:
dependant = getattr(route, "dependant", None)
if dependant is None:
continue
stack, calls = [dependant], set()
while stack:
d = stack.pop()
calls.add(d.call)
stack.extend(d.dependencies)
if files_caller in calls:
for method in route.methods:
takers.add((method, route.path))
assert all(path.startswith("/api/files") for _, path in takers)
assert takers == {
("GET", "/api/files/folders"),
("GET", "/api/files"),
("GET", "/api/files/{job_id}/file"),
("GET", "/api/files/{job_id}/sha256"),
("POST", "/api/files/upload"),
("DELETE", "/api/files/{job_id}"),
}
def _reset_password(db, client, alice):
from app.routes.auth import _issue_token
link = _issue_token(db, alice, "reset", timedelta(hours=1))
r = client.post(
"/auth/password-reset/confirm", json={"token": link, "password": "a brand new password"}
)
assert r.status_code == 200, r.text
def _change_password(db, client, alice):
r = client.post(
"/auth/set-password",
json={"password": "a brand new password", "current_password": "correct horse battery"},
)
assert r.status_code == 200, r.text
def _log_out_others(db, client, alice):
assert client.post("/auth/logout-others").status_code == 200
def _revoke_untrusted_password(db, client, alice):
from app.routes.auth import _revoke_password
_revoke_password(db, alice)
db.commit()
@pytest.mark.parametrize(
"end_sessions",
[_reset_password, _change_password, _log_out_others, _revoke_untrusted_password],
)
def test_whatever_ends_every_session_ends_every_token_too(db, make_user, client, end_sessions):
"""A token minted from a stolen session must not outlive the owner's answer to the theft."""
alice = make_user(email="alice@example.com")
bob = make_user(email="bob@example.com")
raw = _make(_signed_in(client), preset="hub")["token"]
bobs = apitokens.mint()
db.add(ApiToken(user_id=bob.id, name="b", prefix=bobs[:12], token_hash=hash_token(bobs)))
db.commit()
assert _bearer(raw).get("/api/files/folders").status_code == 200
end_sessions(db, client, alice)
assert _bearer(raw).get("/api/files/folders").status_code == 401
assert db.execute(select(ApiToken.user_id)).scalars().all() == [bob.id] # only hers went
[revoked] = _audits(db, "api_token_revoke")
assert revoked.actor_id == alice.id and "every session" in (revoked.summary or "")
def test_a_token_asked_for_as_every_session_ends_is_refused(db, make_user, client, override_dep):
"""The session was still good when its request came in, and ended before the token was made."""
alice = make_user(email="alice@example.com")
c = _signed_in(client)
def ended_meanwhile(request: Request, session=Depends(get_db)):
user = current_user(request, session)
other = SessionLocal()
try:
bump_session_epoch(other, other.get(User, alice.id))
other.commit()
finally:
other.close()
return user
override_dep(current_user, ended_meanwhile)
r = c.post("/api/tokens", json={"name": "late", "preset": "hub"})
assert r.status_code == 401
assert db.execute(select(ApiToken.id)).scalars().all() == []
def _meanwhile(work):
"""Run `work(other_session)` in another thread and session, committed; (thread, outcome)."""
import threading
outcome: list[object] = []
def run():
other = SessionLocal()
try:
outcome.append(work(other))
other.commit()
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
outcome.append(e)
finally:
other.close()
t = threading.Thread(target=run)
t.start()
return t, outcome
def test_ending_every_session_never_deadlocks_with_a_tokens_expiry_notice(db, make_user):
"""The notice's sweep holds the token's row and then writes a row naming the user; the sessions'
end holds the user and then deletes the token. A lock on the user that a mere reference to it
waits on would make that a deadlock — one of the two failing."""
alice = make_user()
raw = apitokens.mint()
soon = datetime.now(timezone.utc) + timedelta(days=1)
tok = ApiToken(
user_id=alice.id, name="t", prefix=raw[:12], token_hash=hash_token(raw), expires_at=soon
)
db.add(tok)
db.commit()
# `warn_expiring`, its two halves held apart: the token row taken…
db.execute(update(ApiToken).where(ApiToken.id == tok.id).values(expiry_warned=True))
t, outcome = _meanwhile(lambda other: bump_session_epoch(other, other.get(User, alice.id)))
t.join(timeout=1)
assert t.is_alive() # the sessions' end waits for the token row
# …and then its notice, naming the user.
create_notification(db, alice.id, "api_token_expiring", title="t", commit=False)
db.commit()
t.join(timeout=10)
assert outcome == [None]
db.expire_all()
assert db.execute(select(ApiToken.id)).scalars().all() == []
def test_a_token_granted_a_folder_being_deleted_never_deadlocks(db, make_user):
"""The delete holds the folder and then writes its audit line naming the user; the token holds
the user and then waits for the folder."""
alice = make_user()
fid = _own(db, alice.id, "Mine").id
folders.delete_own(db, db.get(FileFolder, fid)) # the folder held, as the delete route does
body = tokens_routes.TokenIn(
name="x", grants=[tokens_routes.GrantIn(folder_id=fid, rights=["read"])]
)
t, outcome = _meanwhile(
lambda other: tokens_routes.create_token(body, other.get(User, alice.id), other)
)
t.join(timeout=1)
assert t.is_alive() # the token waits for the folder
audit.record(db, alice.id, "file_folder_delete", target_type="file_folder", target_id=fid)
db.commit()
t.join(timeout=10)
[refusal] = outcome
assert getattr(refusal, "status_code", None) == 409, refusal # the folder went first
def test_a_token_made_as_every_session_ends_goes_with_them(db, make_user):
"""The other order: the token is on its way in when the sessions end."""
import threading
alice = make_user(email="alice@example.com")
db.execute(select(User.id).where(User.id == alice.id).with_for_update()) # as create_token
raw = apitokens.mint()
db.add(ApiToken(user_id=alice.id, name="t", prefix=raw[:12], token_hash=hash_token(raw)))
db.flush()
outcome: list[object] = []
def end_sessions():
other = SessionLocal()
try:
bump_session_epoch(other, other.get(User, alice.id))
other.commit()
outcome.append("done")
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
outcome.append(e)
finally:
other.close()
t = threading.Thread(target=end_sessions)
t.start()
t.join(timeout=1)
assert t.is_alive() # it waits for the token being made
db.commit()
t.join(timeout=10)
assert outcome == ["done"]
db.expire_all()
assert db.execute(select(ApiToken.id)).scalars().all() == []
# --- a token that does not hold ------------------------------------------------------------------
def _refused(m: TestClient) -> int:
return m.get("/api/files/folders").status_code
def test_a_malformed_unknown_expired_revoked_or_suspended_token_holds_nothing(
db, make_user, as_user
):
alice = make_user()
c = as_user(alice)
raw = _make(c, preset="hub")["token"]
assert _refused(_bearer(raw)) == 200
assert _refused(_bearer("sfl_short")) == 401
assert _refused(_bearer(apitokens.mint())) == 401 # well formed, never made
expiring = _make(c, preset="hub", expires_days=1)
token = db.get(ApiToken, expiring["id"])
token.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
db.commit()
assert _refused(_bearer(expiring["token"])) == 401
assert c.get("/api/tokens").json()[1]["expired"] is True
assert c.delete(f"/api/tokens/{expiring['id']}").json() == {
"ok": True,
"deleted": expiring["id"],
}
assert (
db.execute(select(ApiTokenGrant).where(ApiTokenGrant.token_id == expiring["id"])).all()
== []
)
assert len(_audits(db, "api_token_revoke")) == 1
revoked = _make(c, preset="hub")
assert c.delete(f"/api/tokens/{revoked['id']}").status_code == 200
assert _refused(_bearer(revoked["token"])) == 401
alice.is_suspended = True
db.commit()
assert _refused(_bearer(raw)) == 401
def test_a_token_about_to_run_out_warns_its_user_once_and_is_marked_in_the_list(
db, make_user, as_user
):
alice, bob = make_user(), make_user()
c = as_user(alice)
soon = _make(c, preset="hub", expires_days=3)
_make(c, preset="hub", expires_days=30) # outside the span
_make(c, preset="hub") # never runs out
gone = _make(c, preset="hub", expires_days=1)
db.get(ApiToken, gone["id"]).expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
_make(as_user(bob), preset="hub", expires_days=2)
db.commit()
assert [t["expiring"] for t in c.get("/api/tokens").json()] == [True, False, False, False]
assert apitokens.warn_expiring(db) == 2 # alice's and bob's; the run-out one is not "soon"
notes = db.execute(select(Notification).where(Notification.user_id == alice.id)).scalars()
[note] = notes.all()
assert note.type == "api_token_expiring"
assert note.data["token_id"] == soon["id"] and note.data["name"] == "a machine"
assert note.data["expires_at"] == soon["expires_at"]
assert apitokens.warn_expiring(db) == 0 # once per token
def test_the_files_gc_runs_the_token_warning(db, make_user, as_user, _root):
_make(as_user(make_user()), preset="hub", expires_days=3)
assert gcmod.run_download_gc(db)["tokens_warned"] == 1
def test_another_users_token_cannot_be_revoked(db, make_user, as_user):
alice, bob = make_user(), make_user()
made = _make(as_user(alice), preset="hub")
assert as_user(bob).delete(f"/api/tokens/{made['id']}").status_code == 404
assert db.get(ApiToken, made["id"]) is not None
def test_a_client_trying_tokens_is_slowed_down(db, make_user, as_user, monkeypatch):
alice = make_user()
raw = _make(as_user(alice), preset="hub")["token"]
monkeypatch.setattr(apitokens, "_bad", RateLimiter(3, 300))
for _ in range(3):
assert _refused(_bearer(apitokens.mint())) == 401
slowed = _bearer(apitokens.mint()).get("/api/files/folders")
assert slowed.status_code == 429
assert slowed.json()["detail"]["reason"] == "api_token_rate_limited"
# Even a good token from that client waits out the window: a guesser cannot tell a hit.
assert _refused(_bearer(raw)) == 429
def test_a_runaway_machine_is_held_back_per_token(db, make_user, as_user, monkeypatch):
c = as_user(make_user())
first, second = _make(c, preset="hub")["token"], _make(c, preset="hub")["token"]
monkeypatch.setattr(apitokens, "_calls", RateLimiter(2, 60))
assert [_refused(_bearer(first)) for _ in range(3)] == [200, 200, 429]
assert _refused(_bearer(second)) == 200
# --- what a token leaves behind ------------------------------------------------------------------
def test_a_tokens_use_is_stamped_and_its_uploads_and_deletes_name_it_in_the_audit(
db, make_user, as_user, _root, _no_probe, upload_headers
):
alice = make_user()
made = _make(as_user(alice), preset="hub")
hub = made["grants"][0]["folder_id"]
assert db.get(ApiToken, made["id"]).last_used_at is None
m = _bearer(made["token"])
up = m.post("/api/files/upload", content=b"hi", headers=upload_headers("h.txt", folder=hub))
assert up.status_code == 200
db.expire_all()
assert db.get(ApiToken, made["id"]).last_used_at is not None
assert m.delete(f"/api/files/{up.json()['id']}").status_code == 200
via = f"api token {made['prefix']}…"
assert [a.summary for a in _audits(db, "file_upload")] == [via]
assert [a.summary for a in _audits(db, "file_delete")] == [via]