refactor(capture): the HTTP layer reads through queries and writes through the service
Capture's routes (downloads, files, public watch links) and the token list no longer build SQL, call the session or set a model's fields: reads are downloads/queries, writes the unit's service (service, folders, links, quota), and each write flushes for the handler to commit. Whose a row is goes through userscope (owned_by, owned_row), which owned_or_404 now uses too. The fence holds Capture below its routes as well: it reads only Catalog's and Plex's tables, writes no other unit's, filters owners through userscope and commits only in its jobs and the upload ingest. A test per service use case, among them that the tray lists only the caller's files.
This commit is contained in:
@@ -0,0 +1,295 @@
|
||||
"""Capture's service and queries, one use case each, below the HTTP layer.
|
||||
|
||||
The routes are thin callers now, and their own tests stay the contract a client sees. These hold
|
||||
what each write does and leaves on its own: that it flushes and leaves the commit to its caller,
|
||||
that a row of someone else's reads as a missing one, and the rule each write carries (a hash kept
|
||||
once, a password change that retires the old grants, a quota floored). DB-backed (real Postgres).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.audit import AuditAction
|
||||
from app.downloads import folders, links, queries, quota, service
|
||||
from app.downloads.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
DownloadQuota,
|
||||
DownloadShare,
|
||||
FileFolder,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
)
|
||||
from app.models import AuditLog, Notification
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
|
||||
def _file(db, user, *, sha256=None) -> DownloadJob:
|
||||
asset = MediaAsset(
|
||||
source_kind="url",
|
||||
source_ref=f"https://example.com/{user.id}.mp4",
|
||||
format_sig=f"sig-{user.id}",
|
||||
status="ready",
|
||||
rel_path=f"f-{user.id}.mp4",
|
||||
ref_count=1,
|
||||
media_kind="video",
|
||||
container="mp4",
|
||||
sha256=sha256,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
job = DownloadJob(
|
||||
user_id=user.id,
|
||||
asset_id=asset.id,
|
||||
source_kind="url",
|
||||
source_ref=asset.source_ref,
|
||||
status="done",
|
||||
profile_snapshot={},
|
||||
display_name="Clip",
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
return job
|
||||
|
||||
|
||||
def _uncommitted(db):
|
||||
"""What the caller still has to commit: the session holds pending work."""
|
||||
return bool(db.new or db.dirty or db.deleted) or db.in_transaction()
|
||||
|
||||
|
||||
# --- whose a row is ------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_row_of_someone_elses_reads_as_a_missing_one(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert owned_row(db, DownloadJob, job.id, owner) is job
|
||||
assert owned_row(db, DownloadJob, job.id, stranger) is None
|
||||
assert owned_row(db, DownloadJob, job.id + 1000, owner) is None
|
||||
mine = db.execute(select(DownloadJob.id).where(owned_by(DownloadJob.user_id, owner.id)))
|
||||
assert mine.scalars().all() == [job.id]
|
||||
|
||||
|
||||
def test_a_file_is_readable_by_its_owner_and_whoever_it_was_shared_with(db, make_user):
|
||||
owner, friend, stranger = make_user(), make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.share(db, job, owner, friend) is True
|
||||
db.commit()
|
||||
assert queries.accessible_job(db, owner, job.id) is job
|
||||
assert queries.accessible_job(db, friend, job.id) is job
|
||||
assert queries.accessible_job(db, stranger, job.id) is None
|
||||
assert queries.own_job(db, friend, job.id) is None # shared is not owned
|
||||
|
||||
|
||||
def test_the_tray_and_the_library_list_only_the_callers_own_files(db, make_user):
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
mine, theirs = make_user(), make_user()
|
||||
own, other = _file(db, mine), _file(db, theirs)
|
||||
since = datetime.now(timezone.utc) - timedelta(days=1)
|
||||
for job in (own, other):
|
||||
job.ready_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
assert [j.id for j in queries.queue(db, mine, since)] == [own.id]
|
||||
library = queries.library(
|
||||
db,
|
||||
mine,
|
||||
folder_id=None,
|
||||
listable=None,
|
||||
shared=False,
|
||||
term="",
|
||||
sort="added",
|
||||
order="desc",
|
||||
lang="en",
|
||||
)
|
||||
assert [j.id for j in library] == [own.id]
|
||||
assert [r for r, _ in queries.statuses_by_source(db, mine, ["done"])] == [own.source_ref]
|
||||
|
||||
|
||||
# --- locks -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_job_lock_answers_none_for_a_job_not_the_callers(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.lock_own_job(db, stranger, job.id) is None
|
||||
assert service.lock_own_job(db, owner, job.id) is job
|
||||
assert service.hold_for_share(db, stranger, job.id) is None
|
||||
assert service.hold_for_share(db, owner, job.id) is job
|
||||
|
||||
|
||||
# --- profiles --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_profile_is_made_after_the_users_others_and_only_theirs_to_change(db, make_user):
|
||||
user, other = make_user(), make_user()
|
||||
first = service.create_profile(db, user, "x" * 100, {"kind": "video"})
|
||||
second = service.create_profile(db, user, "Audio", {"kind": "audio"})
|
||||
assert _uncommitted(db)
|
||||
db.commit()
|
||||
assert first.name == "x" * 80 # names are capped
|
||||
assert second.sort_order == first.sort_order + 1
|
||||
assert queries.own_profile(db, other, first.id) is None
|
||||
service.update_profile(first, name="Video", spec=None)
|
||||
db.commit()
|
||||
assert (first.name, first.spec) == ("Video", {"kind": "video"})
|
||||
service.delete_profile(db, first)
|
||||
db.commit()
|
||||
assert [p.id for p in queries.profiles_for(db, user)] == [second.id]
|
||||
|
||||
|
||||
# --- resume points ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_resume_point_is_one_row_per_user_and_file_and_a_zero_clears_it(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
service.set_position(db, user, job.id, 40)
|
||||
service.set_position(db, user, job.id, 90)
|
||||
db.commit()
|
||||
rows = db.execute(select(LibraryState.position_seconds)).scalars().all()
|
||||
assert rows == [90]
|
||||
assert queries.positions_for(db, user, [job]) == {job.id: 90}
|
||||
service.set_position(db, user, job.id, 0)
|
||||
service.set_position(db, user, job.id, 0) # the beacon's second clear deletes nothing
|
||||
db.commit()
|
||||
assert db.execute(select(LibraryState)).first() is None
|
||||
|
||||
|
||||
# --- shares ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_share_is_made_once_with_one_notice_and_undone_by_either_side(db, make_user):
|
||||
owner, friend = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.share(db, job, owner, friend) is True
|
||||
assert service.share(db, job, owner, friend) is False
|
||||
assert _uncommitted(db)
|
||||
db.commit()
|
||||
notices = db.execute(select(Notification).where(Notification.user_id == friend.id)).scalars()
|
||||
assert [n.data["job_id"] for n in notices] == [job.id]
|
||||
assert service.unshare(db, job.id, friend) is True
|
||||
assert service.unshare(db, job.id, friend) is False
|
||||
db.commit()
|
||||
assert db.execute(select(DownloadShare)).first() is None
|
||||
|
||||
|
||||
# --- a file's display fields, its access clock, its hash ------------------------------------
|
||||
|
||||
|
||||
def test_only_display_fields_can_be_set_on_a_file(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
service.set_display(job, display_name="New", extra_links=None)
|
||||
assert job.display_name == "New"
|
||||
with pytest.raises(ValueError):
|
||||
service.set_display(job, status="error")
|
||||
assert job.status == "done"
|
||||
|
||||
|
||||
def test_a_hash_is_kept_once_and_never_overwritten(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
service.store_sha256(db, job.asset_id, "a" * 64)
|
||||
service.store_sha256(db, job.asset_id, "b" * 64)
|
||||
db.commit()
|
||||
assert db.get(MediaAsset, job.asset_id).sha256 == "a" * 64
|
||||
|
||||
|
||||
def test_serving_a_file_restarts_its_idle_clock(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
asset = queries.asset_of(db, job)
|
||||
before = asset.last_access_at
|
||||
service.touch(asset)
|
||||
assert asset.last_access_at is not None and asset.last_access_at != before
|
||||
|
||||
|
||||
# --- links -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _actions(db) -> list[str]:
|
||||
return db.execute(select(AuditLog.action).order_by(AuditLog.id)).scalars().all()
|
||||
|
||||
|
||||
def test_a_link_is_made_changed_and_revoked_with_an_audit_row_each(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
link, token = links.create(
|
||||
db, user, password="pw", allow_download=False, expires_days=3, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
assert links.find(db, token) is link
|
||||
assert link.expires_at is not None and link.password_hash is not None
|
||||
version = link.password_version
|
||||
links.update(db, user, link, password="other", sent=frozenset({"password"}))
|
||||
db.commit()
|
||||
assert link.password_version == version + 1 # the old grants stop working
|
||||
links.update(db, user, link, allow_download=True, sent=frozenset()) # nothing was sent
|
||||
assert link.allow_download is False
|
||||
assert [link.id for link in queries.links_to(db, job_id=job.id)] == [link.id]
|
||||
links.revoke(db, user, link)
|
||||
db.commit()
|
||||
assert db.execute(select(DownloadLink)).first() is None
|
||||
assert _actions(db) == [
|
||||
AuditAction.DOWNLOAD_LINK_CREATE,
|
||||
AuditAction.DOWNLOAD_LINK_UPDATE,
|
||||
AuditAction.DOWNLOAD_LINK_REVOKE,
|
||||
]
|
||||
|
||||
|
||||
def test_a_link_is_the_callers_only_through_a_file_or_folder_of_theirs(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
link, _ = links.create(
|
||||
db, owner, password="", allow_download=True, expires_days=None, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
assert queries.own_link(db, owner, link.id) is link
|
||||
assert queries.own_link(db, stranger, link.id) is None
|
||||
|
||||
|
||||
def test_a_view_is_counted_in_sql(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
link, _ = links.create(
|
||||
db, user, password="", allow_download=True, expires_days=None, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
links.count_view(db, link)
|
||||
links.count_view(db, link)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
assert db.get(DownloadLink, link.id).view_count == 2
|
||||
|
||||
|
||||
# --- folders ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_folder_name_is_one_per_user_and_a_source_folder_keeps_its_own(db, make_user):
|
||||
user, other = make_user(), make_user()
|
||||
mine = folders.create_own(db, user.id, "Trips")
|
||||
with pytest.raises(folders.FolderError, match="file_folder_name_taken"):
|
||||
folders.create_own(db, user.id, "trips")
|
||||
folders.create_own(db, other.id, "Trips") # another user's namesake is theirs
|
||||
db.commit()
|
||||
source = folders.source_folder(db, user.id, folders.SourceKey.UPLOAD)
|
||||
with pytest.raises(folders.FolderError, match="file_folder_not_own"):
|
||||
folders.rename(db, source, "Mine now")
|
||||
folders.rename(db, mine, "Holidays")
|
||||
db.commit()
|
||||
assert db.get(FileFolder, mine.id).name == "Holidays"
|
||||
assert queries.own_folder(db, other, mine.id) is None
|
||||
|
||||
|
||||
# --- quota -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_custom_quota_starts_from_what_applies_and_floors_each_value(db, make_user):
|
||||
user = make_user()
|
||||
defaults = quota.resolve(db, user.id)
|
||||
quota.set_custom(db, user.id, max_concurrent=0, max_bytes=-5)
|
||||
db.commit()
|
||||
row = queries.custom_quota(db, user.id)
|
||||
assert (row.max_concurrent, row.max_bytes, row.max_jobs) == (1, 0, defaults.max_jobs)
|
||||
assert quota.reset_custom(db, user.id) is True
|
||||
assert quota.reset_custom(db, user.id) is False
|
||||
db.commit()
|
||||
assert db.get(DownloadQuota, user.id) is None
|
||||
@@ -89,6 +89,7 @@ def test_overlapping_requests_for_one_job_release_its_hold_once(
|
||||
# The second request waits for the first one's locks instead of deciding on the same row.
|
||||
assert not finished.wait(0.5), "the second request did not wait for the first"
|
||||
transition(first, locked)
|
||||
first.commit() # the handler's commit: the service only flushes
|
||||
other.join(10)
|
||||
assert finished.is_set()
|
||||
finally:
|
||||
|
||||
@@ -180,6 +180,7 @@ def test_a_suspended_users_queued_jobs_wait(db):
|
||||
|
||||
|
||||
def test_list_downloads_is_capped(db, monkeypatch):
|
||||
from app.downloads import queries
|
||||
from app.routes import downloads as dl
|
||||
|
||||
u = User(email="heavy@example.com")
|
||||
@@ -189,7 +190,7 @@ def test_list_downloads_is_capped(db, monkeypatch):
|
||||
_job(db, u.id, queue_pos=i, status="done")
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(dl, "MAX_LIST", 2)
|
||||
monkeypatch.setattr(queries, "MAX_LIST", 2)
|
||||
rows = dl.list_downloads(user=u, db=db)
|
||||
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
|
||||
|
||||
|
||||
@@ -194,7 +194,7 @@ def test_every_open_counts_as_a_view(shelf, client):
|
||||
|
||||
def test_the_folder_list_is_capped_like_the_owners_files_list():
|
||||
from app.downloads import links as linksmod
|
||||
from app.routes.files import MAX_LIST
|
||||
from app.downloads.queries import MAX_LIST
|
||||
|
||||
assert linksmod.FOLDER_MAX == MAX_LIST
|
||||
|
||||
|
||||
@@ -11,7 +11,11 @@ until its entry is lowered (the fence only tightens, as `baseline.json` does for
|
||||
empty `ALLOWED` is the end state — a file listed there is a debt, not an exemption.
|
||||
|
||||
A core route (identity, admin, ops) is not a unit's HTTP layer and may use the session; it may
|
||||
still not reach into a unit's tables.
|
||||
still not reach into a unit's tables. A route may NAME a unit's model as a type (under
|
||||
`TYPE_CHECKING`), never use one as a value.
|
||||
|
||||
Below the HTTP layer, a fenced unit's own code keeps to its edges (`FENCED`, `READS`): see the
|
||||
second half of this file.
|
||||
"""
|
||||
|
||||
import ast
|
||||
@@ -108,9 +112,7 @@ UNIT_ROUTES = {
|
||||
ALLOWED = {
|
||||
"channels.py": 83,
|
||||
"discover.py": 7,
|
||||
"downloads.py": 88,
|
||||
"feed.py": 36,
|
||||
"files.py": 58,
|
||||
"playlists.py": 72,
|
||||
"plex/_shared.py": 1,
|
||||
"plex/admin.py": 12,
|
||||
@@ -121,14 +123,12 @@ ALLOWED = {
|
||||
"plex/people.py": 20,
|
||||
"plex/playback.py": 42,
|
||||
"plex/playlists.py": 80,
|
||||
"public.py": 12,
|
||||
"runs.py": 1,
|
||||
"saved_views.py": 17,
|
||||
"scheduler.py": 3,
|
||||
"search.py": 46,
|
||||
"sync.py": 38,
|
||||
"tags.py": 10,
|
||||
"tokens.py": 1,
|
||||
}
|
||||
|
||||
# The session methods that read or write data. `commit` and `rollback` stay with the handler.
|
||||
@@ -185,6 +185,20 @@ def _session_names(tree: ast.AST) -> set[str]:
|
||||
return names
|
||||
|
||||
|
||||
def _type_only_imports(tree: ast.AST) -> set[ast.AST]:
|
||||
"""The imports under `if TYPE_CHECKING:` — they name a type for an annotation and are never
|
||||
loaded at run time, so a model imported there cannot be queried or written."""
|
||||
out: set[ast.AST] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.If):
|
||||
test = node.test
|
||||
name = test.id if isinstance(test, ast.Name) else getattr(test, "attr", None)
|
||||
if name == "TYPE_CHECKING":
|
||||
for child in node.body:
|
||||
out |= set(ast.walk(child))
|
||||
return out
|
||||
|
||||
|
||||
def _root(node: ast.expr) -> ast.expr:
|
||||
while isinstance(node, (ast.Attribute, ast.Subscript)):
|
||||
node = node.value
|
||||
@@ -197,8 +211,9 @@ def violations(source: str, unit: str | None) -> list[str]:
|
||||
tree = ast.parse(source)
|
||||
found = []
|
||||
sql_names = set()
|
||||
typing_only = _type_only_imports(tree)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
if isinstance(node, ast.ImportFrom) and node not in typing_only:
|
||||
module = node.module or ""
|
||||
for alias in node.names:
|
||||
bound = alias.asname or alias.name
|
||||
@@ -296,6 +311,9 @@ DETECTOR_CASES = {
|
||||
"from app.capture.models import DownloadJob as J": 1,
|
||||
"def f(cache): cache.get('k')": 0,
|
||||
"from sqlalchemy.exc import IntegrityError\ntry:\n pass\nexcept IntegrityError:\n pass": 0,
|
||||
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.downloads.models import Job": 0,
|
||||
"import typing\nif typing.TYPE_CHECKING:\n from app.models import DownloadJob": 0,
|
||||
"if TYPE_CHECKING:\n pass\nfrom app.downloads.models import DownloadJob": 1,
|
||||
}
|
||||
|
||||
|
||||
@@ -305,3 +323,185 @@ def test_the_detector_sees_each_kind_of_database_work():
|
||||
assert violations("from app.models import Video\nfrom sqlalchemy import select", None) == [
|
||||
"1: imports catalog's Video"
|
||||
]
|
||||
|
||||
|
||||
# --- inside a unit: its package --------------------------------------------------------------
|
||||
# A unit whose sprint has landed is fenced beyond its HTTP layer too. Its own code reads another
|
||||
# unit's tables only along a declared edge (the feed joins Video with VideoState in one query; a
|
||||
# boundary that split it would cost the speed the app is for), and writes another unit's tables
|
||||
# never — that goes through the owner's service. Whose a row is, it says through `userscope`, and
|
||||
# it commits only where a job owns its session.
|
||||
|
||||
# Each unit's code outside the HTTP layer: its package, and the entry point that runs its jobs.
|
||||
UNIT_PACKAGES = {"capture": ("downloads/", "worker.py")}
|
||||
# The units fenced so far; the rest join with their sprint.
|
||||
FENCED = {"capture"}
|
||||
# Whose tables a unit's own code may READ. Core code reads a fenced unit's tables through its queries.
|
||||
READS = {"capture": {"catalog", "plex"}}
|
||||
# The columns that say whose a row is.
|
||||
OWNER_COLUMNS = {"user_id", "to_user_id", "from_user_id", "created_by"}
|
||||
# Where a fenced unit's code may commit: the jobs that own their session (the worker and what it
|
||||
# runs, the GC, the backfill — none of them inside a request), and the one write that must commit
|
||||
# where it stands: an upload's rows and the file it placed on disk land together or not at all, so
|
||||
# a failed commit can remove the file (`upload.ingest_upload`). Everything else flushes; the handler
|
||||
# commits.
|
||||
UNIT_COMMITS = {
|
||||
"capture": {
|
||||
"worker.py",
|
||||
"downloads/backfill.py",
|
||||
"downloads/gc.py",
|
||||
"downloads/jobdb.py",
|
||||
"downloads/pipeline.py",
|
||||
"downloads/upload.py:ingest_upload",
|
||||
}
|
||||
}
|
||||
SQL_WRITES = {"insert", "update", "delete", "pg_insert"}
|
||||
|
||||
|
||||
def _package_unit(rel: str) -> str | None:
|
||||
for unit, prefixes in UNIT_PACKAGES.items():
|
||||
if any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in prefixes):
|
||||
return unit
|
||||
return None
|
||||
|
||||
|
||||
def _app_files():
|
||||
for path in sorted(APP.rglob("*.py")):
|
||||
rel = path.relative_to(APP).as_posix()
|
||||
if not rel.startswith("routes/"):
|
||||
yield rel, path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _imported_models(tree: ast.AST) -> dict[str, str]:
|
||||
"""The unit models a module imports at run time (a `TYPE_CHECKING` import names a type only),
|
||||
by the name it binds them to."""
|
||||
typing_only = _type_only_imports(tree)
|
||||
out = {}
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node not in typing_only:
|
||||
module = node.module or ""
|
||||
if module.startswith("app.") and "models" in module:
|
||||
for alias in node.names:
|
||||
if alias.name in UNIT_MODEL_NAMES:
|
||||
out[alias.asname or alias.name] = alias.name
|
||||
return out
|
||||
|
||||
|
||||
def _enclosing_functions(tree: ast.AST) -> dict[ast.AST, str]:
|
||||
out = {}
|
||||
for fn in ast.walk(tree):
|
||||
if isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
for node in ast.walk(fn):
|
||||
out.setdefault(node, fn.name)
|
||||
return out
|
||||
|
||||
|
||||
def package_violations(rel: str, source: str) -> list[str]:
|
||||
"""What a module outside the HTTP layer does against a fenced unit's boundary, as `line: what`."""
|
||||
tree = ast.parse(source)
|
||||
mine = _package_unit(rel)
|
||||
found = []
|
||||
models = _imported_models(tree)
|
||||
for name in models.values():
|
||||
theirs = UNIT_MODEL_NAMES[name]
|
||||
if theirs == mine or (mine not in FENCED and theirs not in FENCED):
|
||||
continue
|
||||
if theirs not in READS.get(mine or "", set()):
|
||||
found.append(f"0: {mine or 'core'} imports {theirs}'s {name}")
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name):
|
||||
continue
|
||||
written = None
|
||||
if node.func.id in models:
|
||||
written = models[node.func.id]
|
||||
elif node.func.id in SQL_WRITES and node.args and isinstance(node.args[0], ast.Name):
|
||||
written = models.get(node.args[0].id)
|
||||
if written is None:
|
||||
continue
|
||||
theirs = UNIT_MODEL_NAMES[written]
|
||||
if theirs != mine and (mine in FENCED or theirs in FENCED):
|
||||
found.append(f"{node.lineno}: {mine or 'core'} writes {theirs}'s {written}")
|
||||
if mine in FENCED:
|
||||
found += _owner_filters(tree)
|
||||
functions = _enclosing_functions(tree)
|
||||
allowed = UNIT_COMMITS.get(mine, set())
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "commit"
|
||||
and rel not in allowed
|
||||
and f"{rel}:{functions.get(node)}" not in allowed
|
||||
):
|
||||
found.append(f"{node.lineno}: commits in {functions.get(node)}")
|
||||
return sorted(found, key=lambda v: int(v.split(":")[0]))
|
||||
|
||||
|
||||
def _owner_filters(tree: ast.AST) -> list[str]:
|
||||
"""A comparison on a table's owner column written inline (`Model.user_id == …`), not through
|
||||
`userscope.owned_by`."""
|
||||
found = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Compare):
|
||||
for side in (node.left, *node.comparators):
|
||||
if (
|
||||
isinstance(side, ast.Attribute)
|
||||
and isinstance(side.value, ast.Name)
|
||||
and side.value.id in set(UNIT_MODEL_NAMES) | CORE_MODELS
|
||||
and side.attr in OWNER_COLUMNS
|
||||
):
|
||||
found.append(f"{node.lineno}: filters {side.value.id}.{side.attr} inline")
|
||||
return found
|
||||
|
||||
|
||||
def test_a_fenced_units_code_keeps_to_its_edges():
|
||||
hits = {rel: found for rel, src in _app_files() if (found := package_violations(rel, src))}
|
||||
assert hits == {}, (
|
||||
"read another unit's tables only along READS, write them only through their service, say"
|
||||
" whose a row is through userscope, commit only where UNIT_COMMITS says"
|
||||
)
|
||||
|
||||
|
||||
def test_a_fenced_units_routes_filter_owners_through_userscope():
|
||||
hits = {
|
||||
rel: found
|
||||
for rel, src in _http_files()
|
||||
if _unit_of(rel) in FENCED and (found := _owner_filters(ast.parse(src)))
|
||||
}
|
||||
assert hits == {}
|
||||
|
||||
|
||||
def test_every_unit_package_exists():
|
||||
for prefixes in UNIT_PACKAGES.values():
|
||||
for p in prefixes:
|
||||
assert (APP / p).exists(), p
|
||||
for commits in UNIT_COMMITS.values():
|
||||
for entry in commits:
|
||||
assert (APP / entry.split(":")[0]).exists(), entry
|
||||
|
||||
|
||||
PACKAGE_CASES = {
|
||||
# (module path, source): violations
|
||||
("downloads/x.py", "from app.models import Video"): 0, # capture reads catalog
|
||||
("downloads/x.py", "from app.models import Playlist"): 1, # no capture→reader edge
|
||||
("downloads/x.py", "from app.models import Video\nVideo(id=1)"): 1, # a write across
|
||||
("downloads/x.py", "from app.models import Video\nupdate(Video)"): 1,
|
||||
("downloads/x.py", "from app.downloads.models import DownloadJob\nDownloadJob()"): 0,
|
||||
("search.py", "from app.downloads.models import DownloadJob"): 1, # core reads capture
|
||||
("search.py", "from app.models import Video\nVideo()"): 0, # neither side fenced yet
|
||||
("downloads/x.py", "q = DownloadJob.user_id == 1"): 1,
|
||||
("downloads/x.py", "q = owned_by(DownloadJob.user_id, 1)"): 0,
|
||||
("downloads/x.py", "def f(db):\n db.commit()"): 1,
|
||||
("downloads/gc.py", "def f(db):\n db.commit()"): 0,
|
||||
("downloads/upload.py", "def ingest_upload(db):\n db.commit()"): 0,
|
||||
("downloads/upload.py", "def other(db):\n db.commit()"): 1,
|
||||
(
|
||||
"downloads/x.py",
|
||||
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.models import Playlist",
|
||||
): 0,
|
||||
}
|
||||
|
||||
|
||||
def test_the_package_detector_sees_each_kind():
|
||||
for (rel, source), expected in PACKAGE_CASES.items():
|
||||
assert len(package_violations(rel, source)) == expected, (rel, source)
|
||||
|
||||
Reference in New Issue
Block a user