peter
9c5f81fd94
- An ad-hoc request made while its target's run is under way is the run after it: the scheduler dropped a second fire of a running id, so a second Takeout file was never imported and its key stayed queued until a restart. - Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view no longer marks it watched again; un-marks take the user's Takeout lock. - One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the others are settled, and a failed lookup spends none of the daily share. - The upload reads the user's id before giving the read connection back. - A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a mark never pins it against the cleanup jobs. - The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and two uploads at once across all users (takeout_server_busy). - A Watch later list goes in as one presence read and one insert.
712 lines
30 KiB
Python
712 lines
30 KiB
Python
"""The HTTP layer of a unit does no database work of its own, held in code rather than prose.
|
|
|
|
A route validates, calls its unit's `queries` (reads) or `service` (writes), serializes and commits.
|
|
What it may not do is build SQL, call the session's data methods, change a model's attributes or
|
|
import a unit's tables: when a route did those, ownership, the commit and the conditional `WHERE`
|
|
lived in a different place in each handler, and a fix in one never reached its siblings.
|
|
|
|
The rule has no exemptions: a single `select()` in a unit's route file fails it.
|
|
|
|
A core route (identity, admin, ops) is not a unit's HTTP layer and may use the session; it may
|
|
still not reach into a unit's tables. A route may NAME a unit's model as a type (under
|
|
`TYPE_CHECKING`), never use one as a value.
|
|
|
|
Below the HTTP layer, a unit's own code keeps to its edges (`READS`): see the second half of this
|
|
file. Whose a row is, every module says through `userscope` — a core one too.
|
|
"""
|
|
|
|
import ast
|
|
from pathlib import Path
|
|
|
|
import app.tables # noqa: F401 — every unit's tables, for the ownership check
|
|
from app.db import Base
|
|
|
|
APP = Path(__file__).parent.parent / "app"
|
|
ROUTES = APP / "routes"
|
|
|
|
# Each unit's tables. A new model is placed here (or in CORE_MODELS) by the test below, so a table
|
|
# never joins the schema without an owner.
|
|
UNIT_MODELS = {
|
|
"capture": {
|
|
"MediaAsset",
|
|
"DownloadProfile",
|
|
"FileFolder",
|
|
"DownloadJob",
|
|
"DownloadShare",
|
|
"DownloadQuota",
|
|
"DownloadLink",
|
|
"DownloadCredential",
|
|
"LibraryState",
|
|
},
|
|
"reader": {
|
|
"VideoState",
|
|
"Playlist",
|
|
"PlaylistItem",
|
|
"SavedView",
|
|
"ComfortPin",
|
|
"TakeoutItem",
|
|
"VideoUnmark",
|
|
"Tag",
|
|
"ChannelTag",
|
|
},
|
|
"catalog": {
|
|
"Channel",
|
|
"Subscription",
|
|
"Video",
|
|
"SearchFind",
|
|
"ExploredChannel",
|
|
"BlockedChannel",
|
|
},
|
|
"plex": {
|
|
"PlexLibrary",
|
|
"PlexShow",
|
|
"PlexSeason",
|
|
"PlexItem",
|
|
"PlexPerson",
|
|
"PlexCollection",
|
|
"PlexState",
|
|
"PlexLink",
|
|
"PlexPlaylist",
|
|
"PlexPlaylistItem",
|
|
},
|
|
}
|
|
CORE_MODELS = {
|
|
"User",
|
|
"OAuthToken",
|
|
"AuthToken",
|
|
"Invite",
|
|
"ApiQuotaUsage",
|
|
"QuotaEvent",
|
|
"AuditLog",
|
|
"LogEvent",
|
|
"DebugSession",
|
|
"DebugScreenshot",
|
|
"AppState",
|
|
"JobRun",
|
|
"SystemConfig",
|
|
"Notification",
|
|
"ApiToken",
|
|
"ApiTokenGrant",
|
|
}
|
|
|
|
# Which unit each HTTP file belongs to (path relative to app/routes; a folder covers its files).
|
|
# Anything not listed is a core route.
|
|
UNIT_ROUTES = {
|
|
"downloads.py": "capture",
|
|
"files.py": "capture",
|
|
"public.py": "capture",
|
|
"feed.py": "reader",
|
|
"playlists.py": "reader",
|
|
"saved_views.py": "reader",
|
|
"discover.py": "reader",
|
|
"tags.py": "reader",
|
|
"channels.py": "catalog",
|
|
"search.py": "catalog",
|
|
"sync.py": "catalog",
|
|
"plex/": "plex",
|
|
}
|
|
|
|
# The session methods that read or write data. `commit` and `rollback` stay with the handler.
|
|
SESSION_DATA = {
|
|
"add",
|
|
"add_all",
|
|
"delete",
|
|
"execute",
|
|
"get",
|
|
"merge",
|
|
"flush",
|
|
"refresh",
|
|
"query",
|
|
"scalar",
|
|
"scalars",
|
|
"expunge",
|
|
}
|
|
# What a route may name from SQLAlchemy: the session's type, for its `db: Session` parameter, and
|
|
# the errors its commit raises (the commit is the handler's, so is catching what it throws).
|
|
SQLALCHEMY_ALLOWED = {"Session"}
|
|
SQLALCHEMY_ALLOWED_MODULES = {"sqlalchemy.exc", "sqlalchemy.orm.exc"}
|
|
# Objects a route owns and may set attributes on; anything else it writes to is a model.
|
|
OWN_OBJECTS = {"self", "cls", "request", "response"}
|
|
UNIT_MODEL_NAMES = {name: unit for unit, names in UNIT_MODELS.items() for name in names}
|
|
|
|
|
|
def _unit_of(rel: str) -> str | None:
|
|
for key, unit in UNIT_ROUTES.items():
|
|
if rel == key or (key.endswith("/") and rel.startswith(key)):
|
|
return unit
|
|
return None
|
|
|
|
|
|
def _session_names(tree: ast.AST) -> set[str]:
|
|
"""The names a module binds to a database session: a parameter typed `Session`, or a name
|
|
assigned (or `with`-bound) from `SessionLocal()`."""
|
|
names = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.arg) and isinstance(node.annotation, ast.Name):
|
|
if node.annotation.id == "Session":
|
|
names.add(node.arg)
|
|
made = None
|
|
if isinstance(node, ast.Assign) and len(node.targets) == 1:
|
|
made, target = node.value, node.targets[0]
|
|
elif isinstance(node, ast.withitem):
|
|
made, target = node.context_expr, node.optional_vars
|
|
if (
|
|
isinstance(made, ast.Call)
|
|
and isinstance(made.func, ast.Name)
|
|
and made.func.id == "SessionLocal"
|
|
and isinstance(target, ast.Name)
|
|
):
|
|
names.add(target.id)
|
|
return names
|
|
|
|
|
|
def _type_only_imports(tree: ast.AST) -> set[ast.AST]:
|
|
"""The imports under `if TYPE_CHECKING:` — they name a type for an annotation and are never
|
|
loaded at run time, so a model imported there cannot be queried or written."""
|
|
out: set[ast.AST] = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.If):
|
|
test = node.test
|
|
name = test.id if isinstance(test, ast.Name) else getattr(test, "attr", None)
|
|
if name == "TYPE_CHECKING":
|
|
for child in node.body:
|
|
out |= set(ast.walk(child))
|
|
return out
|
|
|
|
|
|
def _root(node: ast.expr) -> ast.expr:
|
|
while isinstance(node, (ast.Attribute, ast.Subscript)):
|
|
node = node.value
|
|
return node
|
|
|
|
|
|
def violations(source: str, unit: str | None) -> list[str]:
|
|
"""Each piece of database work in an HTTP file, as `line: what`. A core route (`unit` None) is
|
|
held only to the last rule: no unit's tables."""
|
|
tree = ast.parse(source)
|
|
found = []
|
|
sql_names = set()
|
|
typing_only = _type_only_imports(tree)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node not in typing_only:
|
|
module = node.module or ""
|
|
for alias in node.names:
|
|
bound = alias.asname or alias.name
|
|
if (
|
|
module.split(".")[0] == "sqlalchemy"
|
|
and module not in SQLALCHEMY_ALLOWED_MODULES
|
|
and alias.name not in SQLALCHEMY_ALLOWED
|
|
):
|
|
sql_names.add(bound)
|
|
if (
|
|
alias.name in UNIT_MODEL_NAMES
|
|
and module.startswith("app.")
|
|
and "models" in module
|
|
):
|
|
found.append(
|
|
f"{node.lineno}: imports {UNIT_MODEL_NAMES[alias.name]}'s {alias.name}"
|
|
)
|
|
elif isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
if alias.name.split(".")[0] == "sqlalchemy":
|
|
sql_names.add((alias.asname or alias.name).split(".")[0])
|
|
if unit is None:
|
|
return found
|
|
sessions = _session_names(tree)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) and node.id in sql_names:
|
|
found.append(f"{node.lineno}: uses sqlalchemy's {node.id}")
|
|
if isinstance(node, ast.Call):
|
|
f = node.func
|
|
if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name):
|
|
if f.value.id in sessions and f.attr in SESSION_DATA:
|
|
found.append(f"{node.lineno}: {f.value.id}.{f.attr}()")
|
|
targets = []
|
|
if isinstance(node, ast.Assign):
|
|
targets = node.targets
|
|
elif isinstance(node, (ast.AugAssign, ast.AnnAssign)):
|
|
targets = [node.target]
|
|
for target in targets:
|
|
if isinstance(target, ast.Attribute):
|
|
base = _root(target)
|
|
if not (isinstance(base, ast.Name) and base.id in OWN_OBJECTS):
|
|
found.append(f"{node.lineno}: sets .{target.attr}")
|
|
return sorted(found, key=lambda v: int(v.split(":")[0]))
|
|
|
|
|
|
def _http_files():
|
|
for path in sorted(ROUTES.rglob("*.py")):
|
|
rel = path.relative_to(ROUTES).as_posix()
|
|
yield rel, path.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_every_table_has_an_owner():
|
|
tables = {mapper.class_.__name__ for mapper in Base.registry.mappers}
|
|
owned = set(UNIT_MODEL_NAMES) | CORE_MODELS
|
|
assert tables - owned == set(), "a model with no unit: place it in UNIT_MODELS or CORE_MODELS"
|
|
assert owned - tables == set(), "UNIT_MODELS/CORE_MODELS name a model that no longer exists"
|
|
assert not set(UNIT_MODEL_NAMES) & CORE_MODELS
|
|
|
|
|
|
def test_every_listed_route_exists():
|
|
rels = [rel for rel, _ in _http_files()]
|
|
for key in UNIT_ROUTES:
|
|
assert any(rel == key or (key.endswith("/") and rel.startswith(key)) for rel in rels), key
|
|
|
|
|
|
def test_no_http_file_does_database_work():
|
|
hits = {rel: found for rel, src in _http_files() if (found := violations(src, _unit_of(rel)))}
|
|
assert hits == {}, (
|
|
"a route does database work: reads go through the unit's queries, writes through its"
|
|
" service; a core route reaches a unit's tables through that unit's queries too"
|
|
)
|
|
|
|
|
|
# The detector itself: each kind of database work it must see, and what it must leave alone.
|
|
DETECTOR_CASES = {
|
|
"from sqlalchemy import select\nq = select(1)": 1,
|
|
"import sqlalchemy as sa\nsa.select(1).where(sa.true())": 2,
|
|
"from sqlalchemy import func\nn = func.count()": 1,
|
|
"from sqlalchemy.orm import Session\ndef f(db: Session): db.commit(); db.rollback()": 0,
|
|
"from sqlalchemy.orm import Session\ndef f(db: Session): db.add(x)": 1,
|
|
"def f(conn: Session): conn.execute(q)": 1,
|
|
"from app.db import SessionLocal\nwith SessionLocal() as s:\n s.get(X, 1)": 1,
|
|
"from app.db import SessionLocal\ns = SessionLocal()\ns.scalars(q)": 1,
|
|
"def f(job): job.status = 'x'": 1,
|
|
"def f(job): job.meta['k'] = 1": 0,
|
|
"def f(job): job.parts.count += 1": 1,
|
|
"def f(self, request, response): self.x = 1; request.state.y = 2; response.status_code = 3": 0,
|
|
"from app.models import DownloadJob, User": 1,
|
|
"from app.capture.models import DownloadJob as J": 1,
|
|
"def f(cache): cache.get('k')": 0,
|
|
"from sqlalchemy.exc import IntegrityError\ntry:\n pass\nexcept IntegrityError:\n pass": 0,
|
|
"from sqlalchemy.orm.exc import StaleDataError\ntry:\n pass\nexcept StaleDataError:\n pass": 0,
|
|
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.downloads.models import Job": 0,
|
|
"import typing\nif typing.TYPE_CHECKING:\n from app.models import DownloadJob": 0,
|
|
"if TYPE_CHECKING:\n pass\nfrom app.downloads.models import DownloadJob": 1,
|
|
}
|
|
|
|
|
|
def test_the_detector_sees_each_kind_of_database_work():
|
|
for source, expected in DETECTOR_CASES.items():
|
|
assert len(violations(source, "capture")) == expected, source
|
|
assert violations("from app.models import Video\nfrom sqlalchemy import select", None) == [
|
|
"1: imports catalog's Video"
|
|
]
|
|
|
|
|
|
# --- inside a unit: its package --------------------------------------------------------------
|
|
# Every unit is fenced beyond its HTTP layer too. Its own code reads another unit's tables only
|
|
# along a declared edge (the feed joins Video with VideoState in one query; a boundary that split
|
|
# it would cost the speed the app is for), and writes another unit's tables never — that goes
|
|
# through the owner's service. It commits only where a job owns its session. Whose a row is, it
|
|
# says through `userscope`, and so does core code: the owner filter is checked in every module.
|
|
|
|
# Each unit's code outside the HTTP layer: its package, and the entry point that runs its jobs.
|
|
UNIT_PACKAGES = {
|
|
"capture": ("downloads/", "worker.py"),
|
|
# Home (`discover/`) is Reader's landing page; a playlist's YouTube sync and the tag suggestions
|
|
# work on Reader's tables alone.
|
|
"reader": (
|
|
"reader/",
|
|
"discover/",
|
|
"feedquery.py",
|
|
"sync/playlists.py",
|
|
"sync/playlist_merge.py",
|
|
"sync/tagsuggest.py",
|
|
),
|
|
# The catalog's ingest jobs: the subscription import, the video sync and backfills, the explore
|
|
# and search ingests, the maintenance re-checks and the auto-tagger.
|
|
"catalog": (
|
|
"catalog/",
|
|
"sync/autotag.py",
|
|
"sync/explore.py",
|
|
"sync/maintenance.py",
|
|
"sync/runner.py",
|
|
"sync/subscriptions.py",
|
|
"sync/videos.py",
|
|
),
|
|
# The Plex mirror: its sync, the watch-state sync with a linked account, TMDB enrichment, the
|
|
# pages' reads and writes.
|
|
"plex": ("plex/",),
|
|
}
|
|
# Whose tables a unit's own code may READ. Core code reads a unit's tables through its queries.
|
|
READS = {
|
|
"capture": {"catalog", "plex"},
|
|
"reader": {"catalog", "plex"},
|
|
"catalog": set(),
|
|
"plex": set(),
|
|
}
|
|
# The columns that say whose a row is.
|
|
OWNER_COLUMNS = {"user_id", "to_user_id", "from_user_id", "created_by"}
|
|
# Where a unit's code may commit: the jobs that own their session (the worker and what it
|
|
# runs, the GC, the backfill — none of them inside a request), and the one write that must commit
|
|
# where it stands: an upload's rows and the file it placed on disk land together or not at all, so
|
|
# a failed commit can remove the file (`upload.ingest_upload`). Everything else flushes; the handler
|
|
# commits.
|
|
UNIT_COMMITS = {
|
|
"capture": {
|
|
"worker.py",
|
|
"downloads/backfill.py",
|
|
"downloads/gc.py",
|
|
"downloads/jobdb.py",
|
|
"downloads/pipeline.py",
|
|
"downloads/upload.py:ingest_upload",
|
|
},
|
|
# A playlist's YouTube sync runs over many lists and many YouTube calls: it commits what each
|
|
# step settled before the next (a list created on YouTube, one list mirrored), so a later
|
|
# failure neither forgets a write YouTube already holds nor drops the lists already done. Its
|
|
# claim is a lock held on a connection of its own.
|
|
"reader": {
|
|
"sync/playlists.py:_advisory_claim",
|
|
"sync/playlists.py:pull_user_playlists",
|
|
"sync/playlists.py:merge_playlist",
|
|
"sync/playlists.py:repull_playlist",
|
|
# The Takeout import is a recorded run with its own session: the watched part lands before
|
|
# the Watch later part asks YouTube (which commits the videos it fetched).
|
|
"reader/takeout.py:run_import",
|
|
# Run inside the Shorts job, on its session: each user's waiting rows land on their own.
|
|
"reader/takeout.py:apply_waiting",
|
|
},
|
|
# The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page
|
|
# fetched and stored is not lost when the next call fails, and an enrich that fails rolls back
|
|
# to the stubs the step before committed (a search's results, a playlist's unseen videos).
|
|
"catalog": {
|
|
"catalog/search.py:classify_shorts",
|
|
"catalog/search.py:ingest_candidates",
|
|
"sync/autotag.py:apply_channel_autotags",
|
|
"sync/autotag.py:run_autotag_all",
|
|
"sync/explore.py:_enrich_new",
|
|
"sync/explore.py:purge_unkept_explored",
|
|
"sync/explore.py:purge_unkept_search",
|
|
"sync/maintenance.py:_notify_removed",
|
|
"sync/maintenance.py:_recheck_flagged",
|
|
"sync/maintenance.py:_revalidate_rolling",
|
|
"sync/runner.py:run_feed_fallback",
|
|
"sync/runner.py:_stamp_fallback_attempt",
|
|
"sync/subscriptions.py:import_subscriptions",
|
|
"sync/videos.py:insert_stubs",
|
|
"sync/videos.py:apply_rss_feed",
|
|
"sync/videos.py:backfill_channel_recent",
|
|
"sync/videos.py:backfill_channel_deep",
|
|
"sync/videos.py:reconcile_full_history",
|
|
"sync/videos.py:_apply_video_batch",
|
|
"sync/videos.py:run_shorts_classification",
|
|
"sync/videos.py:ensure_videos",
|
|
},
|
|
# The Plex jobs commit per step too: a library sync per section and per enrich/probe batch, the
|
|
# TMDB enrichment per batch, the watch sync per pull and push. The pushes and a person's bio
|
|
# fetch run as background tasks after the response, on a session of their own; the owner's
|
|
# first import runs after the handler committed the link it reads.
|
|
"plex": {
|
|
"plex/people.py:prune_orphan_people",
|
|
"plex/sync.py:sync",
|
|
"plex/sync.py:_probe_media_batch",
|
|
"plex/sync.py:_enrich_people_batch",
|
|
"plex/tmdb.py:enrich_batch",
|
|
"plex/tmdb.py:backfill_country_codes",
|
|
"plex/tmdb.py:enrich_person",
|
|
"plex/tmdb.py:enrich_people_batch",
|
|
"plex/watch_sync.py:import_owner_watch_state",
|
|
"plex/watch_sync.py:push_state_to_plex",
|
|
"plex/watch_sync.py:push_bulk_state_to_plex",
|
|
"plex/watch_sync.py:_resolve_account_id",
|
|
"plex/watch_sync.py:run_plex_watch_sync",
|
|
"plex/watch_sync.py:run_plex_watch_reconcile",
|
|
},
|
|
}
|
|
# Where core code may commit, each with why. Everything else in core flushes and leaves the commit
|
|
# to its caller, as a unit's code does; a route's own helpers are held by test_route_conventions.
|
|
CORE_COMMITS = {
|
|
# Quota spend and a background run's row are durable whatever the caller does next: each opens
|
|
# a session of its own.
|
|
"quota.py",
|
|
"jobruns.py",
|
|
# The scheduler's jobs own their session.
|
|
"scheduler.py",
|
|
# The instance's one settings row: created on first read, and the setup steps that finish it.
|
|
"state.py",
|
|
# A token's last-used stamp lands ahead of the route: a read-only route never commits, and a
|
|
# write route's rollback must not take it along. The expiry warning is a scheduled job.
|
|
"apitokens.py:_token_caller",
|
|
"apitokens.py:warn_expiring",
|
|
# An account's erasure is one operation shared by the self-service and the admin delete; the
|
|
# Google revoke and the confirmation run after it, in the background.
|
|
"auth.py:purge_user",
|
|
# The undelivered-email notice opens its own session (its callers are often background tasks).
|
|
"email.py:_alert_admins",
|
|
# One notice, committed unless its caller batches (`commit=False`).
|
|
"notifications.py:create_notification",
|
|
# A refreshed (or dead) YouTube grant must persist even when the call that needed it fails and
|
|
# its caller rolls back.
|
|
"youtube/client.py:_access_token",
|
|
"youtube/client.py:_drop_dead_grant",
|
|
}
|
|
SQL_WRITES = {"insert", "update", "delete", "pg_insert"}
|
|
ALL_MODEL_NAMES = set(UNIT_MODEL_NAMES) | CORE_MODELS
|
|
# The column methods that compare an owner column as a filter does (`Model.user_id.in_(ids)`).
|
|
# `.is_(None)` is not one: it asks for the rows nobody owns (the system's own tags).
|
|
OWNER_METHODS = {"in_", "not_in", "notin_", "__eq__", "__ne__"}
|
|
|
|
|
|
def _package_unit(rel: str) -> str | None:
|
|
for unit, prefixes in UNIT_PACKAGES.items():
|
|
if any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in prefixes):
|
|
return unit
|
|
return None
|
|
|
|
|
|
def _app_files():
|
|
for path in sorted(APP.rglob("*.py")):
|
|
rel = path.relative_to(APP).as_posix()
|
|
if not rel.startswith("routes/"):
|
|
yield rel, path.read_text(encoding="utf-8")
|
|
|
|
|
|
def _imported_models(tree: ast.AST) -> dict[str, str]:
|
|
"""The unit models a module imports at run time (a `TYPE_CHECKING` import names a type only),
|
|
by the name it binds them to."""
|
|
typing_only = _type_only_imports(tree)
|
|
out = {}
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node not in typing_only:
|
|
module = node.module or ""
|
|
if module.startswith("app.") and "models" in module:
|
|
for alias in node.names:
|
|
if alias.name in UNIT_MODEL_NAMES:
|
|
out[alias.asname or alias.name] = alias.name
|
|
return out
|
|
|
|
|
|
def _model_aliases(tree: ast.AST) -> dict[str, str]:
|
|
"""Every other name a module binds to a model, by that name: an import under another name, and
|
|
`x = aliased(Model)` — an alias is the same table, and its owner column the same filter."""
|
|
out = {}
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and "models" in (node.module or ""):
|
|
for alias in node.names:
|
|
if alias.name in ALL_MODEL_NAMES:
|
|
out[alias.asname or alias.name] = alias.name
|
|
for node in ast.walk(tree):
|
|
if (
|
|
isinstance(node, ast.Assign)
|
|
and isinstance(node.value, ast.Call)
|
|
and _called_name(node.value) == "aliased"
|
|
and node.value.args
|
|
and isinstance(node.value.args[0], ast.Name)
|
|
):
|
|
source = node.value.args[0].id
|
|
model = out.get(source, source if source in ALL_MODEL_NAMES else None)
|
|
for target in node.targets:
|
|
if model and isinstance(target, ast.Name):
|
|
out[target.id] = model
|
|
return out
|
|
|
|
|
|
def _called_name(call: ast.Call) -> str | None:
|
|
func = call.func
|
|
return func.id if isinstance(func, ast.Name) else getattr(func, "attr", None)
|
|
|
|
|
|
def _sql_write_names(tree: ast.AST) -> tuple[set[str], set[str]]:
|
|
"""The names a module calls SQLAlchemy's statement writes by — `insert`/`update`/`delete` as
|
|
imported, under any alias — and the names it binds SQLAlchemy's modules to (`sa.update(...)`)."""
|
|
functions, modules = set(SQL_WRITES), set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == "sqlalchemy":
|
|
for alias in node.names:
|
|
if alias.name in {"insert", "update", "delete"}:
|
|
functions.add(alias.asname or alias.name)
|
|
elif alias.name in {"dialects", "postgresql", "sql"}:
|
|
modules.add(alias.asname or alias.name)
|
|
elif isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
if alias.name.split(".")[0] == "sqlalchemy":
|
|
modules.add(alias.asname or alias.name.split(".")[0])
|
|
return functions, modules
|
|
|
|
|
|
def _written_table(call: ast.Call) -> str | None:
|
|
"""The name of what a statement write's first argument names: `Model`, or `Model.__table__`."""
|
|
if not call.args:
|
|
return None
|
|
arg = call.args[0]
|
|
if isinstance(arg, ast.Attribute) and arg.attr == "__table__":
|
|
arg = arg.value
|
|
return arg.id if isinstance(arg, ast.Name) else None
|
|
|
|
|
|
def _enclosing_functions(tree: ast.AST) -> dict[ast.AST, str]:
|
|
out = {}
|
|
for fn in ast.walk(tree):
|
|
if isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
for node in ast.walk(fn):
|
|
out.setdefault(node, fn.name)
|
|
return out
|
|
|
|
|
|
def package_violations(rel: str, source: str) -> list[str]:
|
|
"""What a module outside the HTTP layer does against a unit's boundary, as `line: what`."""
|
|
tree = ast.parse(source)
|
|
mine = _package_unit(rel)
|
|
found = []
|
|
models = _imported_models(tree)
|
|
for name in models.values():
|
|
theirs = UNIT_MODEL_NAMES[name]
|
|
if theirs == mine:
|
|
continue
|
|
if theirs not in READS.get(mine or "", set()):
|
|
found.append(f"0: {mine or 'core'} imports {theirs}'s {name}")
|
|
write_functions, sql_modules = _sql_write_names(tree)
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.Call):
|
|
continue
|
|
func = node.func
|
|
written = None
|
|
if isinstance(func, ast.Name) and func.id in models:
|
|
written = models[func.id]
|
|
elif (isinstance(func, ast.Name) and func.id in write_functions) or (
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr in {"insert", "update", "delete"}
|
|
and isinstance(_root(func), ast.Name)
|
|
and _root(func).id in sql_modules
|
|
):
|
|
written = models.get(_written_table(node) or "")
|
|
if written is None:
|
|
continue
|
|
theirs = UNIT_MODEL_NAMES[written]
|
|
if theirs != mine:
|
|
found.append(f"{node.lineno}: {mine or 'core'} writes {theirs}'s {written}")
|
|
found += _owner_filters(tree)
|
|
functions = _enclosing_functions(tree)
|
|
allowed = UNIT_COMMITS.get(mine, set()) if mine is not None else CORE_COMMITS
|
|
for node in ast.walk(tree):
|
|
if (
|
|
isinstance(node, ast.Call)
|
|
and isinstance(node.func, ast.Attribute)
|
|
and node.func.attr == "commit"
|
|
and rel not in allowed
|
|
and f"{rel}:{functions.get(node)}" not in allowed
|
|
):
|
|
found.append(f"{node.lineno}: commits in {functions.get(node)}")
|
|
return sorted(found, key=lambda v: int(v.split(":")[0]))
|
|
|
|
|
|
def _owner_filters(tree: ast.AST) -> list[str]:
|
|
"""A comparison on a table's owner column written inline (`Model.user_id == …`), not through
|
|
`userscope.owned_by`."""
|
|
names = {name: name for name in ALL_MODEL_NAMES} | _model_aliases(tree)
|
|
|
|
def owner_column(node: ast.expr) -> bool:
|
|
return (
|
|
isinstance(node, ast.Attribute)
|
|
and isinstance(node.value, ast.Name)
|
|
and node.value.id in names
|
|
and node.attr in OWNER_COLUMNS
|
|
)
|
|
|
|
found = []
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Compare):
|
|
for side in (node.left, *node.comparators):
|
|
if owner_column(side):
|
|
found.append(f"{node.lineno}: filters {side.value.id}.{side.attr} inline")
|
|
elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
if node.func.attr in OWNER_METHODS and owner_column(node.func.value):
|
|
found.append(f"{node.lineno}: filters an owner column with {node.func.attr}")
|
|
elif node.func.attr == "filter_by" and any(
|
|
k.arg in OWNER_COLUMNS for k in node.keywords
|
|
):
|
|
found.append(f"{node.lineno}: filters an owner column with filter_by")
|
|
return found
|
|
|
|
|
|
def test_a_units_code_keeps_to_its_edges():
|
|
hits = {rel: found for rel, src in _app_files() if (found := package_violations(rel, src))}
|
|
assert hits == {}, (
|
|
"read another unit's tables only along READS, write them only through their service, say"
|
|
" whose a row is through userscope, commit only where UNIT_COMMITS says"
|
|
)
|
|
|
|
|
|
def test_every_route_filters_owners_through_userscope():
|
|
hits = {rel: found for rel, src in _http_files() if (found := _owner_filters(ast.parse(src)))}
|
|
assert hits == {}
|
|
|
|
|
|
def test_every_unit_package_exists():
|
|
for prefixes in UNIT_PACKAGES.values():
|
|
for p in prefixes:
|
|
assert (APP / p).exists(), p
|
|
for commits in (*UNIT_COMMITS.values(), CORE_COMMITS):
|
|
for entry in commits:
|
|
assert (APP / entry.split(":")[0]).exists(), entry
|
|
|
|
|
|
def test_every_listed_commit_is_still_there():
|
|
"""An entry names a commit that exists: one left behind by a rename, or by a function that no
|
|
longer commits, would go on allowing a commit to whatever takes its name next."""
|
|
stale = []
|
|
for commits in (*UNIT_COMMITS.values(), CORE_COMMITS):
|
|
for entry in commits:
|
|
rel, _, name = entry.partition(":")
|
|
tree = ast.parse((APP / rel).read_text(encoding="utf-8"))
|
|
functions = _enclosing_functions(tree)
|
|
committing = {
|
|
functions.get(node)
|
|
for node in ast.walk(tree)
|
|
if isinstance(node, ast.Call)
|
|
and isinstance(node.func, ast.Attribute)
|
|
and node.func.attr == "commit"
|
|
}
|
|
if (name not in committing) if name else not committing:
|
|
stale.append(entry)
|
|
assert stale == []
|
|
|
|
|
|
PACKAGE_CASES = {
|
|
# (module path, source): violations
|
|
("downloads/x.py", "from app.models import Video"): 0, # capture reads catalog
|
|
("downloads/x.py", "from app.models import Playlist"): 1, # no capture→reader edge
|
|
("downloads/x.py", "from app.models import Video\nVideo(id=1)"): 1, # a write across
|
|
("downloads/x.py", "from app.models import Video\nupdate(Video)"): 1,
|
|
("downloads/x.py", "from app.downloads.models import DownloadJob\nDownloadJob()"): 0,
|
|
("search.py", "from app.downloads.models import DownloadJob"): 1, # core reads capture
|
|
("search.py", "from app.plex.models import PlexItem\nPlexItem()"): 2, # core reads, writes plex
|
|
("plex/x.py", "from app.catalog.models import Video"): 1, # no plex→catalog edge
|
|
("search.py", "from app.models import Video\nVideo()"): 2, # core reads and writes catalog
|
|
("sync/videos.py", "from app.models import Playlist"): 1, # no catalog→reader edge
|
|
("downloads/x.py", "q = DownloadJob.user_id == 1"): 1,
|
|
("downloads/x.py", "q = owned_by(DownloadJob.user_id, 1)"): 0,
|
|
("quota.py", "q = QuotaEvent.user_id == 1"): 1, # core too
|
|
("downloads/x.py", "from app.downloads.models import DownloadJob as J\nq = J.user_id == 1"): 1,
|
|
("downloads/x.py", "st = aliased(VideoState)\nq = st.user_id == 1"): 1,
|
|
("downloads/x.py", "q = DownloadJob.user_id.in_([1])"): 1,
|
|
("downloads/x.py", "q = select(DownloadJob).filter_by(user_id=1)"): 1,
|
|
("downloads/x.py", "q = Tag.user_id.is_(None)"): 0, # the rows nobody owns
|
|
(
|
|
"downloads/x.py",
|
|
"from sqlalchemy import update as up\nfrom app.models import Video\nup(Video)",
|
|
): 1,
|
|
(
|
|
"downloads/x.py",
|
|
"import sqlalchemy as sa\nfrom app.models import Video\nsa.update(Video)",
|
|
): 1,
|
|
("downloads/x.py", "from app.models import Video\nupdate(Video.__table__)"): 1,
|
|
("notifications.py", "def f(db):\n db.commit()"): 1, # core flushes too
|
|
("state.py", "def f(db):\n db.commit()"): 0,
|
|
("downloads/x.py", "def f(db):\n db.commit()"): 1,
|
|
("downloads/gc.py", "def f(db):\n db.commit()"): 0,
|
|
("downloads/upload.py", "def ingest_upload(db):\n db.commit()"): 0,
|
|
("downloads/upload.py", "def other(db):\n db.commit()"): 1,
|
|
(
|
|
"downloads/x.py",
|
|
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.models import Playlist",
|
|
): 0,
|
|
}
|
|
|
|
|
|
def test_the_package_detector_sees_each_kind():
|
|
for (rel, source), expected in PACKAGE_CASES.items():
|
|
assert len(package_violations(rel, source)) == expected, (rel, source)
|