peter
bd2340c204
The import asks YouTube how long the watched videos of the candidate channels are (five or more watched, none the user follows, blocked, put out or owns), keeps the length on the Takeout row and names the channels that make the list: five videos of five minutes or more. The shorts job asks what an import could not. A channel's counted videos come in as an exploration's when its list is opened, ready to play; a channel can be put out of the list and back.
715 lines
30 KiB
Python
715 lines
30 KiB
Python
"""The HTTP layer of a unit does no database work of its own, held in code rather than prose.
|
|
|
|
A route validates, calls its unit's `queries` (reads) or `service` (writes), serializes and commits.
|
|
What it may not do is build SQL, call the session's data methods, change a model's attributes or
|
|
import a unit's tables: when a route did those, ownership, the commit and the conditional `WHERE`
|
|
lived in a different place in each handler, and a fix in one never reached its siblings.
|
|
|
|
The rule has no exemptions: a single `select()` in a unit's route file fails it.
|
|
|
|
A core route (identity, admin, ops) is not a unit's HTTP layer and may use the session; it may
|
|
still not reach into a unit's tables. A route may NAME a unit's model as a type (under
|
|
`TYPE_CHECKING`), never use one as a value.
|
|
|
|
Below the HTTP layer, a unit's own code keeps to its edges (`READS`): see the second half of this
|
|
file. Whose a row is, every module says through `userscope` — a core one too.
|
|
"""
|
|
|
|
import ast
|
|
from pathlib import Path
|
|
|
|
import app.tables # noqa: F401 — every unit's tables, for the ownership check
|
|
from app.db import Base
|
|
|
|
APP = Path(__file__).parent.parent / "app"
|
|
ROUTES = APP / "routes"
|
|
|
|
# Each unit's tables. A new model is placed here (or in CORE_MODELS) by the test below, so a table
|
|
# never joins the schema without an owner.
|
|
UNIT_MODELS = {
|
|
"capture": {
|
|
"MediaAsset",
|
|
"DownloadProfile",
|
|
"FileFolder",
|
|
"DownloadJob",
|
|
"DownloadShare",
|
|
"DownloadQuota",
|
|
"DownloadLink",
|
|
"DownloadCredential",
|
|
"LibraryState",
|
|
},
|
|
"reader": {
|
|
"VideoState",
|
|
"Playlist",
|
|
"PlaylistItem",
|
|
"SavedView",
|
|
"ComfortPin",
|
|
"TakeoutItem",
|
|
"TakeoutChannelDismissal",
|
|
"VideoUnmark",
|
|
"Tag",
|
|
"ChannelTag",
|
|
},
|
|
"catalog": {
|
|
"Channel",
|
|
"Subscription",
|
|
"Video",
|
|
"SearchFind",
|
|
"ExploredChannel",
|
|
"BlockedChannel",
|
|
},
|
|
"plex": {
|
|
"PlexLibrary",
|
|
"PlexShow",
|
|
"PlexSeason",
|
|
"PlexItem",
|
|
"PlexPerson",
|
|
"PlexCollection",
|
|
"PlexState",
|
|
"PlexLink",
|
|
"PlexPlaylist",
|
|
"PlexPlaylistItem",
|
|
},
|
|
}
|
|
CORE_MODELS = {
|
|
"User",
|
|
"OAuthToken",
|
|
"AuthToken",
|
|
"Invite",
|
|
"ApiQuotaUsage",
|
|
"QuotaEvent",
|
|
"AuditLog",
|
|
"LogEvent",
|
|
"DebugSession",
|
|
"DebugScreenshot",
|
|
"AppState",
|
|
"JobRun",
|
|
"SystemConfig",
|
|
"Notification",
|
|
"ApiToken",
|
|
"ApiTokenGrant",
|
|
}
|
|
|
|
# Which unit each HTTP file belongs to (path relative to app/routes; a folder covers its files).
|
|
# Anything not listed is a core route.
|
|
UNIT_ROUTES = {
|
|
"downloads.py": "capture",
|
|
"files.py": "capture",
|
|
"public.py": "capture",
|
|
"feed.py": "reader",
|
|
"playlists.py": "reader",
|
|
"saved_views.py": "reader",
|
|
"discover.py": "reader",
|
|
"tags.py": "reader",
|
|
"channels.py": "catalog",
|
|
"search.py": "catalog",
|
|
"sync.py": "catalog",
|
|
"plex/": "plex",
|
|
}
|
|
|
|
# The session methods that read or write data. `commit` and `rollback` stay with the handler.
|
|
SESSION_DATA = {
|
|
"add",
|
|
"add_all",
|
|
"delete",
|
|
"execute",
|
|
"get",
|
|
"merge",
|
|
"flush",
|
|
"refresh",
|
|
"query",
|
|
"scalar",
|
|
"scalars",
|
|
"expunge",
|
|
}
|
|
# What a route may name from SQLAlchemy: the session's type, for its `db: Session` parameter, and
|
|
# the errors its commit raises (the commit is the handler's, so is catching what it throws).
|
|
SQLALCHEMY_ALLOWED = {"Session"}
|
|
SQLALCHEMY_ALLOWED_MODULES = {"sqlalchemy.exc", "sqlalchemy.orm.exc"}
|
|
# Objects a route owns and may set attributes on; anything else it writes to is a model.
|
|
OWN_OBJECTS = {"self", "cls", "request", "response"}
|
|
UNIT_MODEL_NAMES = {name: unit for unit, names in UNIT_MODELS.items() for name in names}
|
|
|
|
|
|
def _unit_of(rel: str) -> str | None:
|
|
for key, unit in UNIT_ROUTES.items():
|
|
if rel == key or (key.endswith("/") and rel.startswith(key)):
|
|
return unit
|
|
return None
|
|
|
|
|
|
def _session_names(tree: ast.AST) -> set[str]:
|
|
"""The names a module binds to a database session: a parameter typed `Session`, or a name
|
|
assigned (or `with`-bound) from `SessionLocal()`."""
|
|
names = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.arg) and isinstance(node.annotation, ast.Name):
|
|
if node.annotation.id == "Session":
|
|
names.add(node.arg)
|
|
made = None
|
|
if isinstance(node, ast.Assign) and len(node.targets) == 1:
|
|
made, target = node.value, node.targets[0]
|
|
elif isinstance(node, ast.withitem):
|
|
made, target = node.context_expr, node.optional_vars
|
|
if (
|
|
isinstance(made, ast.Call)
|
|
and isinstance(made.func, ast.Name)
|
|
and made.func.id == "SessionLocal"
|
|
and isinstance(target, ast.Name)
|
|
):
|
|
names.add(target.id)
|
|
return names
|
|
|
|
|
|
def _type_only_imports(tree: ast.AST) -> set[ast.AST]:
|
|
"""The imports under `if TYPE_CHECKING:` — they name a type for an annotation and are never
|
|
loaded at run time, so a model imported there cannot be queried or written."""
|
|
out: set[ast.AST] = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.If):
|
|
test = node.test
|
|
name = test.id if isinstance(test, ast.Name) else getattr(test, "attr", None)
|
|
if name == "TYPE_CHECKING":
|
|
for child in node.body:
|
|
out |= set(ast.walk(child))
|
|
return out
|
|
|
|
|
|
def _root(node: ast.expr) -> ast.expr:
|
|
while isinstance(node, (ast.Attribute, ast.Subscript)):
|
|
node = node.value
|
|
return node
|
|
|
|
|
|
def violations(source: str, unit: str | None) -> list[str]:
|
|
"""Each piece of database work in an HTTP file, as `line: what`. A core route (`unit` None) is
|
|
held only to the last rule: no unit's tables."""
|
|
tree = ast.parse(source)
|
|
found = []
|
|
sql_names = set()
|
|
typing_only = _type_only_imports(tree)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node not in typing_only:
|
|
module = node.module or ""
|
|
for alias in node.names:
|
|
bound = alias.asname or alias.name
|
|
if (
|
|
module.split(".")[0] == "sqlalchemy"
|
|
and module not in SQLALCHEMY_ALLOWED_MODULES
|
|
and alias.name not in SQLALCHEMY_ALLOWED
|
|
):
|
|
sql_names.add(bound)
|
|
if (
|
|
alias.name in UNIT_MODEL_NAMES
|
|
and module.startswith("app.")
|
|
and "models" in module
|
|
):
|
|
found.append(
|
|
f"{node.lineno}: imports {UNIT_MODEL_NAMES[alias.name]}'s {alias.name}"
|
|
)
|
|
elif isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
if alias.name.split(".")[0] == "sqlalchemy":
|
|
sql_names.add((alias.asname or alias.name).split(".")[0])
|
|
if unit is None:
|
|
return found
|
|
sessions = _session_names(tree)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) and node.id in sql_names:
|
|
found.append(f"{node.lineno}: uses sqlalchemy's {node.id}")
|
|
if isinstance(node, ast.Call):
|
|
f = node.func
|
|
if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name):
|
|
if f.value.id in sessions and f.attr in SESSION_DATA:
|
|
found.append(f"{node.lineno}: {f.value.id}.{f.attr}()")
|
|
targets = []
|
|
if isinstance(node, ast.Assign):
|
|
targets = node.targets
|
|
elif isinstance(node, (ast.AugAssign, ast.AnnAssign)):
|
|
targets = [node.target]
|
|
for target in targets:
|
|
if isinstance(target, ast.Attribute):
|
|
base = _root(target)
|
|
if not (isinstance(base, ast.Name) and base.id in OWN_OBJECTS):
|
|
found.append(f"{node.lineno}: sets .{target.attr}")
|
|
return sorted(found, key=lambda v: int(v.split(":")[0]))
|
|
|
|
|
|
def _http_files():
|
|
for path in sorted(ROUTES.rglob("*.py")):
|
|
rel = path.relative_to(ROUTES).as_posix()
|
|
yield rel, path.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_every_table_has_an_owner():
|
|
tables = {mapper.class_.__name__ for mapper in Base.registry.mappers}
|
|
owned = set(UNIT_MODEL_NAMES) | CORE_MODELS
|
|
assert tables - owned == set(), "a model with no unit: place it in UNIT_MODELS or CORE_MODELS"
|
|
assert owned - tables == set(), "UNIT_MODELS/CORE_MODELS name a model that no longer exists"
|
|
assert not set(UNIT_MODEL_NAMES) & CORE_MODELS
|
|
|
|
|
|
def test_every_listed_route_exists():
|
|
rels = [rel for rel, _ in _http_files()]
|
|
for key in UNIT_ROUTES:
|
|
assert any(rel == key or (key.endswith("/") and rel.startswith(key)) for rel in rels), key
|
|
|
|
|
|
def test_no_http_file_does_database_work():
|
|
hits = {rel: found for rel, src in _http_files() if (found := violations(src, _unit_of(rel)))}
|
|
assert hits == {}, (
|
|
"a route does database work: reads go through the unit's queries, writes through its"
|
|
" service; a core route reaches a unit's tables through that unit's queries too"
|
|
)
|
|
|
|
|
|
# The detector itself: each kind of database work it must see, and what it must leave alone.
|
|
DETECTOR_CASES = {
|
|
"from sqlalchemy import select\nq = select(1)": 1,
|
|
"import sqlalchemy as sa\nsa.select(1).where(sa.true())": 2,
|
|
"from sqlalchemy import func\nn = func.count()": 1,
|
|
"from sqlalchemy.orm import Session\ndef f(db: Session): db.commit(); db.rollback()": 0,
|
|
"from sqlalchemy.orm import Session\ndef f(db: Session): db.add(x)": 1,
|
|
"def f(conn: Session): conn.execute(q)": 1,
|
|
"from app.db import SessionLocal\nwith SessionLocal() as s:\n s.get(X, 1)": 1,
|
|
"from app.db import SessionLocal\ns = SessionLocal()\ns.scalars(q)": 1,
|
|
"def f(job): job.status = 'x'": 1,
|
|
"def f(job): job.meta['k'] = 1": 0,
|
|
"def f(job): job.parts.count += 1": 1,
|
|
"def f(self, request, response): self.x = 1; request.state.y = 2; response.status_code = 3": 0,
|
|
"from app.models import DownloadJob, User": 1,
|
|
"from app.capture.models import DownloadJob as J": 1,
|
|
"def f(cache): cache.get('k')": 0,
|
|
"from sqlalchemy.exc import IntegrityError\ntry:\n pass\nexcept IntegrityError:\n pass": 0,
|
|
"from sqlalchemy.orm.exc import StaleDataError\ntry:\n pass\nexcept StaleDataError:\n pass": 0,
|
|
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.downloads.models import Job": 0,
|
|
"import typing\nif typing.TYPE_CHECKING:\n from app.models import DownloadJob": 0,
|
|
"if TYPE_CHECKING:\n pass\nfrom app.downloads.models import DownloadJob": 1,
|
|
}
|
|
|
|
|
|
def test_the_detector_sees_each_kind_of_database_work():
|
|
for source, expected in DETECTOR_CASES.items():
|
|
assert len(violations(source, "capture")) == expected, source
|
|
assert violations("from app.models import Video\nfrom sqlalchemy import select", None) == [
|
|
"1: imports catalog's Video"
|
|
]
|
|
|
|
|
|
# --- inside a unit: its package --------------------------------------------------------------
|
|
# Every unit is fenced beyond its HTTP layer too. Its own code reads another unit's tables only
|
|
# along a declared edge (the feed joins Video with VideoState in one query; a boundary that split
|
|
# it would cost the speed the app is for), and writes another unit's tables never — that goes
|
|
# through the owner's service. It commits only where a job owns its session. Whose a row is, it
|
|
# says through `userscope`, and so does core code: the owner filter is checked in every module.
|
|
|
|
# Each unit's code outside the HTTP layer: its package, and the entry point that runs its jobs.
|
|
UNIT_PACKAGES = {
|
|
"capture": ("downloads/", "worker.py"),
|
|
# Home (`discover/`) is Reader's landing page; a playlist's YouTube sync and the tag suggestions
|
|
# work on Reader's tables alone.
|
|
"reader": (
|
|
"reader/",
|
|
"discover/",
|
|
"feedquery.py",
|
|
"sync/playlists.py",
|
|
"sync/playlist_merge.py",
|
|
"sync/tagsuggest.py",
|
|
),
|
|
# The catalog's ingest jobs: the subscription import, the video sync and backfills, the explore
|
|
# and search ingests, the maintenance re-checks and the auto-tagger.
|
|
"catalog": (
|
|
"catalog/",
|
|
"sync/autotag.py",
|
|
"sync/explore.py",
|
|
"sync/maintenance.py",
|
|
"sync/runner.py",
|
|
"sync/subscriptions.py",
|
|
"sync/videos.py",
|
|
),
|
|
# The Plex mirror: its sync, the watch-state sync with a linked account, TMDB enrichment, the
|
|
# pages' reads and writes.
|
|
"plex": ("plex/",),
|
|
}
|
|
# Whose tables a unit's own code may READ. Core code reads a unit's tables through its queries.
|
|
READS = {
|
|
"capture": {"catalog", "plex"},
|
|
"reader": {"catalog", "plex"},
|
|
"catalog": set(),
|
|
"plex": set(),
|
|
}
|
|
# The columns that say whose a row is.
|
|
OWNER_COLUMNS = {"user_id", "to_user_id", "from_user_id", "created_by"}
|
|
# Where a unit's code may commit: the jobs that own their session (the worker and what it
|
|
# runs, the GC, the backfill — none of them inside a request), and the one write that must commit
|
|
# where it stands: an upload's rows and the file it placed on disk land together or not at all, so
|
|
# a failed commit can remove the file (`upload.ingest_upload`). Everything else flushes; the handler
|
|
# commits.
|
|
UNIT_COMMITS = {
|
|
"capture": {
|
|
"worker.py",
|
|
"downloads/backfill.py",
|
|
"downloads/gc.py",
|
|
"downloads/jobdb.py",
|
|
"downloads/pipeline.py",
|
|
"downloads/upload.py:ingest_upload",
|
|
},
|
|
# A playlist's YouTube sync runs over many lists and many YouTube calls: it commits what each
|
|
# step settled before the next (a list created on YouTube, one list mirrored), so a later
|
|
# failure neither forgets a write YouTube already holds nor drops the lists already done. Its
|
|
# claim is a lock held on a connection of its own.
|
|
"reader": {
|
|
"sync/playlists.py:_advisory_claim",
|
|
"sync/playlists.py:pull_user_playlists",
|
|
"sync/playlists.py:merge_playlist",
|
|
"sync/playlists.py:repull_playlist",
|
|
# The Takeout import is a recorded run with its own session: the watched part lands before
|
|
# the Watch later part asks YouTube (which commits the videos it fetched).
|
|
"reader/takeout.py:run_import",
|
|
# Run inside the Shorts job, on its session: each user's waiting rows land on their own.
|
|
"reader/takeout.py:apply_waiting",
|
|
# Inside either of those: the lengths YouTube gave land before the channels are asked for.
|
|
"reader/takeout.py:_channel_list",
|
|
},
|
|
# The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page
|
|
# fetched and stored is not lost when the next call fails, and an enrich that fails rolls back
|
|
# to the stubs the step before committed (a search's results, a playlist's unseen videos).
|
|
"catalog": {
|
|
"catalog/search.py:classify_shorts",
|
|
"catalog/search.py:ingest_candidates",
|
|
"sync/autotag.py:apply_channel_autotags",
|
|
"sync/autotag.py:run_autotag_all",
|
|
"sync/explore.py:_enrich_new",
|
|
"sync/explore.py:purge_unkept_explored",
|
|
"sync/explore.py:purge_unkept_search",
|
|
"sync/maintenance.py:_notify_removed",
|
|
"sync/maintenance.py:_recheck_flagged",
|
|
"sync/maintenance.py:_revalidate_rolling",
|
|
"sync/runner.py:run_feed_fallback",
|
|
"sync/runner.py:_stamp_fallback_attempt",
|
|
"sync/subscriptions.py:import_subscriptions",
|
|
"sync/videos.py:insert_stubs",
|
|
"sync/videos.py:apply_rss_feed",
|
|
"sync/videos.py:backfill_channel_recent",
|
|
"sync/videos.py:backfill_channel_deep",
|
|
"sync/videos.py:reconcile_full_history",
|
|
"sync/videos.py:_apply_video_batch",
|
|
"sync/videos.py:run_shorts_classification",
|
|
"sync/videos.py:ensure_videos",
|
|
},
|
|
# The Plex jobs commit per step too: a library sync per section and per enrich/probe batch, the
|
|
# TMDB enrichment per batch, the watch sync per pull and push. The pushes and a person's bio
|
|
# fetch run as background tasks after the response, on a session of their own; the owner's
|
|
# first import runs after the handler committed the link it reads.
|
|
"plex": {
|
|
"plex/people.py:prune_orphan_people",
|
|
"plex/sync.py:sync",
|
|
"plex/sync.py:_probe_media_batch",
|
|
"plex/sync.py:_enrich_people_batch",
|
|
"plex/tmdb.py:enrich_batch",
|
|
"plex/tmdb.py:backfill_country_codes",
|
|
"plex/tmdb.py:enrich_person",
|
|
"plex/tmdb.py:enrich_people_batch",
|
|
"plex/watch_sync.py:import_owner_watch_state",
|
|
"plex/watch_sync.py:push_state_to_plex",
|
|
"plex/watch_sync.py:push_bulk_state_to_plex",
|
|
"plex/watch_sync.py:_resolve_account_id",
|
|
"plex/watch_sync.py:run_plex_watch_sync",
|
|
"plex/watch_sync.py:run_plex_watch_reconcile",
|
|
},
|
|
}
|
|
# Where core code may commit, each with why. Everything else in core flushes and leaves the commit
|
|
# to its caller, as a unit's code does; a route's own helpers are held by test_route_conventions.
|
|
CORE_COMMITS = {
|
|
# Quota spend and a background run's row are durable whatever the caller does next: each opens
|
|
# a session of its own.
|
|
"quota.py",
|
|
"jobruns.py",
|
|
# The scheduler's jobs own their session.
|
|
"scheduler.py",
|
|
# The instance's one settings row: created on first read, and the setup steps that finish it.
|
|
"state.py",
|
|
# A token's last-used stamp lands ahead of the route: a read-only route never commits, and a
|
|
# write route's rollback must not take it along. The expiry warning is a scheduled job.
|
|
"apitokens.py:_token_caller",
|
|
"apitokens.py:warn_expiring",
|
|
# An account's erasure is one operation shared by the self-service and the admin delete; the
|
|
# Google revoke and the confirmation run after it, in the background.
|
|
"auth.py:purge_user",
|
|
# The undelivered-email notice opens its own session (its callers are often background tasks).
|
|
"email.py:_alert_admins",
|
|
# One notice, committed unless its caller batches (`commit=False`).
|
|
"notifications.py:create_notification",
|
|
# A refreshed (or dead) YouTube grant must persist even when the call that needed it fails and
|
|
# its caller rolls back.
|
|
"youtube/client.py:_access_token",
|
|
"youtube/client.py:_drop_dead_grant",
|
|
}
|
|
SQL_WRITES = {"insert", "update", "delete", "pg_insert"}
|
|
ALL_MODEL_NAMES = set(UNIT_MODEL_NAMES) | CORE_MODELS
|
|
# The column methods that compare an owner column as a filter does (`Model.user_id.in_(ids)`).
|
|
# `.is_(None)` is not one: it asks for the rows nobody owns (the system's own tags).
|
|
OWNER_METHODS = {"in_", "not_in", "notin_", "__eq__", "__ne__"}
|
|
|
|
|
|
def _package_unit(rel: str) -> str | None:
|
|
for unit, prefixes in UNIT_PACKAGES.items():
|
|
if any(rel == p or (p.endswith("/") and rel.startswith(p)) for p in prefixes):
|
|
return unit
|
|
return None
|
|
|
|
|
|
def _app_files():
|
|
for path in sorted(APP.rglob("*.py")):
|
|
rel = path.relative_to(APP).as_posix()
|
|
if not rel.startswith("routes/"):
|
|
yield rel, path.read_text(encoding="utf-8")
|
|
|
|
|
|
def _imported_models(tree: ast.AST) -> dict[str, str]:
|
|
"""The unit models a module imports at run time (a `TYPE_CHECKING` import names a type only),
|
|
by the name it binds them to."""
|
|
typing_only = _type_only_imports(tree)
|
|
out = {}
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node not in typing_only:
|
|
module = node.module or ""
|
|
if module.startswith("app.") and "models" in module:
|
|
for alias in node.names:
|
|
if alias.name in UNIT_MODEL_NAMES:
|
|
out[alias.asname or alias.name] = alias.name
|
|
return out
|
|
|
|
|
|
def _model_aliases(tree: ast.AST) -> dict[str, str]:
|
|
"""Every other name a module binds to a model, by that name: an import under another name, and
|
|
`x = aliased(Model)` — an alias is the same table, and its owner column the same filter."""
|
|
out = {}
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and "models" in (node.module or ""):
|
|
for alias in node.names:
|
|
if alias.name in ALL_MODEL_NAMES:
|
|
out[alias.asname or alias.name] = alias.name
|
|
for node in ast.walk(tree):
|
|
if (
|
|
isinstance(node, ast.Assign)
|
|
and isinstance(node.value, ast.Call)
|
|
and _called_name(node.value) == "aliased"
|
|
and node.value.args
|
|
and isinstance(node.value.args[0], ast.Name)
|
|
):
|
|
source = node.value.args[0].id
|
|
model = out.get(source, source if source in ALL_MODEL_NAMES else None)
|
|
for target in node.targets:
|
|
if model and isinstance(target, ast.Name):
|
|
out[target.id] = model
|
|
return out
|
|
|
|
|
|
def _called_name(call: ast.Call) -> str | None:
|
|
func = call.func
|
|
return func.id if isinstance(func, ast.Name) else getattr(func, "attr", None)
|
|
|
|
|
|
def _sql_write_names(tree: ast.AST) -> tuple[set[str], set[str]]:
|
|
"""The names a module calls SQLAlchemy's statement writes by — `insert`/`update`/`delete` as
|
|
imported, under any alias — and the names it binds SQLAlchemy's modules to (`sa.update(...)`)."""
|
|
functions, modules = set(SQL_WRITES), set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == "sqlalchemy":
|
|
for alias in node.names:
|
|
if alias.name in {"insert", "update", "delete"}:
|
|
functions.add(alias.asname or alias.name)
|
|
elif alias.name in {"dialects", "postgresql", "sql"}:
|
|
modules.add(alias.asname or alias.name)
|
|
elif isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
if alias.name.split(".")[0] == "sqlalchemy":
|
|
modules.add(alias.asname or alias.name.split(".")[0])
|
|
return functions, modules
|
|
|
|
|
|
def _written_table(call: ast.Call) -> str | None:
|
|
"""The name of what a statement write's first argument names: `Model`, or `Model.__table__`."""
|
|
if not call.args:
|
|
return None
|
|
arg = call.args[0]
|
|
if isinstance(arg, ast.Attribute) and arg.attr == "__table__":
|
|
arg = arg.value
|
|
return arg.id if isinstance(arg, ast.Name) else None
|
|
|
|
|
|
def _enclosing_functions(tree: ast.AST) -> dict[ast.AST, str]:
|
|
out = {}
|
|
for fn in ast.walk(tree):
|
|
if isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
for node in ast.walk(fn):
|
|
out.setdefault(node, fn.name)
|
|
return out
|
|
|
|
|
|
def package_violations(rel: str, source: str) -> list[str]:
|
|
"""What a module outside the HTTP layer does against a unit's boundary, as `line: what`."""
|
|
tree = ast.parse(source)
|
|
mine = _package_unit(rel)
|
|
found = []
|
|
models = _imported_models(tree)
|
|
for name in models.values():
|
|
theirs = UNIT_MODEL_NAMES[name]
|
|
if theirs == mine:
|
|
continue
|
|
if theirs not in READS.get(mine or "", set()):
|
|
found.append(f"0: {mine or 'core'} imports {theirs}'s {name}")
|
|
write_functions, sql_modules = _sql_write_names(tree)
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.Call):
|
|
continue
|
|
func = node.func
|
|
written = None
|
|
if isinstance(func, ast.Name) and func.id in models:
|
|
written = models[func.id]
|
|
elif (isinstance(func, ast.Name) and func.id in write_functions) or (
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr in {"insert", "update", "delete"}
|
|
and isinstance(_root(func), ast.Name)
|
|
and _root(func).id in sql_modules
|
|
):
|
|
written = models.get(_written_table(node) or "")
|
|
if written is None:
|
|
continue
|
|
theirs = UNIT_MODEL_NAMES[written]
|
|
if theirs != mine:
|
|
found.append(f"{node.lineno}: {mine or 'core'} writes {theirs}'s {written}")
|
|
found += _owner_filters(tree)
|
|
functions = _enclosing_functions(tree)
|
|
allowed = UNIT_COMMITS.get(mine, set()) if mine is not None else CORE_COMMITS
|
|
for node in ast.walk(tree):
|
|
if (
|
|
isinstance(node, ast.Call)
|
|
and isinstance(node.func, ast.Attribute)
|
|
and node.func.attr == "commit"
|
|
and rel not in allowed
|
|
and f"{rel}:{functions.get(node)}" not in allowed
|
|
):
|
|
found.append(f"{node.lineno}: commits in {functions.get(node)}")
|
|
return sorted(found, key=lambda v: int(v.split(":")[0]))
|
|
|
|
|
|
def _owner_filters(tree: ast.AST) -> list[str]:
|
|
"""A comparison on a table's owner column written inline (`Model.user_id == …`), not through
|
|
`userscope.owned_by`."""
|
|
names = {name: name for name in ALL_MODEL_NAMES} | _model_aliases(tree)
|
|
|
|
def owner_column(node: ast.expr) -> bool:
|
|
return (
|
|
isinstance(node, ast.Attribute)
|
|
and isinstance(node.value, ast.Name)
|
|
and node.value.id in names
|
|
and node.attr in OWNER_COLUMNS
|
|
)
|
|
|
|
found = []
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Compare):
|
|
for side in (node.left, *node.comparators):
|
|
if owner_column(side):
|
|
found.append(f"{node.lineno}: filters {side.value.id}.{side.attr} inline")
|
|
elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
if node.func.attr in OWNER_METHODS and owner_column(node.func.value):
|
|
found.append(f"{node.lineno}: filters an owner column with {node.func.attr}")
|
|
elif node.func.attr == "filter_by" and any(
|
|
k.arg in OWNER_COLUMNS for k in node.keywords
|
|
):
|
|
found.append(f"{node.lineno}: filters an owner column with filter_by")
|
|
return found
|
|
|
|
|
|
def test_a_units_code_keeps_to_its_edges():
|
|
hits = {rel: found for rel, src in _app_files() if (found := package_violations(rel, src))}
|
|
assert hits == {}, (
|
|
"read another unit's tables only along READS, write them only through their service, say"
|
|
" whose a row is through userscope, commit only where UNIT_COMMITS says"
|
|
)
|
|
|
|
|
|
def test_every_route_filters_owners_through_userscope():
|
|
hits = {rel: found for rel, src in _http_files() if (found := _owner_filters(ast.parse(src)))}
|
|
assert hits == {}
|
|
|
|
|
|
def test_every_unit_package_exists():
|
|
for prefixes in UNIT_PACKAGES.values():
|
|
for p in prefixes:
|
|
assert (APP / p).exists(), p
|
|
for commits in (*UNIT_COMMITS.values(), CORE_COMMITS):
|
|
for entry in commits:
|
|
assert (APP / entry.split(":")[0]).exists(), entry
|
|
|
|
|
|
def test_every_listed_commit_is_still_there():
|
|
"""An entry names a commit that exists: one left behind by a rename, or by a function that no
|
|
longer commits, would go on allowing a commit to whatever takes its name next."""
|
|
stale = []
|
|
for commits in (*UNIT_COMMITS.values(), CORE_COMMITS):
|
|
for entry in commits:
|
|
rel, _, name = entry.partition(":")
|
|
tree = ast.parse((APP / rel).read_text(encoding="utf-8"))
|
|
functions = _enclosing_functions(tree)
|
|
committing = {
|
|
functions.get(node)
|
|
for node in ast.walk(tree)
|
|
if isinstance(node, ast.Call)
|
|
and isinstance(node.func, ast.Attribute)
|
|
and node.func.attr == "commit"
|
|
}
|
|
if (name not in committing) if name else not committing:
|
|
stale.append(entry)
|
|
assert stale == []
|
|
|
|
|
|
PACKAGE_CASES = {
|
|
# (module path, source): violations
|
|
("downloads/x.py", "from app.models import Video"): 0, # capture reads catalog
|
|
("downloads/x.py", "from app.models import Playlist"): 1, # no capture→reader edge
|
|
("downloads/x.py", "from app.models import Video\nVideo(id=1)"): 1, # a write across
|
|
("downloads/x.py", "from app.models import Video\nupdate(Video)"): 1,
|
|
("downloads/x.py", "from app.downloads.models import DownloadJob\nDownloadJob()"): 0,
|
|
("search.py", "from app.downloads.models import DownloadJob"): 1, # core reads capture
|
|
("search.py", "from app.plex.models import PlexItem\nPlexItem()"): 2, # core reads, writes plex
|
|
("plex/x.py", "from app.catalog.models import Video"): 1, # no plex→catalog edge
|
|
("search.py", "from app.models import Video\nVideo()"): 2, # core reads and writes catalog
|
|
("sync/videos.py", "from app.models import Playlist"): 1, # no catalog→reader edge
|
|
("downloads/x.py", "q = DownloadJob.user_id == 1"): 1,
|
|
("downloads/x.py", "q = owned_by(DownloadJob.user_id, 1)"): 0,
|
|
("quota.py", "q = QuotaEvent.user_id == 1"): 1, # core too
|
|
("downloads/x.py", "from app.downloads.models import DownloadJob as J\nq = J.user_id == 1"): 1,
|
|
("downloads/x.py", "st = aliased(VideoState)\nq = st.user_id == 1"): 1,
|
|
("downloads/x.py", "q = DownloadJob.user_id.in_([1])"): 1,
|
|
("downloads/x.py", "q = select(DownloadJob).filter_by(user_id=1)"): 1,
|
|
("downloads/x.py", "q = Tag.user_id.is_(None)"): 0, # the rows nobody owns
|
|
(
|
|
"downloads/x.py",
|
|
"from sqlalchemy import update as up\nfrom app.models import Video\nup(Video)",
|
|
): 1,
|
|
(
|
|
"downloads/x.py",
|
|
"import sqlalchemy as sa\nfrom app.models import Video\nsa.update(Video)",
|
|
): 1,
|
|
("downloads/x.py", "from app.models import Video\nupdate(Video.__table__)"): 1,
|
|
("notifications.py", "def f(db):\n db.commit()"): 1, # core flushes too
|
|
("state.py", "def f(db):\n db.commit()"): 0,
|
|
("downloads/x.py", "def f(db):\n db.commit()"): 1,
|
|
("downloads/gc.py", "def f(db):\n db.commit()"): 0,
|
|
("downloads/upload.py", "def ingest_upload(db):\n db.commit()"): 0,
|
|
("downloads/upload.py", "def other(db):\n db.commit()"): 1,
|
|
(
|
|
"downloads/x.py",
|
|
"from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from app.models import Playlist",
|
|
): 0,
|
|
}
|
|
|
|
|
|
def test_the_package_detector_sees_each_kind():
|
|
for (rel, source), expected in PACKAGE_CASES.items():
|
|
assert len(package_violations(rel, source)) == expected, (rel, source)
|