fix(i18n): read every refusal from the backend's syntax tree, not by regex

The contract read Python with regexes, and each guard added for a missed shape had shapes of its own
it missed: a reason after another key, a status passed by name, a second `e.reason` handler with the
same text, a pin naming a type nothing checked.

backend/tests/refusal_sites.py (stdlib `ast`, run by the contract on the host's Python) reads every
`{"code": ...}` detail and follows each reason to its values: literals, both arms of a conditional,
module constants, the `Literal` on the parameter or attribute it travels in (on the class an `except`
catches), every call of a `str` helper with its defaults, a map's values with its keys held to the
index's type. What it cannot follow fails the contract, unless no screen shows that code; no pins.

The reason attributes are typed with their Literal, so a subclass or an assignment cannot widen
them; InvalidSetting takes a SettingReason, and the settings map is held to it (a reason the map
lacked was a 500). The per-family reverse checks are one check by prefix (the admin actions
included), and the leaf-key filter applies only to the suffixed sign-in family.
This commit is contained in:
2026-10-31 15:07:38 +01:00
parent ceadf69e43
commit 5e8589bbf8
13 changed files with 1022 additions and 254 deletions
+1 -1
View File
@@ -78,7 +78,7 @@ class ClipError(Exception):
message names (the cap itself, so the client never says a stale one)."""
def __init__(self, reason: ClipReason, **params: float):
self.reason = reason
self.reason: ClipReason = reason
self.params = params
super().__init__(reason)
+1 -1
View File
@@ -174,7 +174,7 @@ class InvalidCookies(ValueError):
def __init__(self, reason: CookieReason):
super().__init__(reason)
self.reason = reason
self.reason: CookieReason = reason
def validate_cookies(raw: str) -> str:
+1 -1
View File
@@ -197,7 +197,7 @@ class FolderError(Exception):
def __init__(self, reason: FolderReason) -> None:
super().__init__(reason)
self.reason = reason
self.reason: FolderReason = reason
def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
+1 -1
View File
@@ -60,7 +60,7 @@ class QuotaExceeded(Exception):
"""Raised by check_enqueue; `reason` names the cap it hit (an i18n key on the client)."""
def __init__(self, reason: QuotaReason, limit: int, current: int):
self.reason = reason
self.reason: QuotaReason = reason
self.limit = limit
self.current = current
super().__init__(f"download quota exceeded: {reason} ({current}/{limit})")
+1 -1
View File
@@ -49,7 +49,7 @@ class EditError(Exception):
"""Bad edit request (empty spec / source not ready / not media); `reason` names the sentence."""
def __init__(self, reason: EditReason):
self.reason = reason
self.reason: EditReason = reason
super().__init__(reason)
+1 -1
View File
@@ -59,7 +59,7 @@ class BlockedUrl(Exception):
def __init__(self, reason: UrlBlockedReason):
super().__init__(reason)
self.reason = reason
self.reason: UrlBlockedReason = reason
def _is_forbidden(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+1 -1
View File
@@ -311,7 +311,7 @@ class MoveRefused(Exception):
def __init__(self, reason: MoveReason):
super().__init__(reason)
self.reason = reason
self.reason: MoveReason = reason
def touch(pl: Playlist) -> None:
+13 -10
View File
@@ -7,16 +7,19 @@ whatever sentence the route happened to hand-write. `refused` is the generic fam
(`errors.refused.<reason>`, the params interpolated into the sentence); the older specific codes
(`not_found`, `youtube`, `playlist`, the admin refusals, …) stay, since each is already translated.
A test (`test_http_details.py`) reads every `HTTPException` in the app and fails on a string detail,
and the frontend's contract test (`i18n/backendIds.contract.test.ts`) requires every `code` a detail
carries to be in one record of `lib/api/refusalCodes.ts`, and every reason it can be sent with — a
literal, or a `Literal` type when it travels in a variable — to have its sentence in both languages.
A code or reason that scan cannot read fails it until it is listed with what holds it. So the rule is
enforced, not described.
A test (`test_http_details.py`) reads every `HTTPException` in the app and fails on a string detail.
`tests/refusal_sites.py` reads every `{"code": …}` detail from the syntax tree and follows each reason
to its values: a literal, a `Literal` type on the parameter or attribute it travels in, every call of
a helper like `refused` below, a map's values. The frontend's contract test
(`i18n/backendIds.contract.test.ts`) runs it and requires each code in one record of
`lib/api/refusalCodes.ts`, and each reason's sentence in both languages; a code or reason it cannot
follow fails it, unless no screen shows that code. So the rule is enforced, not described.
"""
from fastapi import HTTPException
from app.sysconfig import InvalidSetting, SettingReason
def refused(status: int, reason: str, **params: object) -> HTTPException:
"""The refusal to raise: `raise refused(409, "view_name_taken", name=name)`. `params` travel in
@@ -24,9 +27,9 @@ def refused(status: int, reason: str, **params: object) -> HTTPException:
return HTTPException(status_code=status, detail={"code": "refused", "reason": reason, **params})
# `sysconfig.InvalidSetting.reason` → the refusal it answers as. A dict of literals, so the frontend's
# contract test can read every reason this can produce.
_SETTING_REASONS = {
# `sysconfig.InvalidSetting.reason` → the refusal it answers as. A dict of literals, so the refusal
# scan reads every reason this can produce, and holds its keys to `SettingReason`.
_SETTING_REASONS: dict[SettingReason, str] = {
"not_a_number": "setting_not_a_number",
"too_low": "setting_too_low",
"too_high": "setting_too_high",
@@ -35,6 +38,6 @@ _SETTING_REASONS = {
}
def setting_refused(exc) -> HTTPException:
def setting_refused(exc: InvalidSetting) -> HTTPException:
"""A value the admin entered that `sysconfig.set_value` refused (`exc`: its `InvalidSetting`)."""
return refused(400, _SETTING_REASONS[exc.reason], key=exc.key, **exc.params)
+7 -2
View File
@@ -443,15 +443,20 @@ def _default(s: ConfigSpec):
return getattr(settings, s.default_attr)
# Every reason `set_value` refuses with. `refusal.setting_refused` looks each one up in its map, which
# the refusal scan holds to exactly these keys: a reason the map lacks would answer a 500.
SettingReason = Literal["not_a_number", "too_low", "too_high", "not_a_choice", "no_encryption_key"]
class InvalidSetting(ValueError):
"""A value `set_value` refuses, as a reason the UI translates (`errors.refused.setting_<reason>`)
plus what the sentence needs — the English message is for logs only. A ValueError still, so a
caller that only needs "refused or not" keeps working."""
def __init__(self, key: str, reason: str, **params: object):
def __init__(self, key: str, reason: SettingReason, **params: object):
super().__init__(f"{key}: {reason} {params}")
self.key = key
self.reason = reason
self.reason: SettingReason = reason
self.params = params
+625
View File
@@ -0,0 +1,625 @@
"""Every refusal the backend can answer, read from its syntax tree: each `{"code": …, "reason": …}` detail,
with every reason its code can be sent with.
The frontend's contract (`i18n/backendIds.contract.test.ts`) runs this and requires a sentence for each
code and reason in both languages. A regex over the source read only the shapes it was written for: a
reason after another key, a status passed by name, a second handler with the same text all went past it.
Here a reason is followed to where it comes from, and what cannot be followed is reported, never guessed:
- a literal; both arms of `a if c else b`, every operand of `a or b`;
- a module constant (`urlguard.REASON_MALFORMED`), a local variable's assignments;
- a parameter or attribute typed with a `Literal` (basedpyright holds every value passed in), the
attribute found on the class an `except … as e` catches, or the class a parameter is typed with;
- a `str` parameter of a helper (`refused`, `not_found`): followed to every call of it, its default
included, and a helper passed around as a value is unread;
- a call: its return annotation; a map lookup (`_SETTING_REASONS[exc.reason]`): the map's values, and
its keys must be exactly what the index can be.
Anything else is UNREAD, and the contract fails on it unless no screen shows that code. Stdlib only: it
runs on the host, beside the frontend's tests (`python -I backend/tests/refusal_sites.py` → JSON).
"""
import ast
import json
import pathlib
import sys
from dataclasses import dataclass, field
APP = pathlib.Path(__file__).resolve().parents[1] / "app"
# Files whose `"code"` dicts are data, not refusals: a subtitle track's status payload, TMDB country codes.
NOT_REFUSALS = {
"plex/subtitles.py": "a subtitle track's status payload (its own `code`)",
"plex/tmdb.py": "a production country's ISO code",
}
Node = ast.AST
@dataclass
class Res:
values: set[str] = field(default_factory=set)
unread: list[str] = field(default_factory=list)
def __or__(self, other: "Res") -> "Res":
return Res(self.values | other.values, self.unread + other.unread)
@dataclass
class Module:
path: str # relative to app/, posix
name: str # dotted, `app.…`
tree: ast.Module
parents: dict[Node, Node]
imports: dict[
str, tuple[str, str | None]
] # local name → (module, symbol or None for the module)
top: dict[str, Node] # module-level bindings: def, class, or the assigned value
def _module_name(path: str) -> str:
parts = path.removesuffix(".py").split("/")
if parts[-1] == "__init__":
parts = parts[:-1]
return ".".join(["app", *parts])
def _load(sources: dict[str, str]) -> dict[str, Module]:
mods: dict[str, Module] = {}
for path, src in sources.items():
tree = ast.parse(src)
parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
imports: dict[str, tuple[str, str | None]] = {}
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
for a in node.names:
imports[a.asname or a.name] = (node.module, a.name)
elif isinstance(node, ast.Import):
for a in node.names:
if a.asname:
imports[a.asname] = (a.name, None)
top: dict[str, Node] = {}
for stmt in tree.body:
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
top[stmt.name] = stmt
elif isinstance(stmt, ast.Assign):
for t in stmt.targets:
if isinstance(t, ast.Name):
top[t.id] = stmt
elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
top[stmt.target.id] = stmt
name = _module_name(path)
mods[name] = Module(path, name, tree, parents, imports, top)
return mods
class Scan:
def __init__(self, sources: dict[str, str]):
self.mods = _load(sources)
self.errors: list[str] = []
self._following: set[tuple[str, str, str]] = set()
# --- names ---------------------------------------------------------------------------------------
def lookup(self, m: Module, name: str) -> tuple[Module, Node] | tuple[str, None] | None:
"""What a module-level name is: (its module, its binding), or (a module's name, None)."""
if name in m.top:
return m, m.top[name]
if name in m.imports:
mod, sym = m.imports[name]
if sym is None:
return (mod, None) if mod in self.mods else None
if f"{mod}.{sym}" in self.mods:
return f"{mod}.{sym}", None
if mod in self.mods:
return self.lookup(self.mods[mod], sym)
return None
def lookup_expr(self, m: Module, e: Node) -> tuple[Module, Node] | tuple[str, None] | None:
if isinstance(e, ast.Name):
return self.lookup(m, e.id)
if isinstance(e, ast.Attribute):
base = self.lookup_expr(m, e.value)
if base and base[1] is None and isinstance(base[0], str):
return self.lookup(self.mods[base[0]], e.attr)
return None
def at(self, m: Module, e: Node) -> str:
return f"{m.path}:{getattr(e, 'lineno', '?')}: {ast.unparse(e)}"
def enclosing(self, m: Module, e: Node, kind: type | tuple[type, ...]) -> Node | None:
node = m.parents.get(e)
while node is not None and not isinstance(node, kind):
node = m.parents.get(node)
return node
# --- types ---------------------------------------------------------------------------------------
def ann_values(self, m: Module, ann: Node | None) -> set[str] | None:
"""The values a `Literal` annotation allows, or None when it is not one (`str`, a class, …)."""
if ann is None:
return None
if isinstance(ann, ast.Constant):
if ann.value is None:
return set()
if isinstance(ann.value, str):
return self.ann_values(m, ast.parse(ann.value, mode="eval").body)
return None
if isinstance(ann, ast.Subscript) and getattr(ann.value, "id", None) == "Literal":
items = ann.slice.elts if isinstance(ann.slice, ast.Tuple) else [ann.slice]
if all(isinstance(i, ast.Constant) and isinstance(i.value, str) for i in items):
return {i.value for i in items}
return None
if isinstance(ann, ast.BinOp) and isinstance(ann.op, ast.BitOr):
left, right = self.ann_values(m, ann.left), self.ann_values(m, ann.right)
return None if left is None or right is None else left | right
found = self.lookup_expr(m, ann)
if found and isinstance(found[1], ast.Assign) and isinstance(found[0], Module):
if isinstance(found[1].value, (ast.Subscript, ast.BinOp, ast.Name, ast.Attribute)):
return self.ann_values(found[0], found[1].value)
return None
def classes(self, m: Module, e: Node | None) -> list[tuple[Module, ast.ClassDef]] | None:
if e is None:
return None
if isinstance(e, ast.Tuple):
out = [self.classes(m, x) for x in e.elts]
return None if any(o is None for o in out) else [c for o in out for c in o]
found = self.lookup_expr(m, e)
if found and isinstance(found[0], Module) and isinstance(found[1], ast.ClassDef):
return [(found[0], found[1])]
return None
def attr_values(self, cm: Module, cls: ast.ClassDef, attr: str) -> set[str] | None:
"""An attribute's `Literal` values, from its annotation on the class or `self.attr: T` in a method,
else from a base class."""
for node in ast.walk(cls):
if isinstance(node, ast.AnnAssign):
t = node.target
if (isinstance(t, ast.Name) and t.id == attr and node in cls.body) or (
isinstance(t, ast.Attribute)
and t.attr == attr
and getattr(t.value, "id", None) == "self"
):
return self.ann_values(cm, node.annotation)
for base in cls.bases:
for bm, bc in self.classes(cm, base) or []:
v = self.attr_values(bm, bc, attr)
if v is not None:
return v
return None
# --- reasons -------------------------------------------------------------------------------------
def resolve(self, m: Module, e: Node) -> Res:
if isinstance(e, ast.Constant):
if e.value is None:
return Res()
if isinstance(e.value, str):
return Res({e.value})
if isinstance(e, ast.IfExp):
return self.resolve(m, e.body) | self.resolve(m, e.orelse)
if isinstance(e, ast.BoolOp) and isinstance(e.op, ast.Or):
out = Res()
for v in e.values:
out = out | self.resolve(m, v)
return out
if isinstance(e, ast.Name):
return self.resolve_name(m, e)
if isinstance(e, ast.Attribute):
return self.resolve_attr(m, e)
if isinstance(e, ast.Call):
found = self.lookup_expr(m, e.func)
if (
found
and isinstance(found[0], Module)
and isinstance(found[1], (ast.FunctionDef, ast.AsyncFunctionDef))
):
v = self.ann_values(found[0], found[1].returns)
if v is not None:
return Res(v)
if isinstance(e, ast.Subscript):
return self.resolve_lookup(m, e)
return Res(unread=[self.at(m, e)])
def resolve_name(self, m: Module, e: ast.Name) -> Res:
fn = self.enclosing(m, e, (ast.FunctionDef, ast.AsyncFunctionDef))
while fn is not None:
assert isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef))
if self.caught_as(m, e, e.id) is not None:
return Res(unread=[self.at(m, e)]) # the exception itself, not a reason
params = {a.arg: a for a in [*fn.args.posonlyargs, *fn.args.args, *fn.args.kwonlyargs]}
if e.id in params:
return self.resolve_param(m, fn, params[e.id], e)
if (
fn.args.vararg
and fn.args.vararg.arg == e.id
or fn.args.kwarg
and fn.args.kwarg.arg == e.id
):
return Res(unread=[self.at(m, e)])
assigned = self.assignments(fn, e.id)
if assigned:
out = Res()
for a in assigned:
if (
isinstance(a, ast.AnnAssign)
and (v := self.ann_values(m, a.annotation)) is not None
):
out = out | Res(v)
elif (
isinstance(a, (ast.Assign, ast.AnnAssign))
and a.value is not None
and not any(
isinstance(t, (ast.Tuple, ast.List)) for t in getattr(a, "targets", [])
)
):
out = out | self.resolve(m, a.value)
else:
out = out | Res(unread=[self.at(m, e)])
return out
fn = self.enclosing(m, fn, (ast.FunctionDef, ast.AsyncFunctionDef))
found = self.lookup(m, e.id)
if (
found
and isinstance(found[0], Module)
and isinstance(found[1], (ast.Assign, ast.AnnAssign))
):
stmt = found[1]
if stmt.value is not None and isinstance(stmt.value, ast.Constant):
return self.resolve(found[0], stmt.value)
return Res(unread=[self.at(m, e)])
def assignments(self, fn: Node, name: str) -> list[Node]:
"""The statements that bind `name` in this function's own body (not in a nested one)."""
out: list[Node] = []
stack = list(ast.iter_child_nodes(fn))
while stack:
node = stack.pop()
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)):
continue
if isinstance(node, ast.Assign):
if any(
name in {n.id for n in ast.walk(t) if isinstance(n, ast.Name)}
for t in node.targets
):
out.append(node)
elif isinstance(node, ast.AnnAssign) and getattr(node.target, "id", None) == name:
out.append(node)
elif isinstance(
node, (ast.For, ast.With, ast.AugAssign, ast.NamedExpr, ast.comprehension)
):
targets = [getattr(node, "target", None)] + [
i.optional_vars
for i in getattr(node, "items", [])
if i.optional_vars is not None
]
if any(
t is not None and name in {n.id for n in ast.walk(t) if isinstance(n, ast.Name)}
for t in targets
):
out.append(
node
) # bound by a loop, a `with`, an augmented or a walrus: not followed
stack.extend(ast.iter_child_nodes(node))
return out
def caught_as(self, m: Module, e: Node, name: str) -> ast.ExceptHandler | None:
node = m.parents.get(e)
while node is not None and not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
if isinstance(node, ast.ExceptHandler) and node.name == name:
return node
node = m.parents.get(node)
return None
def resolve_param(self, m: Module, fn: Node, param: ast.arg, e: Node) -> Res:
v = self.ann_values(m, param.annotation)
if v is not None:
return Res(v)
if param.annotation is not None and getattr(param.annotation, "id", None) != "str":
return Res(unread=[self.at(m, e)])
return self.follow(m, fn, param)
def follow(self, m: Module, fn: Node, param: ast.arg) -> Res:
"""A `str` parameter of a helper: what every call of the helper passes it."""
assert isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef))
key = (m.name, fn.name, param.arg)
if key in self._following:
return Res()
if m.top.get(fn.name) is not fn:
return Res(
unread=[
f"{m.path}:{fn.lineno}: {fn.name}({param.arg}) is not a module-level helper"
]
)
self._following.add(key)
positional = [a.arg for a in [*fn.args.posonlyargs, *fn.args.args]]
defaults = dict(
zip(
positional[len(positional) - len(fn.args.defaults) :], fn.args.defaults, strict=True
)
)
defaults |= {
a.arg: d
for a, d in zip(fn.args.kwonlyargs, fn.args.kw_defaults, strict=True)
if d is not None
}
out = Res()
for cm in self.mods.values():
for node in ast.walk(cm.tree):
if not isinstance(node, (ast.Name, ast.Attribute)) or isinstance(
cm.parents.get(node), ast.Attribute
):
continue
found = self.lookup_expr(cm, node)
if not (found and found[0] is m and found[1] is fn):
continue
call = cm.parents.get(node)
if not (isinstance(call, ast.Call) and call.func is node):
out = out | Res(unread=[f"{self.at(cm, node)} (the helper used as a value)"])
continue
kw = {k.arg: k.value for k in call.keywords}
given = param.arg in positional and positional.index(param.arg) < len(call.args)
if param.arg in kw:
out = out | self.resolve(cm, kw[param.arg])
elif any(isinstance(a, ast.Starred) for a in call.args):
out = out | Res(unread=[self.at(cm, call)])
elif given: # given by position, a `**` spread cannot pass it again (a TypeError)
out = out | self.resolve(cm, call.args[positional.index(param.arg)])
elif None in kw:
out = out | Res(unread=[self.at(cm, call)])
elif param.arg in defaults:
out = out | self.resolve(m, defaults[param.arg])
return out
def resolve_attr(self, m: Module, e: ast.Attribute) -> Res:
base = e.value
if isinstance(base, ast.Name):
handler = self.caught_as(m, e, base.id)
fn = self.enclosing(m, e, (ast.FunctionDef, ast.AsyncFunctionDef))
typed = None
if handler is not None:
typed = self.classes(m, handler.type)
elif fn is not None:
params = {
a.arg: a for a in [*fn.args.posonlyargs, *fn.args.args, *fn.args.kwonlyargs]
}
if base.id in params:
typed = self.classes(m, params[base.id].annotation)
if typed is not None:
out = Res()
for cm, cls in typed:
v = self.attr_values(cm, cls, e.attr)
out = out | (
Res(v)
if v is not None
else Res(
unread=[f"{self.at(m, e)} ({cls.name}.{e.attr} has no Literal type)"]
)
)
return out
found = self.lookup_expr(m, e)
if (
found
and isinstance(found[0], Module)
and isinstance(found[1], (ast.Assign, ast.AnnAssign))
):
stmt = found[1]
if isinstance(stmt.value, ast.Constant):
return self.resolve(found[0], stmt.value)
return Res(unread=[self.at(m, e)])
def resolve_lookup(self, m: Module, e: ast.Subscript) -> Res:
found = self.lookup_expr(m, e.value)
if not (
found
and isinstance(found[0], Module)
and isinstance(found[1], (ast.Assign, ast.AnnAssign))
):
return Res(unread=[self.at(m, e)])
table = found[1].value
if not isinstance(table, ast.Dict) or not all(
isinstance(k, ast.Constant) and isinstance(k.value, str) for k in table.keys
):
return Res(unread=[self.at(m, e)])
out = Res()
for v in table.values:
out = out | self.resolve(found[0], v)
index = self.resolve(m, e.slice)
keys = {k.value for k in table.keys}
if index.unread or index.values != keys:
self.errors.append(
f"{self.at(m, e)}: the map's keys {sorted(keys)} are not what its index can be "
f"{sorted(index.values)}{' + unread ' + str(index.unread) if index.unread else ''}"
)
return out
# --- sites ---------------------------------------------------------------------------------------
def spread_is_safe(self, m: Module, e: Node, src: Node) -> bool:
"""A `**x` in a detail, or `detail |= {…}` from it, cannot carry a reason when `x` is the `**kwargs`
of a function that takes `reason` by name: Python binds that one to the parameter."""
if isinstance(src, ast.Attribute):
return self.spread_is_safe_attr(m, e, src)
fn = self.enclosing(m, e, (ast.FunctionDef, ast.AsyncFunctionDef))
if fn is None or fn.args.kwarg is None:
return False
names = {a.arg for a in [*fn.args.posonlyargs, *fn.args.args, *fn.args.kwonlyargs]}
kwarg = fn.args.kwarg.arg
if isinstance(src, ast.Name):
return src.id == kwarg and "reason" in names
if isinstance(src, ast.DictComp):
it = src.generators[0].iter
return (
isinstance(it, ast.Call)
and isinstance(it.func, ast.Attribute)
and it.func.attr == "items"
and getattr(it.func.value, "id", None) == kwarg
and "reason" in names
)
return False
def spread_is_safe_attr(self, m: Module, e: Node, src: ast.Attribute) -> bool:
"""`e.params` of a caught exception whose `__init__(self, reason, **params)` keeps `self.params = params`."""
if not isinstance(src.value, ast.Name):
return False
handler = self.caught_as(m, e, src.value.id)
typed = self.classes(m, handler.type) if handler else None
return bool(typed) and all(self.keeps_kwargs(cls, src.attr) for _, cls in typed or [])
def keeps_kwargs(self, cls: ast.ClassDef, attr: str) -> bool:
init = next(
(s for s in cls.body if isinstance(s, ast.FunctionDef) and s.name == "__init__"), None
)
if init is None or init.args.kwarg is None:
return False
names = {a.arg for a in [*init.args.posonlyargs, *init.args.args, *init.args.kwonlyargs]}
kwarg = init.args.kwarg.arg
return "reason" in names and any(
isinstance(n, ast.Assign)
and any(
isinstance(t, ast.Attribute)
and t.attr == attr
and getattr(t.value, "id", None) == "self"
for t in n.targets
)
and getattr(n.value, "id", None) == kwarg
for n in ast.walk(init)
)
def entries(self, d: Node) -> list[tuple[str | None, Node]]:
if isinstance(d, ast.Dict):
return [
(k.value if isinstance(k, ast.Constant) else None if k is None else "?", v)
for k, v in zip(d.keys, d.values, strict=True)
]
assert isinstance(d, ast.Call)
return [(k.arg, k.value) for k in d.keywords]
def site(self, m: Module, d: Node, out: dict) -> None:
items = self.entries(d)
code_expr = next(v for k, v in items if k == "code")
code = self.resolve(m, code_expr)
if code.unread or len(code.values) != 1:
out["unreadCodes"].append(self.at(m, code_expr))
return
entry = out["codes"].setdefault(
next(iter(code.values)), {"reasons": set(), "unread": [], "sites": 0, "reasonless": []}
)
entry["sites"] += 1
res, has_reason = Res(), False
for k, v in items:
if k == "reason":
res, has_reason = res | self.resolve(m, v), True
elif k is None and not self.spread_is_safe(m, d, v):
res = res | Res(unread=[f"{self.at(m, v)} (a spread that may carry a reason)"])
res, more = self.mutations(m, d, res)
has_reason = has_reason or more
if not has_reason:
entry["reasonless"].append(f"{m.path}:{d.lineno}")
entry["reasons"] |= res.values
entry["unread"] += res.unread
def mutations(self, m: Module, d: Node, res: Res) -> tuple[Res, bool]:
"""`detail |= {…}`, `detail["reason"] = …`, `detail.update(…)` on the variable a detail is put in."""
stmt = m.parents.get(d)
if not (isinstance(stmt, (ast.Assign, ast.AnnAssign)) and stmt.value is d):
return res, False
target = stmt.targets[0] if isinstance(stmt, ast.Assign) else stmt.target
fn = self.enclosing(m, d, (ast.FunctionDef, ast.AsyncFunctionDef))
if not isinstance(target, ast.Name) or fn is None:
return res, False
found = False
for node in ast.walk(fn):
if isinstance(node, ast.AugAssign) and getattr(node.target, "id", None) == target.id:
v = node.value
if isinstance(v, ast.Dict):
for k, val in self.entries(v):
if k == "reason":
res, found = res | self.resolve(m, val), True
elif k is None and not self.spread_is_safe(m, node, val):
res = res | Res(
unread=[f"{self.at(m, val)} (a spread that may carry a reason)"]
)
elif not self.spread_is_safe(m, node, v):
res = res | Res(unread=[f"{self.at(m, node)} (may carry a reason)"])
elif (
isinstance(node, ast.Subscript)
and getattr(node.value, "id", None) == target.id
and isinstance(node.ctx, ast.Store)
):
if isinstance(node.slice, ast.Constant) and node.slice.value != "reason":
continue
assign = m.parents.get(node)
if isinstance(node.slice, ast.Constant) and isinstance(assign, ast.Assign):
res, found = res | self.resolve(m, assign.value), True
else:
res = res | Res(unread=[f"{self.at(m, node)} (may carry a reason)"])
elif (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and getattr(node.func.value, "id", None) == target.id
and node.func.attr in {"update", "setdefault"}
):
res = res | Res(unread=[f"{self.at(m, node)} (may carry a reason)"])
return res, found
def run(self) -> dict:
out: dict = {
"codes": {},
"unreadCodes": [],
"errors": self.errors,
"excluded": {},
"aliases": {},
"notRefusals": sorted(NOT_REFUSALS),
}
for m in self.mods.values():
for name, stmt in m.top.items():
if (
isinstance(stmt, ast.Assign)
and isinstance(stmt.value, ast.Subscript)
and (v := self.ann_values(m, stmt.value))
):
out["aliases"].setdefault(m.path, {})[name] = sorted(v)
for node in ast.walk(m.tree):
is_dict = isinstance(node, ast.Dict) and any(
isinstance(k, ast.Constant) and k.value == "code" for k in node.keys
)
is_call = (
isinstance(node, ast.Call)
and getattr(node.func, "id", None) == "dict"
and any(k.arg == "code" for k in node.keywords)
)
if not (is_dict or is_call):
continue
if m.path in NOT_REFUSALS:
out["excluded"][m.path] = out["excluded"].get(m.path, 0) + 1
continue
self.site(m, node, out)
for entry in out["codes"].values():
entry["reasons"] = sorted(entry["reasons"])
return out
def scan(sources: dict[str, str]) -> dict:
return Scan(sources).run()
def scan_app(app: pathlib.Path = APP) -> dict:
return scan(
{
p.relative_to(app).as_posix(): p.read_text(encoding="utf-8")
for p in sorted(app.rglob("*.py"))
if "__pycache__" not in p.parts
}
)
if __name__ == "__main__":
json.dump(
scan_app(pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else APP),
sys.stdout,
indent=1,
sort_keys=True,
)
+278
View File
@@ -0,0 +1,278 @@
"""The refusal scan (`refusal_sites.py`) on small sources: every shape a reason travels in is followed to
its values or reported unread. A scan that silently drops a shape is a guard green over nothing, so each
shape that once went past the regex it replaced is here. Pure: no app import, no database."""
import textwrap
from tests.refusal_sites import scan, scan_app
def _scan(**files: str) -> dict:
return scan(
{f"{name.replace('__', '/')}.py": textwrap.dedent(src) for name, src in files.items()}
)
def _code(result: dict, code: str) -> dict:
return result["codes"][code]
REFUSAL = """
from fastapi import HTTPException
def refused(status: int, reason: str, **params: object) -> HTTPException:
return HTTPException(status_code=status, detail={"code": "refused", "reason": reason, **params})
"""
def test_a_literal_wherever_it_sits_in_the_dict():
r = _scan(
a="""
def f(x):
raise HTTPException(409, {"code": "clip", "limit_s": 5, "reason": "too_long"})
def g(x):
raise HTTPException(409, {"reason": "before", "code": "clip"})
def h(x):
raise HTTPException(409, {
"code": "clip", # a comment between
"reason": "after_comment",
})
def i(x):
raise HTTPException(409, detail=dict(code="clip", reason="by_call"))
"""
)
assert _code(r, "clip")["reasons"] == ["after_comment", "before", "by_call", "too_long"]
assert _code(r, "clip")["unread"] == []
def test_every_arm_of_a_conditional_and_every_operand_of_or():
r = _scan(
a="""
def f(x, e):
raise HTTPException(409, {"code": "edit", "reason": "a" if x else "b"})
def g(x):
raise HTTPException(409, {"code": "edit", "reason": x.why or "c"})
"""
)
assert _code(r, "edit")["reasons"] == ["a", "b", "c"]
assert [u.split(": ", 1)[1] for u in _code(r, "edit")["unread"]] == ["x.why"]
def test_a_helper_is_followed_to_every_call_however_it_is_written():
r = _scan(
refusal=REFUSAL,
routes__a="""
from http import HTTPStatus
from app import refusal
from app.refusal import refused
def f(status):
raise refused(status, "by_name_status")
def g():
raise refused(HTTPStatus.CONFLICT, reason="by_keyword")
def h():
raise refusal.refused(409, "qualified")
def i(x):
raise refused(409, "a" if x else "second_arm")
""",
)
assert _code(r, "refused")["reasons"] == [
"a",
"by_keyword",
"by_name_status",
"qualified",
"second_arm",
]
assert _code(r, "refused")["unread"] == []
def test_a_helper_passed_as_a_value_or_spread_into_is_unread():
r = _scan(
refusal=REFUSAL,
routes__a="""
from app.refusal import refused
def f(args):
pick = refused
raise refused(*args)
""",
)
unread = _code(r, "refused")["unread"]
assert any("the helper used as a value" in u for u in unread)
assert any("refused(*args)" in u for u in unread)
def test_a_default_is_what_a_call_that_omits_it_sends():
r = _scan(
routes___common="""
def not_found(what: str = "generic"):
return HTTPException(404, {"code": "not_found", "reason": what})
def get_or_404(db, id_, *, what: str = "generic"):
raise not_found(what)
""",
routes__a="""
from app.routes._common import get_or_404, not_found
def f(db):
get_or_404(db, 1, what="user")
get_or_404(db, 2)
raise not_found("debug_session")
""",
)
assert _code(r, "not_found")["reasons"] == ["debug_session", "generic", "user"]
def test_a_caught_exception_answers_with_its_attribute_type_every_class_it_catches():
r = _scan(
errors="""
from typing import Literal
ClipReason = Literal["bad_range", "too_short"]
EditReason = Literal["not_media"]
class ClipError(Exception):
def __init__(self, reason: ClipReason, **params: float):
self.reason: ClipReason = reason
self.params = params
class EditError(Exception):
def __init__(self, reason: EditReason):
self.reason: EditReason = reason
class Untyped(Exception):
def __init__(self, reason: str):
self.reason = reason
class Narrower(ClipError):
pass
""",
routes__a="""
from app import errors
def f():
try:
pass
except (errors.ClipError, errors.EditError) as e:
raise HTTPException(422, {"code": "clip", "reason": e.reason, **e.params})
except errors.Untyped as e:
raise HTTPException(422, {"code": "clip", "reason": e.reason})
except errors.Narrower as e:
raise HTTPException(422, {"code": "edit", "reason": e.reason})
""",
)
assert _code(r, "clip")["reasons"] == ["bad_range", "not_media", "too_short"]
unread = _code(r, "clip")["unread"]
assert any("Untyped.reason has no Literal type" in u for u in unread)
# EditError keeps no `**params`, so its `e.params` could be anything
assert any("e.params" in u for u in unread)
assert _code(r, "edit")["reasons"] == ["bad_range", "too_short"]
def test_a_typed_parameter_is_its_literal_and_an_untyped_one_is_unread_when_never_called():
r = _scan(
a="""
from typing import Literal
Why = Literal["one", "two"]
def typed(reason: Why):
return HTTPException(409, {"code": "playlist", "reason": reason})
class C:
def method(self, reason: str):
return HTTPException(409, {"code": "playlist", "reason": reason})
"""
)
assert _code(r, "playlist")["reasons"] == ["one", "two"]
assert any("is not a module-level helper" in u for u in _code(r, "playlist")["unread"])
def test_a_value_the_scan_cannot_follow_is_unread():
r = _scan(
a="""
def f(svc):
hit, code = svc.fetch()
raise HTTPException(422, {"code": "subtitle", "reason": code})
def g(svc):
raise HTTPException(422, {"code": "subtitle", "reason": svc.reason()})
def h(e):
raise HTTPException(422, {"code": "subtitle", "reason": e.reason})
"""
)
assert len(_code(r, "subtitle")["unread"]) == 3
assert _code(r, "subtitle")["reasons"] == []
def test_a_map_lookup_is_its_values_and_its_keys_must_be_what_the_index_can_be():
src = """
from typing import Literal
SettingReason = Literal["too_low", "too_high"]
class InvalidSetting(ValueError):
def __init__(self, key: str, reason: SettingReason, **params: object):
self.reason: SettingReason = reason
self.params = params
_MAP: dict[SettingReason, str] = {{{keys}}}
def refused(status: int, reason: str, **params: object):
return HTTPException(status, {{"code": "refused", "reason": reason, **params}})
def setting_refused(exc: InvalidSetting):
return refused(400, _MAP[exc.reason], **exc.params)
"""
whole = _scan(a=src.format(keys='"too_low": "setting_too_low", "too_high": "setting_too_high"'))
assert _code(whole, "refused")["reasons"] == ["setting_too_high", "setting_too_low"]
assert whole["errors"] == []
short = _scan(a=src.format(keys='"too_low": "setting_too_low"'))
assert len(short["errors"]) == 1 and "not what its index can be" in short["errors"][0]
def test_a_detail_built_up_in_steps_is_read_through_its_variable():
r = _scan(
a="""
def f(status: int, reason: str, user=None, **extra):
detail: dict = {"code": "login", "reason": "invalid"}
if user is not None:
detail |= {"email": user.email, "reason": "suspended"}
detail |= {k: v for k, v in extra.items() if v is not None}
return HTTPException(status, detail)
def g(more):
detail = {"code": "login"}
detail["reason"] = "late"
detail.update(more)
detail |= more
return HTTPException(401, detail)
"""
)
entry = _code(r, "login")
assert entry["reasons"] == ["invalid", "late", "suspended"]
assert sum("may carry a reason" in u for u in entry["unread"]) == 2
def test_a_code_must_be_one_the_scan_can_read():
r = _scan(
a="""
BUSY = "busy"
def f(code):
raise HTTPException(409, {"code": BUSY})
def g(code):
raise HTTPException(409, {"code": code})
"""
)
assert "busy" in r["codes"] and r["codes"]["busy"]["reasonless"]
assert len(r["unreadCodes"]) == 1 and r["unreadCodes"][0].endswith(": code")
def test_the_data_files_are_excluded_by_name_only():
r = _scan(plex__tmdb='x = {"code": "HU", "name": "Hungary"}', plex__other='y = {"code": "HU"}')
assert r["excluded"] == {"plex/tmdb.py": 1}
assert "HU" in r["codes"]
def test_the_app_scans():
"""The real tree, on the interpreter the backend runs: every refusal it holds is read (the
frontend's contract judges what is read)."""
result = scan_app()
assert {"refused", "not_found", "login", "clip"} <= set(result["codes"])
assert result["errors"] == []
+2 -1
View File
@@ -144,7 +144,8 @@ npm run typecheck # tsc x3 (app / node / e2e)
npm run lint # eslint (typed rules included; warnings fail)
npm run format:check # prettier
npm run knip # unused files/exports/deps
npm run test # vitest (incl. the compiler and gate-register contracts)
npm run test # vitest (incl. the compiler and gate-register contracts; the refusal contract runs
# the host's `python` on backend/tests/refusal_sites.py, stdlib only)
npx vite build # the production bundle (devOnlyGuard fails it on dev-only code, chunkGuard on a lazy page loaded at boot)
npm audit --omit=dev --audit-level=high # what ships to the browser, at high or above
```
+90 -234
View File
@@ -1,4 +1,6 @@
import { execFileSync } from "node:child_process";
import { readdirSync, readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { RUN_STATUSES } from "../lib/activityUrlState";
@@ -27,6 +29,25 @@ import { vocabKey } from "../lib/vocab";
const backend = (file: string) =>
readFileSync(new URL(`../../../backend/app/${file}`, import.meta.url), "utf-8");
// Every refusal the backend can answer, read from its syntax tree by `backend/tests/refusal_sites.py`
// (stdlib `ast`, on the host's Python, as the ruff lane is): each code with every reason it can be sent
// with — a reason in a variable followed to its `Literal` type, its helper's every call, its map — and
// what it could not follow, which fails below. It also hands back every `Literal` alias it read.
type RefusalScan = {
codes: Record<string, { reasons: string[]; unread: string[]; reasonless: string[] }>;
unreadCodes: string[];
errors: string[];
notRefusals: string[];
excluded: Record<string, number>;
aliases: Record<string, Record<string, string[]>>;
};
const scan = JSON.parse(
execFileSync("python", ["-I", "backend/tests/refusal_sites.py"], {
cwd: fileURLToPath(new URL("../../../", import.meta.url)),
encoding: "utf-8",
}),
) as RefusalScan;
const mods = import.meta.glob("./locales/*/*.json", { eager: true }) as Record<
string,
{ default: Record<string, unknown> }
@@ -54,12 +75,11 @@ const keysOf = (lang: string): Set<string> => {
const specs = [
...backend("sysconfig.py").matchAll(/ConfigSpec\(\s*"(\w+)",\s*"(\w+)",\s*"(\w+)",\s*"(\w+)"/g),
].map((m) => ({ key: m[1]!, type: m[2]!, group: m[3]!, attr: m[4]! }));
// A choice key's options, as config.py spells them: `<attr>: <Alias> = …` → `<Alias> = Literal[…]`.
// A choice key's options, as config.py spells them: `<attr>: <Alias> = …`, the alias's values from the scan.
const configPy = backend("config.py");
const choicesOf = (attr: string): string[] => {
const alias = configPy.match(new RegExp(`^\\s+${attr}: (\\w+) =`, "m"))?.[1];
const body = alias && configPy.match(new RegExp(`^${alias} = Literal\\[([^\\]]*)\\]`, "m"))?.[1];
return [...(body ?? "").matchAll(/"(\w+)"/g)].map((m) => m[1]!);
return (alias && scan.aliases["config.py"]?.[alias]) || [];
};
const choiceIds = specs
.filter((s) => s.type === "choice")
@@ -90,16 +110,10 @@ const runEnum = (name: string) =>
const runStatuses = runEnum("RunStatus");
const runTriggers = runEnum("RunTrigger");
const quotaActions = enumValues("quota.py");
// The admin actions' refusal reasons → `errors.admin.<reason>`. The codes are the refusal records
// under that prefix (lib/api/refusalCodes), so a code recorded there is scanned here too; a detail
// dict ruff wraps over several lines still matches.
// The admin actions' refusals → `errors.admin.<reason>`: the codes recorded under that prefix.
const adminCodes = Object.entries(REFUSAL_FAMILIES)
.filter(([, f]) => f.prefix === "errors.admin")
.map(([code]) => code);
const adminRefusals = [
...backend("routes/admin.py").matchAll(/\{\s*"code":\s*"(\w+)",\s*"reason":\s*"(\w+)"/g),
].filter((m) => adminCodes.includes(m[1]!));
const adminReasons = [...new Set(adminRefusals.map((m) => m[2]!))];
// The download failure catalog's host-specific entries → `downloads.errors.fail.catalog.<detail>`.
// A code holds a dot (`facebook.login_wall`), so the reverse check strips the prefix and the part
// rather than splitting on dots.
@@ -120,66 +134,6 @@ const failCategories = [
]),
];
// What a structured 404 says is gone → `errors.notFound.<what>`: every `not_found("x")` and every
// `owned_or_404(..., what="x")` / `get_or_404(..., what="x")` under routes/, read from the sources.
const routeFiles = (dir = "routes"): string[] =>
readdirSync(new URL(`../../../backend/app/${dir}/`, import.meta.url), {
withFileTypes: true,
}).flatMap((e) =>
e.isDirectory()
? routeFiles(`${dir}/${e.name}`)
: e.name.endsWith(".py")
? [`${dir}/${e.name}`]
: [],
);
const routeSources = (): string[] => routeFiles().map(backend);
// The words a job's result gives for not doing its work → `scheduler.skip.<vocabKey(word)>` (the Scheduler
// page, the run notice and the audit row name them; lib/jobResult). Read as the `"skipped": "…"` /
// `"reason": "…"` literals of the non-HTTP backend: under routes/ (and main.py) a `reason` is a refusal
// code, which `errors.*` names. A word from an exception's text is not a literal and is not covered.
const appFiles = (dir: string): string[] =>
readdirSync(new URL(`../../../backend/app/${dir}`, import.meta.url), {
withFileTypes: true,
}).flatMap((e) =>
e.isDirectory()
? e.name === "routes" || e.name === "__pycache__"
? []
: appFiles(`${dir}${e.name}/`)
: e.name.endsWith(".py") && `${dir}${e.name}` !== "main.py"
? [`${dir}${e.name}`]
: [],
);
const skipWords = [
...new Set(
appFiles("").flatMap((f) =>
[...backend(f).matchAll(/"(?:skipped|reason)": "([^"]+)"/g)].map((m) => m[1]!),
),
),
];
// A target the scan cannot read — `what=SOME_CONSTANT` — is a 404 no locale is checked for (one hid
// `debug_session`). Only the helpers' own pass-through in `_common.py` may name it by variable.
const opaqueWhats = routeFiles()
.filter((f) => !f.endsWith("/_common.py"))
.flatMap((f) =>
[...backend(f).matchAll(/not_found\((?!"|\))[^)]*\)|what=(?!")\S+/g)].map(
(m) => `${f}: ${m[0]}`,
),
);
const goneWhats = [
...new Set(
routeSources().flatMap((src) =>
[...src.matchAll(/not_found\("(\w+)"\)|what="(\w+)"/g)].map((m) => (m[1] ?? m[2])!),
),
),
];
// A backend `Name = Literal["a", "b"]` alias, by name: the reasons a typed parameter accepts, which
// basedpyright then holds at every call site — a reason passed by variable included.
const literalOf = (file: string, name: string) =>
[
...(
backend(file).match(new RegExp(`^${name} = Literal\\[([^\\]]*)\\]`, "m"))?.[1] ?? ""
).matchAll(/"(\w+)"/g),
].map((m) => m[1]!);
// Every backend source, by its path under app/.
const appPaths = (dir = ""): string[] =>
readdirSync(new URL(`../../../backend/app/${dir}`, import.meta.url), {
@@ -193,114 +147,32 @@ const appPaths = (dir = ""): string[] =>
? [`${dir}${e.name}`]
: [],
);
const appSources = (): string[] => appPaths().map(backend);
// Every generic refusal → `errors.refused.<reason>`: each `refused(<status>, "<reason>")` anywhere in
// the backend (not `_refused`, a router's own family), the values of the settings map, which
// `setting_refused` picks from at runtime, and the folder rules' `FolderReason`, answered with
// `refused(409, e.reason)`.
const refusedReasons = [
...new Set([
...appSources().flatMap((src) =>
[...src.matchAll(/(?<![\w.])refused\(\s*\d+,\s*"(\w+)"/g)].map((m) => m[1]!),
),
...literalOf("downloads/folders.py", "FolderReason"),
...[
...(backend("refusal.py").split("_SETTING_REASONS = {")[1]?.split("}")[0] ?? "").matchAll(
/:\s*"(\w+)"/g,
),
].map((m) => m[1]!),
]),
];
// A `refused(…)` whose reason the scan above cannot read. Only two may: a folder rule's `e.reason`
// (the nearest `except` above it catches `FolderError`, typed `FolderReason`) and the settings map.
const opaqueRefused = appPaths().flatMap((f) => {
const src = backend(f);
return [...src.matchAll(/(?<![\w.])refused\(\s*\d+,\s*(?![\s"])([^,)]+)/g)]
.filter((m) => {
const arg = m[1]!.trim();
if (f === "refusal.py" && arg.startsWith("_SETTING_REASONS[")) return false;
const caught = [...src.slice(0, m.index).matchAll(/except ([^\n:]*):/g)].pop()?.[1] ?? "";
return !(arg === "e.reason" && /FolderError as e$/.test(caught));
})
.map((m) => `${f}: ${m[0]}`);
});
const youtubeReasons = literalOf("youtube/client.py", "YouTubeRefusal");
const clipReasons = literalOf("downloads/clip.py", "ClipReason");
// Every `"code": …` a backend dict carries, with the `"reason": …` that follows it. Two modules use a
// `code` key for something that is not a refusal; everything else is one.
const NOT_REFUSALS: Record<string, string> = {
"plex/subtitles.py":
"the player's subtitle status vocabulary (`/status`), read by the player itself",
"plex/tmdb.py": "a production country's ISO code",
};
const detailSites = appPaths()
.filter((f) => !(f in NOT_REFUSALS))
.flatMap((f) =>
[
...backend(f).matchAll(
/"code":\s*(?:"(\w+)"|([^\s,}]+))(?:,\s*"reason":\s*(?:"(\w+)"|([^\s,}]+)))?/g,
),
].map((m) => ({ f, code: m[1], codeExpr: m[2], reason: m[3], reasonExpr: m[4] })),
);
const backendCodes = [...new Set(detailSites.flatMap((s) => (s.code ? [s.code] : [])))];
// A code the scan cannot read (`"code": SOME_CONSTANT`) is a refusal no record is checked for.
const opaqueCodes = detailSites.filter((s) => !s.code).map((s) => `${s.f}: "code": ${s.codeExpr}`);
// A reason passed by variable, each pinned to what holds it, so a new one has to say how it is held.
const OPAQUE_REASONS: Record<string, string> = {
"refusal.py: refused ← reason": "a literal at each refused() call, else opaqueRefused",
"routes/auth.py: login ← reason": "_login_refused's LoginRefusal",
"routes/downloads.py: url_blocked ← urlguard.REASON_MALFORMED": "UrlBlockedReason",
"routes/downloads.py: url_blocked ← e.reason": "BlockedUrl's UrlBlockedReason",
"routes/downloads.py: cookies ← e.reason": "InvalidCookies' CookieReason",
"routes/downloads.py: edit ← e.reason": "EditError's EditReason",
"routes/downloads.py: clip ← e.reason": "ClipError's ClipReason",
"routes/files.py: quota ← e.reason": "QuotaExceeded's QuotaReason",
"routes/playlists.py: playlist ← reason": "_refused's PlaylistRefusal",
"routes/playlists.py: playlist_move ← e.reason": "MoveRefused's MoveReason",
"routes/plex/playback.py: subtitle ← code": "REFUSAL_UNSHOWN: no screen reads it",
"routes/_common.py: not_found ← what": "opaqueWhats",
"routes/_common.py: youtube ← reason": "YouTubeRefusal",
};
const opaqueReasons = [
// The words a job's result gives for not doing its work → `scheduler.skip.<vocabKey(word)>` (the Scheduler
// page, the run notice and the audit row name them; lib/jobResult). Read as the `"skipped": "…"` /
// `"reason": "…"` literals of the non-HTTP backend: under routes/ (and main.py) a `reason` is a refusal
// code, which `errors.*` names. A word from an exception's text is not a literal and is not covered.
const skipWords = [
...new Set(
detailSites
.filter((s) => s.code && s.reasonExpr)
.map((s) => `${s.f}: ${s.code} ← ${s.reasonExpr}`),
appPaths()
.filter((f) => !f.startsWith("routes/") && f !== "main.py")
.flatMap((f) =>
[...backend(f).matchAll(/"(?:skipped|reason)": "([^"]+)"/g)].map((m) => m[1]!),
),
),
];
// A reason written as a literal in a detail dict, wherever that dict is built.
const literalReasons = (code: string) =>
detailSites.filter((s) => s.code === code && s.reason).map((s) => s.reason!);
// What each refusal code can carry as its reason: the Literal its typed parameter accepts, or the
// call's own literal — and every literal any detail dict anywhere gives it.
const reasonsOf: Record<string, string[]> = Object.fromEntries(
Object.entries({
refused: refusedReasons,
youtube: youtubeReasons,
not_found: goneWhats,
clip: clipReasons,
...Object.fromEntries(adminCodes.map((c) => [c, []])),
playlist: literalOf("routes/playlists.py", "PlaylistRefusal"),
playlist_move: literalOf("reader/service.py", "MoveReason"),
edit: literalOf("downloads/service.py", "EditReason"),
cookies: literalOf("downloads/credentials.py", "CookieReason"),
url_blocked: literalOf("downloads/urlguard.py", "UrlBlockedReason"),
login: literalOf("routes/auth.py", "LoginRefusal"),
quota: literalOf("downloads/quota.py", "QuotaReason"),
reset: [],
collection: [],
} as Record<string, string[]>).map(([code, rs]) => [
code,
[...new Set([...rs, ...literalReasons(code)])],
]),
);
const backendCodes = Object.keys(scan.codes);
const recordedCodes = [
...Object.keys(REFUSAL_FAMILIES),
...Object.keys(REFUSAL_BY_REASON),
...Object.keys(REFUSAL_FIXED),
...Object.keys(REFUSAL_UNSHOWN),
];
// The codes whose reason picks the sentence, and what each can carry as its reason.
const reasoned = [...Object.keys(REFUSAL_FAMILIES), ...Object.keys(REFUSAL_BY_REASON)];
const reasonsOf: Record<string, string[]> = Object.fromEntries(
reasoned.map((code) => [code, scan.codes[code]?.reasons ?? []]),
);
// The closed English vocabularies the backend WRITES into data, named through `vocab.<group>`
// (lib/vocab) by their letters-and-digits key. A word the backend adds shows in English until named —
@@ -363,10 +235,11 @@ describe("backend ids have their names in both languages", () => {
expect(runTriggers).toEqual(expect.arrayContaining(["scheduled", "ad_hoc"]));
expect(adminCodes).toEqual(expect.arrayContaining(["activate", "resend"]));
expect(failDetails).toEqual(expect.arrayContaining(["facebook.login_wall"]));
expect(goneWhats).toEqual(expect.arrayContaining(["playlist", "download", "job"]));
expect(opaqueWhats).toEqual([]);
expect(youtubeReasons).toEqual(["quota", "reconnect", "unavailable", "gone"]);
expect(clipReasons).toEqual(expect.arrayContaining(["too_long", "past_end", "bad_crop"]));
expect(reasonsOf.not_found).toEqual(
expect.arrayContaining(["playlist", "download", "job", "user", "debug_session"]),
);
expect(reasonsOf.youtube).toEqual(["gone", "quota", "reconnect", "unavailable"]);
expect(reasonsOf.clip).toEqual(expect.arrayContaining(["too_long", "past_end", "bad_crop"]));
expect(crewJobs).toEqual(
expect.arrayContaining(["Director", "Creator", "Director of Photography"]),
);
@@ -382,11 +255,14 @@ describe("backend ids have their names in both languages", () => {
"no service user",
]),
);
expect(refusedReasons).toEqual(
expect.arrayContaining(["weak_password", "setting_too_low", "plex_write_failed"]),
expect(reasonsOf.refused).toEqual(
expect.arrayContaining([
"weak_password",
"setting_too_low",
"plex_write_failed",
"file_folder_gone",
]),
);
// Every code the frontend localizes is raised somewhere the scan can see.
expect(adminCodes.filter((c) => !adminRefusals.some((m) => m[1] === c))).toEqual([]);
});
it("every refusal code the backend answers is said by exactly one record, and no record names another", () => {
@@ -397,12 +273,19 @@ describe("backend ids have their names in both languages", () => {
expect(recordedCodes.filter((c) => !backendCodes.includes(c))).toEqual([]);
});
it("every refusal code and reason is one the scans can read, or says what holds it", () => {
// The modules set aside as not refusals still exist and still use the key.
for (const f of Object.keys(NOT_REFUSALS)) expect(backend(f), f).toContain('"code":');
expect(opaqueCodes).toEqual([]);
expect(opaqueRefused).toEqual([]);
expect(opaqueReasons.sort()).toEqual(Object.keys(OPAQUE_REASONS).sort());
it("every refusal code and reason is one the scan can follow", () => {
// The modules set aside as not refusals still use the key.
expect(Object.keys(scan.excluded).sort()).toEqual(scan.notRefusals);
expect(scan.unreadCodes).toEqual([]);
expect(scan.errors).toEqual([]);
// A reason it could not follow is a sentence nobody is asked for, unless no screen says that code.
expect(
Object.entries(scan.codes)
.filter(([code]) => !(code in REFUSAL_UNSHOWN))
.flatMap(([, c]) => c.unread),
).toEqual([]);
// A code whose reason picks the sentence, sent without one, says only its fallback.
expect(reasoned.flatMap((code) => scan.codes[code]?.reasonless ?? [])).toEqual([]);
});
it("read every refusal's reasons (a scan that matches nothing is green over nothing)", () => {
@@ -533,33 +416,6 @@ describe("backend ids have their names in both languages", () => {
expect([...named].filter((d) => !failDetails.includes(d))).toEqual([]);
});
it(`${lang}: every "that is gone" a 404 names, and every YouTube refusal`, () => {
expect(missing(goneWhats.map((w) => `errors.notFound.${w}`))).toEqual([]);
expect(
[...idsUnder(keys, "errors.notFound")].filter(
(w) => w !== "generic" && !goneWhats.includes(w),
),
).toEqual([]);
expect(missing(youtubeReasons.map((r) => `errors.youtube.${r}`))).toEqual([]);
expect(
[...idsUnder(keys, "errors.youtube")].filter((r) => !youtubeReasons.includes(r)),
).toEqual([]);
});
it(`${lang}: every generic refusal the backend can answer, and none it cannot`, () => {
expect(missing(refusedReasons.map((r) => `errors.refused.${r}`))).toEqual([]);
expect(
[...idsUnder(keys, "errors.refused")].filter((r) => !refusedReasons.includes(r)),
).toEqual([]);
});
it(`${lang}: every reason a clip is refused, and none it cannot be`, () => {
expect(missing(clipReasons.map((r) => `clip.errors.${r}`))).toEqual([]);
expect([...idsUnder(keys, "clip.errors")].filter((r) => !clipReasons.includes(r))).toEqual(
[],
);
});
it(`${lang}: every crew job, auto-tag and built-in profile the backend writes, and no other`, () => {
for (const [group, words] of Object.entries(VOCAB)) {
const wanted = words.map((w) => `vocab.${group}.${vocabKey(w)}`);
@@ -588,32 +444,32 @@ describe("backend ids have their names in both languages", () => {
});
it(`${lang}: no sentence for a refusal reason the backend cannot send`, () => {
// A family whose prefix is its own (the shared `errors.admin` is checked below). A suffixed
// family shares its prefix with more than its refusals — the sign-in screens' `gate`,
// `addressLabel` — so only an id with the refusal's own key counts as a reason there.
// By prefix: the admin actions share `errors.admin`. A suffixed family shares its prefix with more
// than its refusals — the sign-in screens' `gate`, `addressLabel` — so only an id with the
// refusal's own key counts as a reason there.
const byPrefix = new Map<
string,
{ suffix: string; reasons: Set<string>; spare: Set<string> }
>();
for (const [code, f] of Object.entries(REFUSAL_FAMILIES)) {
if (f.prefix === "errors.admin") continue;
const suffix = "suffix" in f ? f.suffix : "";
const fallbackId = f.fallback.startsWith(`${f.prefix}.`)
? f.fallback.slice(f.prefix.length + 1)
: null;
expect(
[...idsUnder(keys, f.prefix)].filter(
(r) =>
keys.has(`${f.prefix}.${r}${suffix}`) &&
r !== fallbackId &&
!reasonsOf[code]!.includes(r),
),
code,
).toEqual([]);
const p = byPrefix.get(f.prefix) ?? {
suffix: "suffix" in f ? f.suffix : "",
reasons: new Set<string>(),
spare: new Set<string>(),
};
for (const r of reasonsOf[code]!) p.reasons.add(r);
if (f.fallback.startsWith(`${f.prefix}.`))
p.spare.add(f.fallback.slice(f.prefix.length + 1));
byPrefix.set(f.prefix, p);
}
});
it(`${lang}: every admin refusal reason`, () => {
expect(missing(adminReasons.map((r) => `errors.admin.${r}`))).toEqual([]);
expect([...idsUnder(keys, "errors.admin")].filter((r) => !adminReasons.includes(r))).toEqual(
[],
);
for (const [prefix, { suffix, reasons, spare }] of byPrefix)
expect(
[...idsUnder(keys, prefix)].filter(
(r) =>
(!suffix || keys.has(`${prefix}.${r}${suffix}`)) && !spare.has(r) && !reasons.has(r),
),
prefix,
).toEqual([]);
});
}
});